From cc3496cce09c668c0983e4ae20c14034fa817976 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 09:47:12 -0700 Subject: [PATCH 01/26] add durable task scheduler support and runtime parity Consolidate the fork's DTS SDK work onto canonical upstream history, excluding rewritten copies of upstream commits. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/pr-validation.yml | 101 +- .gitignore | 9 +- .golangci.yml | 1 - .vscode/launch.json | 8 - .vscode/settings.json | 4 +- CHANGELOG.md | 87 + Dockerfile | 18 - README.md | 631 +- api/context.go | 97 + api/context_test.go | 16 + api/converter.go | 70 + api/converter_test.go | 135 + api/doc.go | 3 + api/entity.go | 200 + api/entity_test.go | 97 + api/failure.go | 185 + api/failure_test.go | 53 + api/history.go | 349 + api/history_test.go | 69 + api/large_payload.go | 138 + {backend => api}/logger.go | 45 +- api/logger_test.go | 77 + api/management.go | 146 + api/management_test.go | 43 + api/orchestration.go | 353 +- api/orchestration_test.go | 116 + backend/activity.go | 102 - backend/backend.go | 203 - backend/client.go | 207 - backend/executor.go | 534 - backend/orchestration.go | 326 - backend/postgres/README.md | 3 - backend/postgres/postgres.go | 1053 -- backend/postgres/schema.sql | 63 - backend/runtimestate.go | 345 - backend/sqlite/schema.sql | 55 - backend/sqlite/sqlite.go | 1027 -- backend/taskhub.go | 73 - backend/worker.go | 255 - backend/workitem.go | 67 - client/client_grpc.go | 432 +- client/client_grpc_history_test.go | 289 + client/client_grpc_large_payload_test.go | 209 + client/client_grpc_management.go | 384 + client/client_grpc_management_test.go | 164 + client/client_grpc_tag_query_test.go | 379 + client/client_grpc_test.go | 84 + client/client_grpc_wait_test.go | 58 + client/doc.go | 3 + client/entity_batch.go | 75 + client/entity_batch_test.go | 130 + client/entity_client_test.go | 181 + client/errors.go | 39 + client/grpc_errors.go | 129 + client/grpc_errors_test.go | 105 + client/grpc_worker.go | 1370 ++ client/grpc_worker_bench_test.go | 128 + client/grpc_worker_compat.go | 60 + client/grpc_worker_processor.go | 929 ++ client/grpc_worker_test.go | 1793 +++ client/grpc_worker_transport_test.go | 929 ++ client/worker_grpc.go | 212 - client/workitem.go | 10 + cmd/orchestratorvet/README.md | 261 + .../analysis/orchestratorgo/analyzer.go | 81 + .../analysis/orchestratorgo/analyzer_test.go | 250 + .../analysis/orchestratorgo/bench_test.go | 164 + .../analysis/orchestratorgo/checks.go | 677 + .../analysis/orchestratorgo/fixes.go | 265 + .../analysis/orchestratorgo/index.go | 419 + .../analysis/orchestratorgo/loops.go | 143 + .../analysis/orchestratorgo/reach.go | 71 + .../analysis/orchestratorgo/registration.go | 396 + .../analysis/orchestratorgo/rules.go | 113 + .../testdata/src/callgraph/callgraph.go | 126 + .../testdata/src/concurrency/concurrency.go | 122 + .../testdata/src/crosspackage/crosspackage.go | 17 + .../testdata/src/externalio/externalio.go | 108 + .../testdata/src/fixbindings/fixbindings.go | 47 + .../src/fixbindings/fixbindings.go.golden | 47 + .../src/fixbindings/fixbindings_test.go | 25 + .../testdata/src/fixes/fixes.go | 60 + .../testdata/src/fixes/fixes.go.golden | 60 + .../testdata/src/fixesbatch/fixesbatch.go | 18 + .../src/fixesbatch/fixesbatch.go.golden | 16 + .../src/fixesbatchalias/fixesbatchalias.go | 22 + .../fixesbatchalias/fixesbatchalias.go.golden | 20 + .../fixesbatchstandalone.go | 18 + .../fixesbatchstandalone.go.golden | 18 + .../testdata/src/fixesimport/fixesimport.go | 20 + .../src/fixesimport/fixesimport.go.golden | 18 + .../testdata/src/fixesinline/fixesinline.go | 15 + .../src/fixesinline/fixesinline.go.golden | 18 + .../testdata/src/fixespartial/fixespartial.go | 34 + .../src/fixespartial/fixespartial.go.golden | 34 + .../src/fixesstandalone/fixesstandalone.go | 13 + .../fixesstandalone/fixesstandalone.go.golden | 12 + .../fixesstandalonealias.go | 13 + .../fixesstandalonealias.go.golden | 12 + .../registrationhelpers.go | 19 + .../src/github.com/google/uuid/uuid.go | 31 + .../microsoft/durabletask-go/task/task.go | 140 + .../testdata/src/logging/logging.go | 58 + .../testdata/src/loops/loops.go | 134 + .../testdata/src/negative/negative.go | 128 + .../testdata/src/random/random.go | 129 + .../testdata/src/registration/registration.go | 100 + .../src/registrationflow/registrationflow.go | 186 + .../testdata/src/tasknames/tasknames.go | 49 + .../src/tasknamesalias/tasknamesalias.go | 25 + .../src/tasknamesdynamic/tasknamesdynamic.go | 43 + .../tasknamesexternal/tasknamesexternal.go | 23 + .../src/tasknamesglobal/tasknamesglobal.go | 20 + .../tasknamesinterface/tasknamesinterface.go | 19 + .../tasknamesmethodvalue.go | 23 + .../tasknamesparameter/tasknamesparameter.go | 18 + .../tasknamestuplefactory.go | 25 + .../tasknamesunresolved.go | 18 + .../testdata/src/wallclock/wallclock.go | 52 + cmd/orchestratorvet/go.mod | 10 + cmd/orchestratorvet/go.sum | 8 + cmd/orchestratorvet/main.go | 10 + durabletaskscheduler/README.md | 365 + durabletaskscheduler/authentication_test.go | 842 ++ durabletaskscheduler/client.go | 87 + durabletaskscheduler/client_connection.go | 301 + .../client_connection_test.go | 576 + durabletaskscheduler/connection.go | 573 + durabletaskscheduler/connection_test.go | 1169 ++ durabletaskscheduler/options.go | 435 + durabletaskscheduler/options_test.go | 661 + durabletaskscheduler/scheduled_tasks.go | 1642 +++ .../scheduled_tasks_parity_test.go | 1977 +++ durabletaskscheduler/scheduled_tasks_test.go | 544 + durabletaskscheduler/worker.go | 65 + exporthistory/README.md | 239 + exporthistory/activities.go | 274 + exporthistory/activities_test.go | 883 ++ exporthistory/client.go | 438 + exporthistory/client_test.go | 788 ++ exporthistory/doc.go | 77 + exporthistory/entity.go | 364 + exporthistory/entity_test.go | 847 ++ exporthistory/errors.go | 139 + exporthistory/errors_test.go | 263 + exporthistory/lifecycle_test.go | 351 + exporthistory/models.go | 532 + exporthistory/models_test.go | 353 + exporthistory/options.go | 262 + exporthistory/options_test.go | 459 + exporthistory/orchestrations.go | 387 + exporthistory/orchestrations_test.go | 1131 ++ exporthistory/registration.go | 93 + exporthistory/storage.go | 447 + exporthistory/storage_azurite_test.go | 179 + exporthistory/storage_test.go | 626 + exporthistory/versioning_test.go | 181 + go.mod | 73 +- go.sum | 201 +- internal/contextprop/tags.go | 57 + internal/contextprop/tags_test.go | 61 + internal/failure/failure.go | 399 + internal/failure/failure_test.go | 157 + internal/grpcerrors/reasons.go | 39 + internal/helpers/casing.go | 31 + internal/helpers/casing_test.go | 55 + internal/helpers/entity.go | 21 + internal/helpers/entity_ids.go | 52 + internal/helpers/history.go | 238 +- internal/helpers/tracing.go | 251 +- internal/helpers/tracing_test.go | 47 + internal/helpers/worker.go | 27 +- internal/historyconv/history.go | 344 + internal/historyconv/history_test.go | 133 + internal/historyconv/query.go | 79 + internal/historyconv/query_test.go | 329 + internal/historyconv/size.go | 221 + internal/largepayload/bounded.go | 183 + internal/largepayload/bounded_test.go | 486 + internal/largepayload/codec.go | 407 + internal/largepayload/codec_test.go | 465 + internal/protos/orchestrator_service.pb.go | 10853 +++++++++------- .../protos/orchestrator_service_grpc.pb.go | 524 +- internal/tagcodec/tags.go | 95 + main.go | 53 - payload/azure_blob.go | 655 + payload/azure_blob_azurite_test.go | 141 + payload/azure_blob_container_test.go | 403 + payload/azure_blob_test.go | 155 + payload/doc.go | 7 + payload/file.go | 141 + payload/memory.go | 72 + payload/store_test.go | 42 + payload/testdata/blob_tokens.json | 17 + samples/azurefunctions/.gitignore | 3 - .../azurefunctions/HelloCities/function.json | 9 - samples/azurefunctions/SayHello/function.json | 9 - samples/azurefunctions/demo.http | 3 - samples/azurefunctions/extensions.csproj | 12 - samples/azurefunctions/host.json | 13 - samples/azurefunctions/local.settings.json | 7 - samples/azurefunctions/main.go | 49 - samples/azurefunctions/middleware.go | 159 - .../distributedtracing/distributedtracing.go | 145 +- samples/distributedtracing/go.mod | 45 + samples/distributedtracing/go.sum | 90 + .../durabletaskscheduler.go | 136 + samples/entity/entity.go | 266 + samples/exporthistory/exporthistory.go | 187 + samples/externalevents/externalevents.go | 78 +- samples/heterogeneous/heterogeneous.go | 285 - samples/internal/dtssample/dtssample.go | 108 + samples/parallel/parallel.go | 81 +- samples/retries/retries.go | 75 +- samples/sequence/sequence.go | 107 - task/activity.go | 143 +- task/cancel.go | 104 + task/cancel_test.go | 597 + task/context.go | 113 + task/context_test.go | 272 + task/converter_test.go | 195 + task/coroutine.go | 121 + task/custom_status_test.go | 195 + task/doc.go | 3 + task/entity.go | 442 + task/entity_dispatch.go | 249 + task/entity_dispatch_test.go | 707 + task/entity_lock_test.go | 163 + task/entity_object.go | 230 + task/entity_object_test.go | 286 + task/entity_parameters_test.go | 115 + task/entity_test.go | 253 + task/errors.go | 218 + task/eventchannel.go | 141 + task/executor.go | 598 +- task/executor_contract.go | 37 + task/external_event_test.go | 347 + task/failure_contract_test.go | 239 + task/finalization_test.go | 194 + task/history.go | 119 + task/history_test.go | 565 + task/metrics.go | 49 + task/orchestrator.go | 1561 ++- task/orchestrator_test.go | 51 +- task/registry.go | 318 +- task/registry_bench_test.go | 140 + task/registry_test.go | 97 + task/retry.go | 45 + task/retry_observability_test.go | 101 + task/retry_policy_test.go | 388 + task/runservice_patterns_test.go | 178 + task/scheduler.go | 188 + task/scheduler_test.go | 294 + task/select.go | 134 + task/select_test.go | 315 + task/suspension_test.go | 196 + task/task.go | 122 +- task/task_completion_test.go | 26 + task/timer_test.go | 375 + task/versioning.go | 204 + task/versioning_test.go | 494 + task/waitgroup.go | 60 + tests/backend_test.go | 537 - .../durabletaskscheduler/dts_emulator_test.go | 1638 +++ .../dts_export_history_test.go | 430 + .../dts_failure_and_management_test.go | 428 + .../dts_tracing_tree_test.go | 499 + tests/entity_executor_test.go | 865 ++ tests/failurechain/failurechain.go | 88 + tests/grpc/grpc_test.go | 510 - tests/metadata_test.go | 19 +- tests/mocks/Backend.go | 804 -- tests/mocks/Executor.go | 162 - tests/mocks/TaskWorker.go | 157 - tests/mocks/task.go | 128 - tests/orchestrations_test.go | 1492 --- tests/runtimestate_test.go | 410 - tests/shared_test.go | 12 + tests/task_executor_test.go | 4 +- tests/taskhub_test.go | 43 - tests/tracing_test.go | 237 - tests/tracingtree/tracingtree.go | 76 + tests/worker_test.go | 227 - .../PROTO_SOURCE_COMMIT_HASH | 4 +- .../protos/orchestrator_service.proto | 360 +- vendored/durabletask-protobuf/update-proto.sh | 2 +- 286 files changed, 63004 insertions(+), 16518 deletions(-) delete mode 100644 Dockerfile create mode 100644 api/context.go create mode 100644 api/context_test.go create mode 100644 api/converter.go create mode 100644 api/converter_test.go create mode 100644 api/doc.go create mode 100644 api/entity.go create mode 100644 api/entity_test.go create mode 100644 api/failure.go create mode 100644 api/failure_test.go create mode 100644 api/history.go create mode 100644 api/history_test.go create mode 100644 api/large_payload.go rename {backend => api}/logger.go (71%) create mode 100644 api/logger_test.go create mode 100644 api/management.go create mode 100644 api/management_test.go create mode 100644 api/orchestration_test.go delete mode 100644 backend/activity.go delete mode 100644 backend/backend.go delete mode 100644 backend/client.go delete mode 100644 backend/executor.go delete mode 100644 backend/orchestration.go delete mode 100644 backend/postgres/README.md delete mode 100644 backend/postgres/postgres.go delete mode 100644 backend/postgres/schema.sql delete mode 100644 backend/runtimestate.go delete mode 100644 backend/sqlite/schema.sql delete mode 100644 backend/sqlite/sqlite.go delete mode 100644 backend/taskhub.go delete mode 100644 backend/worker.go delete mode 100644 backend/workitem.go create mode 100644 client/client_grpc_history_test.go create mode 100644 client/client_grpc_large_payload_test.go create mode 100644 client/client_grpc_management.go create mode 100644 client/client_grpc_management_test.go create mode 100644 client/client_grpc_tag_query_test.go create mode 100644 client/client_grpc_test.go create mode 100644 client/client_grpc_wait_test.go create mode 100644 client/doc.go create mode 100644 client/entity_batch.go create mode 100644 client/entity_batch_test.go create mode 100644 client/entity_client_test.go create mode 100644 client/errors.go create mode 100644 client/grpc_errors.go create mode 100644 client/grpc_errors_test.go create mode 100644 client/grpc_worker.go create mode 100644 client/grpc_worker_bench_test.go create mode 100644 client/grpc_worker_compat.go create mode 100644 client/grpc_worker_processor.go create mode 100644 client/grpc_worker_test.go create mode 100644 client/grpc_worker_transport_test.go delete mode 100644 client/worker_grpc.go create mode 100644 client/workitem.go create mode 100644 cmd/orchestratorvet/README.md create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/bench_test.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/checks.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/fixes.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/index.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/loops.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/reach.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/registration.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/rules.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/callgraph/callgraph.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/concurrency/concurrency.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/crosspackage/crosspackage.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/externalio/externalio.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings_test.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/example/registrationhelpers/registrationhelpers.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/google/uuid/uuid.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/microsoft/durabletask-go/task/task.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/logging/logging.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/loops/loops.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/negative/negative.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/random/random.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registration/registration.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registrationflow/registrationflow.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknames/tasknames.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesalias/tasknamesalias.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesdynamic/tasknamesdynamic.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesexternal/tasknamesexternal.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesglobal/tasknamesglobal.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesinterface/tasknamesinterface.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesmethodvalue/tasknamesmethodvalue.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesparameter/tasknamesparameter.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamestuplefactory/tasknamestuplefactory.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesunresolved/tasknamesunresolved.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/wallclock/wallclock.go create mode 100644 cmd/orchestratorvet/go.mod create mode 100644 cmd/orchestratorvet/go.sum create mode 100644 cmd/orchestratorvet/main.go create mode 100644 durabletaskscheduler/README.md create mode 100644 durabletaskscheduler/authentication_test.go create mode 100644 durabletaskscheduler/client.go create mode 100644 durabletaskscheduler/client_connection.go create mode 100644 durabletaskscheduler/client_connection_test.go create mode 100644 durabletaskscheduler/connection.go create mode 100644 durabletaskscheduler/connection_test.go create mode 100644 durabletaskscheduler/options.go create mode 100644 durabletaskscheduler/options_test.go create mode 100644 durabletaskscheduler/scheduled_tasks.go create mode 100644 durabletaskscheduler/scheduled_tasks_parity_test.go create mode 100644 durabletaskscheduler/scheduled_tasks_test.go create mode 100644 durabletaskscheduler/worker.go create mode 100644 exporthistory/README.md create mode 100644 exporthistory/activities.go create mode 100644 exporthistory/activities_test.go create mode 100644 exporthistory/client.go create mode 100644 exporthistory/client_test.go create mode 100644 exporthistory/doc.go create mode 100644 exporthistory/entity.go create mode 100644 exporthistory/entity_test.go create mode 100644 exporthistory/errors.go create mode 100644 exporthistory/errors_test.go create mode 100644 exporthistory/lifecycle_test.go create mode 100644 exporthistory/models.go create mode 100644 exporthistory/models_test.go create mode 100644 exporthistory/options.go create mode 100644 exporthistory/options_test.go create mode 100644 exporthistory/orchestrations.go create mode 100644 exporthistory/orchestrations_test.go create mode 100644 exporthistory/registration.go create mode 100644 exporthistory/storage.go create mode 100644 exporthistory/storage_azurite_test.go create mode 100644 exporthistory/storage_test.go create mode 100644 exporthistory/versioning_test.go create mode 100644 internal/contextprop/tags.go create mode 100644 internal/contextprop/tags_test.go create mode 100644 internal/failure/failure.go create mode 100644 internal/failure/failure_test.go create mode 100644 internal/grpcerrors/reasons.go create mode 100644 internal/helpers/casing.go create mode 100644 internal/helpers/casing_test.go create mode 100644 internal/helpers/entity.go create mode 100644 internal/helpers/entity_ids.go create mode 100644 internal/helpers/tracing_test.go create mode 100644 internal/historyconv/history.go create mode 100644 internal/historyconv/history_test.go create mode 100644 internal/historyconv/query.go create mode 100644 internal/historyconv/query_test.go create mode 100644 internal/historyconv/size.go create mode 100644 internal/largepayload/bounded.go create mode 100644 internal/largepayload/bounded_test.go create mode 100644 internal/largepayload/codec.go create mode 100644 internal/largepayload/codec_test.go create mode 100644 internal/tagcodec/tags.go delete mode 100644 main.go create mode 100644 payload/azure_blob.go create mode 100644 payload/azure_blob_azurite_test.go create mode 100644 payload/azure_blob_container_test.go create mode 100644 payload/azure_blob_test.go create mode 100644 payload/doc.go create mode 100644 payload/file.go create mode 100644 payload/memory.go create mode 100644 payload/store_test.go create mode 100644 payload/testdata/blob_tokens.json delete mode 100644 samples/azurefunctions/.gitignore delete mode 100644 samples/azurefunctions/HelloCities/function.json delete mode 100644 samples/azurefunctions/SayHello/function.json delete mode 100644 samples/azurefunctions/demo.http delete mode 100644 samples/azurefunctions/extensions.csproj delete mode 100644 samples/azurefunctions/host.json delete mode 100644 samples/azurefunctions/local.settings.json delete mode 100644 samples/azurefunctions/main.go delete mode 100644 samples/azurefunctions/middleware.go create mode 100644 samples/distributedtracing/go.mod create mode 100644 samples/distributedtracing/go.sum create mode 100644 samples/durabletaskscheduler/durabletaskscheduler.go create mode 100644 samples/entity/entity.go create mode 100644 samples/exporthistory/exporthistory.go delete mode 100644 samples/heterogeneous/heterogeneous.go create mode 100644 samples/internal/dtssample/dtssample.go delete mode 100644 samples/sequence/sequence.go create mode 100644 task/cancel.go create mode 100644 task/cancel_test.go create mode 100644 task/context.go create mode 100644 task/context_test.go create mode 100644 task/converter_test.go create mode 100644 task/coroutine.go create mode 100644 task/custom_status_test.go create mode 100644 task/doc.go create mode 100644 task/entity.go create mode 100644 task/entity_dispatch.go create mode 100644 task/entity_dispatch_test.go create mode 100644 task/entity_lock_test.go create mode 100644 task/entity_object.go create mode 100644 task/entity_object_test.go create mode 100644 task/entity_parameters_test.go create mode 100644 task/entity_test.go create mode 100644 task/errors.go create mode 100644 task/eventchannel.go create mode 100644 task/executor_contract.go create mode 100644 task/external_event_test.go create mode 100644 task/failure_contract_test.go create mode 100644 task/finalization_test.go create mode 100644 task/history.go create mode 100644 task/history_test.go create mode 100644 task/metrics.go create mode 100644 task/registry_bench_test.go create mode 100644 task/registry_test.go create mode 100644 task/retry.go create mode 100644 task/retry_observability_test.go create mode 100644 task/retry_policy_test.go create mode 100644 task/runservice_patterns_test.go create mode 100644 task/scheduler.go create mode 100644 task/scheduler_test.go create mode 100644 task/select.go create mode 100644 task/select_test.go create mode 100644 task/suspension_test.go create mode 100644 task/task_completion_test.go create mode 100644 task/timer_test.go create mode 100644 task/versioning.go create mode 100644 task/versioning_test.go create mode 100644 task/waitgroup.go delete mode 100644 tests/backend_test.go create mode 100644 tests/durabletaskscheduler/dts_emulator_test.go create mode 100644 tests/durabletaskscheduler/dts_export_history_test.go create mode 100644 tests/durabletaskscheduler/dts_failure_and_management_test.go create mode 100644 tests/durabletaskscheduler/dts_tracing_tree_test.go create mode 100644 tests/entity_executor_test.go create mode 100644 tests/failurechain/failurechain.go delete mode 100644 tests/grpc/grpc_test.go delete mode 100644 tests/mocks/Backend.go delete mode 100644 tests/mocks/Executor.go delete mode 100644 tests/mocks/TaskWorker.go delete mode 100644 tests/mocks/task.go delete mode 100644 tests/orchestrations_test.go delete mode 100644 tests/runtimestate_test.go create mode 100644 tests/shared_test.go delete mode 100644 tests/taskhub_test.go delete mode 100644 tests/tracing_test.go create mode 100644 tests/tracingtree/tracingtree.go delete mode 100644 tests/worker_test.go diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml index 2190cf1e..212bac0b 100644 --- a/.github/workflows/pr-validation.yml +++ b/.github/workflows/pr-validation.yml @@ -15,55 +15,61 @@ on: env: # Configure protoc and go grpc plugin - PROTOC_VERSION: "25.x" - PROTOC_GEN_GO: "v1.30" - PROTOC_GEN_GO_GRPC: "v1.3" + PROTOC_VERSION: "36.x" + PROTOC_GEN_GO: "v1.36.12" + PROTOC_GEN_GO_GRPC: "v1.6.2" # A workflow run is made up of one or more jobs that can run sequentially or in parallel jobs: # This workflow contains a single job called "build" build: runs-on: ubuntu-latest + timeout-minutes: 45 strategy: + fail-fast: false matrix: - go-version: [ '1.24.x', '1.23.x' ] + include: + - go-version: '1.25.x' + test-kind: coverage + test-command: >- + go test ./... -count=1 -timeout=20m + -coverpkg=./api,./task,./client,./durabletaskscheduler,./exporthistory,./payload,./internal/contextprop,./internal/failure,./internal/grpcerrors,./internal/helpers,./internal/historyconv,./internal/largepayload,./internal/tagcodec + - go-version: '1.27.x' + test-kind: race + test-command: go test -race ./... -count=1 -timeout=30m - # Boot up a local, clean postgres instance for the postgres tests + # Boot up the local Durable Task Scheduler emulator and Azurite, which back + # the end-to-end and blob storage tests. services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:16 + dts: + image: mcr.microsoft.com/dts/dts-emulator:latest@sha256:361323065a608d605f9d3ae56b854eb11f1c47fcc16845a37f948f03ca9c5fac env: - POSTGRES_PASSWORD: postgres - POSTGRES_USER: postgres - POSTGRES_DB: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 + DTS_TASK_HUB_NAMES: default ports: - # Maps tcp port 5432 on service container to the host - - 5432:5432 - + - 8080:8080 + - 8082:8082 + + azurite: + image: mcr.microsoft.com/azure-storage/azurite:3.37.0 + ports: + - 10000:10000 + steps: # Checks-out your repository under $GITHUB_WORKSPACE - - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: fetch-depth: 1 - name: Setup Go environment - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: ${{ matrix.go-version }} - - name: Install dependencies - run: go get . + - name: Download module dependencies + run: go mod download - name: Install Protoc - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0 + uses: arduino/setup-protoc@f4d5893b897028ff5739576ea0409746887fa536 # v3.0.0 with: version: ${{ env.PROTOC_VERSION }} @@ -78,10 +84,45 @@ jobs: - name: Run go vet run: go vet ./... + - name: Build and test orchestratorvet + run: | + cd cmd/orchestratorvet + go mod download + go test ./... + go build -o "$RUNNER_TEMP/orchestratorvet" . + + - name: Check orchestrators for replay hazards + run: | + go vet -vettool="$RUNNER_TEMP/orchestratorvet" ./... + + - name: Build distributed tracing sample module + working-directory: samples/distributedtracing + run: | + go mod download + go test ./... + - name: Run golangci-lint - uses: golangci/golangci-lint-action@9fae48acfc02a90574d7c304a1758ef9895495fa # v7.0.1 + uses: golangci/golangci-lint-action@d583c34f0599d37dbac4a198b9c83201be380893 # v9.3.0 + with: + version: v2.13.2 + + - name: Wait for service emulators + run: | + curl --fail --retry 30 --retry-delay 1 --retry-all-errors http://127.0.0.1:8082/ + timeout 60 bash -c 'until This project is a work-in-progress and should not be used for production workloads. The public API surface is also not yet stable. The project itself is also in the very early stages and is missing some of the basics, such as contribution guidelines, etc. +The SDK gives you two connections: -## Storage providers +- A **management client** starts orchestrations and reads their state. +- A **worker** runs your registered orchestrators, activities, and entities. -This project includes a [sqlite](https://sqlite.org/) storage provider for persisting app state to disk. +## Features -```go -// Persists state to a file named test.sqlite3. Use "" for in-memory storage. -options := sqlite.NewSqliteOptions("test.sqlite3") -be := sqlite.NewSqliteBackend(options, backend.DefaultLogger()) -``` +The features are in three groups. Group 1 has the most important features. Start with group 1. + +### Group 1: Basic features + +You need these features for almost all applications. + +| Feature | Description | +| --- | --- | +| Orchestrations | Write a workflow as a Go function. DTS saves the progress after each step. | +| Activities | Call a function that does work with side effects, for example an HTTP request. | +| Retries and failure handling | Retry a failed task with a policy. Read the cause from typed errors. | +| Orchestration management | Start, query, wait for, terminate, suspend, resume, and purge an orchestration. | + +### Group 2: Common patterns + +Most applications use one or more of these patterns. + +| Feature | Description | +| --- | --- | +| Fan-out and fan-in | Start many activities in parallel. Then wait for all of the results. | +| Durable timers | Wait for a time period. The timer stays correct after a restart. | +| External events | Stop the orchestration until an external system sends an event. | +| Sub-orchestrations | Call one orchestration from a different orchestration. | +| Durable entities | Keep addressable state. An entity does its operations one at a time. | + +### Group 3: Advanced features + +Use these features for large systems or for special conditions. + +| Feature | Description | +| --- | --- | +| Task versions | Register more than one implementation under the same logical name. | +| Data converters | Replace the default JSON serialization of your payloads. | +| Large payloads | Keep large payloads in Azure Blob Storage and send a reference token. | +| Tags and trace context | Attach user tags to actions. Send the caller trace context to DTS. | +| Distributed tracing | Emit OpenTelemetry spans from your process. DTS emits the durable spans. | +| Replay-hazard analysis | Find nondeterministic orchestrator code with `cmd/orchestratorvet`. | +| History export (preview) | Copy terminal orchestration histories to Azure Blob Storage. | + +## Quick start + +This project needs Go 1.25 or later. + +1. Start the [DTS emulator](https://learn.microsoft.com/azure/azure-functions/durable/durable-task-scheduler/quickstart-durable-task-scheduler). Use any OCI runtime: + + ```bash + docker run -d -p 8080:8080 -p 8082:8082 \ + -e DTS_TASK_HUB_NAMES=default \ + mcr.microsoft.com/dts/dts-emulator:latest + ``` + + The dashboard runs on port `8082`. + +2. Set the connection string: -Additional storage providers can be created by extending the `Backend` interface. + ```bash + export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" + ``` -## Creating the standalone gRPC sidecar +3. Run a sample: -See the `main.go` file for an example of how to create a standalone gRPC sidecar that embeds the Durable Task engine. In short, you must create an `Backend` (for storage), an `Executor` (for executing user code), and host them as a `TaskHubWorker`. + ```bash + go run ./samples/durabletaskscheduler + ``` -The following code creates a `TaskHub` worker with sqlite `Backend` and a gRPC `Executor` implementations. +## Connection to DTS + +The [`durabletaskscheduler`](./durabletaskscheduler) package is the integration surface. The package validates your connection string and supplies Azure token credentials. It also owns the management connection and the worker connection separately. The worker stream recovers after a network failure. ```go -// Use the default logger or provide your own -logger := backend.DefaultLogger() - -// Configure the sqlite backend that will store the runtime state -sqliteOptions := sqlite.NewSqliteOptions(sqliteFilePath) -be := sqlite.NewSqliteBackend(sqliteOptions, logger) - -// Create a gRPC server that the language SDKs will connect to -grpcServer := grpc.NewServer() -executor := backend.NewGrpcExecutor(grpcServer, be, logger) - -// Construct and start the task hub worker object, which polls the backend for new work -orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) -activityWorker := backend.NewActivityTaskWorker(be, executor, logger) -taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) -taskHubWorker.Start(context.Background()) - -// Start listening. -lis, _ := net.Listen("tcp", "localhost:4001") -fmt.Printf("server listening at %v\n", lis.Addr()) -grpcServer.Serve(lis) +options, err := durabletaskscheduler.NewOptionsFromConnectionString( + os.Getenv("DTS_CONNECTION_STRING")) +if err != nil { + return err +} + +registry := task.NewTaskRegistry() +if err := registry.AddOrchestratorN("ActivitySequence", ActivitySequence); err != nil { + return err +} +if err := registry.AddActivityN("SayHello", SayHello); err != nil { + return err +} + +logger := api.DefaultLogger() + +// The management client schedules orchestrations and reads their state. +client, err := durabletaskscheduler.NewClient(ctx, options, logger) +if err != nil { + return err +} +defer client.Close() + +// The worker executes the registered orchestrators and activities. +worker, err := durabletaskscheduler.NewWorker( + options, registry, logger, durabletaskclient.WithAutoWorkItemFilters()) +if err != nil { + return err +} +if err := worker.Start(ctx); err != nil { + return err +} +defer worker.Shutdown(ctx) + +id, err := client.ScheduleNewOrchestration(ctx, "ActivitySequence") +if err != nil { + return err +} +metadata, err := client.WaitForOrchestrationCompletion(ctx, id) ``` -Note that the Durable Task gRPC service implementation is designed to serve one client at a time, just like with any sidecar architecture. Scale out is achieved by adding new pod replicas that contain both the app process and the sidecar (connected to a common database). +In this example, `durabletaskclient` is the import alias for the [`client`](./client) package. + +Management clients also read the orchestration history and the recurring interval schedules. These APIs do not expose the generated protobuf messages. + +For more information, read the [DTS transport guide and feature matrix](./durabletaskscheduler/README.md). -### Language SDKs for gRPC +## Samples -The Durable Task Framework for Go currently supports writing orchestrations in the following languages: +Each sample connects to the task hub in `DTS_CONNECTION_STRING`. Set that variable first. Then run `go run ./samples/`. -| Language/Stack | Package | Project Home | Samples | -| - | - | - | - | -| .NET | [![NuGet](https://img.shields.io/nuget/v/Microsoft.DurableTask.Client.svg?style=flat)](https://www.nuget.org/packages/Microsoft.DurableTask.Client/) | [GitHub](https://github.com/microsoft/durabletask-dotnet) | [Samples](https://github.com/microsoft/durabletask-dotnet/tree/main/samples) | -| Java | [![Maven Central](https://img.shields.io/maven-central/v/com.microsoft/durabletask-client?label=durabletask-client)](https://search.maven.org/artifact/com.microsoft/durabletask-client) | [GitHub](https://github.com/microsoft/durabletask-java) | [Samples](https://github.com/microsoft/durabletask-java/tree/main/samples/src/main/java/io/durabletask/samples) | -| Python | [![PyPI version](https://badge.fury.io/py/durabletask.svg)](https://badge.fury.io/py/durabletask) | [GitHub](https://github.com/microsoft/durabletask-python) | [Samples](https://github.com/microsoft/durabletask-python/tree/main/examples) | +| Sample | Description | +| --- | --- | +| [durabletaskscheduler](./samples/durabletaskscheduler) | Connect to DTS and call activities in sequence. | +| [parallel](./samples/parallel) | Run activities in parallel. | +| [externalevents](./samples/externalevents) | Wait for an event from an external system. | +| [retries](./samples/retries) | Retry an activity after a failure. | +| [entity](./samples/entity) | Use durable entities. | +| [exporthistory](./samples/exporthistory) | Export orchestration histories to Azure Blob Storage. | +| [distributedtracing](./samples/distributedtracing) | Send trace data to an OpenTelemetry collector. | -More language SDKs are planned to be added in the future. In particular, SDKs for Python and JavaScript/TypeScript. Anyone can theoretically create an SDK using a language that supports gRPC. However, there is not yet a guide for how to do this, so developers would need to reference existing SDK code as a reference. Starting with the Java implementation is recommended. The gRPC API is defined [here](https://github.com/microsoft/durabletask-protobuf). +Two samples need more steps: -## Embedded orchestrations +- `distributedtracing` is a separate Go module. Run it with `cd samples/distributedtracing && go run .`. +- `exporthistory` also needs `EXPORT_STORAGE_CONNECTION_STRING`. It accepts the optional variable `EXPORT_CONTAINER`. -It's also possible to create orchestrations in Go and run them in the local process. The full set of Durable Task features is not yet available as part of the Go SDK, but will be added over time. +Most samples use the shared helper [`samples/internal/dtssample`](./samples/internal/dtssample). The helper reads `DTS_CONNECTION_STRING`, opens the client, and starts the worker. The `exporthistory` sample does not use the helper. That sample needs the client before it registers the export system tasks. -> You can find code samples in the [samples](./samples/) directory. -> To run them, get into the folder of each sample and run `go run .` +## Orchestrations in Go -### Activity sequence example +### Pattern 1: Activity sequence -Activity sequences like the following are the simplest and most common pattern used in the Durable Task Framework. +An activity sequence is the most simple pattern. The orchestrator calls the activities one after the other. ```go -// ActivitySequenceOrchestrator makes three activity calls in sequence and results the results -// as an array. +// ActivitySequenceOrchestrator makes three activity calls in sequence. +// It returns the results as an array. func ActivitySequenceOrchestrator(ctx *task.OrchestrationContext) (any, error) { var helloTokyo string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("Tokyo")).Await(&helloTokyo); err != nil { + if err := ctx.CallActivity("SayHello", task.WithActivityInput("Tokyo")).Await(&helloTokyo); err != nil { return nil, err } var helloLondon string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("London")).Await(&helloLondon); err != nil { + if err := ctx.CallActivity("SayHello", task.WithActivityInput("London")).Await(&helloLondon); err != nil { return nil, err } var helloSeattle string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("Seattle")).Await(&helloSeattle); err != nil { + if err := ctx.CallActivity("SayHello", task.WithActivityInput("Seattle")).Await(&helloSeattle); err != nil { return nil, err } return []string{helloTokyo, helloLondon, helloSeattle}, nil } -// SayHelloActivity can be called by an orchestrator function and will return a friendly greeting. +// SayHelloActivity returns a greeting. An orchestrator function can call it. func SayHelloActivity(ctx task.ActivityContext) (any, error) { var input string if err := ctx.GetInput(&input); err != nil { @@ -106,28 +188,28 @@ func SayHelloActivity(ctx task.ActivityContext) (any, error) { } ``` -You can find the full sample [here](./samples/sequence). +Full sample: [samples/durabletaskscheduler](./samples/durabletaskscheduler). -### Fan-out / fan-in execution example +### Pattern 2: Fan-out and fan-in -The next most common pattern is "fan-out / fan-in" where multiple activities are run in parallel, as shown in the snippet below (note that the `GetDevicesToUpdate` and `UpdateDevice` activity definitions are left out of the snippet below for brevity): +In this pattern, the orchestrator starts many activities in parallel. Then it waits for all of the results. This example does not show the `GetDevicesToUpdate` and `UpdateDevice` activities. ```go -// UpdateDevicesOrchestrator is an orchestrator that runs activities in parallel +// UpdateDevicesOrchestrator runs activities in parallel. func UpdateDevicesOrchestrator(ctx *task.OrchestrationContext) (any, error) { - // Get a dynamic list of devices to perform updates on + // Get a dynamic list of devices to update. var devices []string - if err := ctx.CallActivity(GetDevicesToUpdate).Await(&devices); err != nil { + if err := ctx.CallActivity("GetDevicesToUpdate").Await(&devices); err != nil { return nil, err } - // Start a dynamic number of tasks in parallel, not waiting for any to complete (yet) + // Start a dynamic number of tasks in parallel. Do not wait for the results now. tasks := make([]task.Task, len(devices)) for i, id := range devices { - tasks[i] = ctx.CallActivity(UpdateDevice, task.WithActivityInput(id)) + tasks[i] = ctx.CallActivity("UpdateDevice", task.WithActivityInput(id)) } - // Now that all are started, wait for them to complete and then return the success rate + // All tasks are started. Wait for the results and calculate the success rate. successCount := 0 for _, task := range tasks { var succeeded bool @@ -140,19 +222,20 @@ func UpdateDevicesOrchestrator(ctx *task.OrchestrationContext) (any, error) { } ``` -The full sample can be found [here](./samples/parallel). +Full sample: [samples/parallel](./samples/parallel). -### External orchestration inputs (events) example +### Pattern 3: External events -Sometimes orchestrations need asynchronous input from external systems. For example, an approval workflow may require a manual approval signal from an authorized user. Or perhaps an orchestration pauses and waits for a command from an operator. The `WaitForSingleEvent` method can be used in an orchestrator function to pause execution and wait for such inputs. You an even specify a timeout value indicating how long to wait for the input before resuming execution (use `-1` to indicate infinite timeout). +An orchestration can wait for input from an external system. For example, an approval workflow waits for a signal from a user. Use the `WaitForSingleEvent` method to stop the orchestration until the event comes. + +Set a timeout value to limit the wait time. Use `-1` for an infinite timeout. ```go -// ExternalEventOrchestrator is an orchestrator function that blocks for 30 seconds or -// until a "Name" event is sent to it. +// ExternalEventOrchestrator waits 30 seconds for a "Name" event. func ExternalEventOrchestrator(ctx *task.OrchestrationContext) (any, error) { var nameInput string if err := ctx.WaitForSingleEvent("Name", 30*time.Second).Await(&nameInput); err != nil { - // Timeout expired + // The timeout expired. return nil, err } @@ -160,39 +243,75 @@ func ExternalEventOrchestrator(ctx *task.OrchestrationContext) (any, error) { } ``` -Sending an event to a waiting orchestration can be done using the `RaiseEvent` method of the task hub client. These events are durably buffered in the orchestration state and are consumed as soon as the target orchestration calls `WaitForSingleEvent` with a matching event name. The following code shows how to use the `RaiseEvent` method to send an event with a payload to a running orchestration. See [Managing local orchestrations](#managing-local-orchestrations) for more information on how to interact with local orchestrations in Go. +Use the `RaiseEvent` method of the client to send an event. ```go -id, _ := client.ScheduleNewOrchestration(ctx, ExternalEventOrchestrator) +id, _ := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrator") -// Prompt the user for their name and send that to the orchestrator +// Ask the user for a name and send the name to the orchestrator. go func() { fmt.Println("Enter your first name: ") var nameInput string fmt.Scanln(&nameInput) - + client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(nameInput)) }() ``` -The full sample can be found [here](./samples/externalevents). +DTS keeps each event in the orchestration state. The orchestration reads the event when it calls `WaitForSingleEvent` with the same event name. + +Two rules control the order: + +- If more than one live wait uses the same event name, the newest wait gets the next event. This rule is last in, first out. It agrees with the Durable Task .NET replay contract. +- If an event arrives before any wait, DTS keeps the event. The orchestration then reads these events in arrival order. This rule is first in, first out. -### Managing local orchestrations +Full sample: [samples/externalevents](./samples/externalevents). -The following code snippet provides an example of how you can configure and run orchestrations. The `TaskRegistry` type allows you to register orchestrator and activity functions, and the `TaskHubClient` allows you to start, query, terminate, suspend, resume, and wait for orchestrations to complete. +### Pattern 4: Retries and failure handling -The code snippet below demonstrates how to register and start a new instance of the `ActivitySequenceOrchestrator` orchestrator and wait for it to complete. The initialization of the client and worker are left out for brevity. +An activity or a sub-orchestration that fails returns a `*task.TaskFailedError`. An entity call that fails returns a `*task.EntityOperationFailedError`. Both errors contain an `api.FailureDetails` value. This value holds the error type, the message, the stack trace, the inner failure, the non-retriable marker, and the custom properties. ```go -r := task.NewTaskRegistry() -r.AddOrchestrator(ActivitySequenceOrchestrator) -r.AddActivity(SayHelloActivity) +err := ctx.CallActivity("ChargeCard").Await(nil) +var failed *task.TaskFailedError +if errors.As(err, &failed) { + fmt.Printf("%s failed: %s\n", failed.TaskName, failed.FailureDetails) +} +``` -ctx := context.Background() -client, worker := Init(ctx, r) -defer worker.Shutdown(ctx) +Attach a retry policy to call a failed task again: + +```go +policy := &task.RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: time.Second, + Handle: func(retry task.RetryContext) bool { + return !retry.LastFailure.IsCausedBy(api.ErrorTypeActivityTaskNotFound) + }, +} +``` -id, err := client.ScheduleNewOrchestration(ctx, ActivitySequenceOrchestrator) +Obey these rules when you write a retry handler: + +- The handler runs again during replay. Do not do I/O in the handler. Do not read the wall clock in the handler. +- The retry options copy the policy when you make the option. Validation does not change your policy value. +- If the next backoff goes past `RetryTimeout`, the SDK stops the retries. The SDK makes this decision from the failure event, so the result is stable across replay and redelivery. +- Some failures are non-retriable and skip your handler. Missing activity registrations, missing entity registrations, and version mismatches are non-retriable. + +`OrchestrationContext.Context()` is separate from the host context of the worker. It holds only the persisted orchestration identity and context fields. It never holds host values, deadlines, or cancellation. Use durable timers and task cancellation scopes to control the orchestration. Use `ctx.Logger()` for replay-safe logging. Use the activity context for host cancellation and for outbound I/O. + +Full sample: [samples/retries](./samples/retries). + +### Orchestration management + +Use a `TaskRegistry` to register your orchestrator, activity, and entity functions. Then use the client from `durabletaskscheduler.NewClient` to control the orchestrations. + +```go +r := task.NewTaskRegistry() +r.AddOrchestratorN("ActivitySequence", ActivitySequenceOrchestrator) +r.AddActivityN("SayHello", SayHelloActivity) + +id, err := client.ScheduleNewOrchestration(ctx, "ActivitySequence") if err != nil { panic(err) } @@ -205,24 +324,145 @@ if err != nil { fmt.Printf("orchestration completed: %v\n", metadata) ``` -Each sample linked above has a full implementation you can use as a reference. +This example does not show the client and worker setup. Read [Connection to DTS](#connection-to-dts) for that code. + +The client also does these operations, if the connected service supplies them: + +- Query the instances with a limit. List the instance IDs. +- Restart and rewind an orchestration. +- Purge in a batch or with a filter. +- Terminate an orchestration immediately. +- Read the tags and the worker capabilities. + +To read a long history, use `StreamOrchestrationHistory`. This method reads the history one part at a time. If you buffer the history instead, the SDK applies a validated event cap. + +## Durable entities + +A durable entity is an addressable object that holds state. An entity does its operations one at a time. + +```go +type Counter struct { + task.EntityObjectBase[CounterState] +} + +type CounterState struct { + Value int `json:"value"` +} + +func (counter *Counter) Add(amount int) { + counter.State().Value += amount +} + +func (counter *Counter) Get() int { + return counter.State().Value +} + +counterFactory := task.NewEntityObjectFactory[CounterState, *Counter]( + func(task.EntityFactoryContext) (*Counter, error) { + return new(Counter), nil + }, +) +registry.AddEntityFactoryN("counter", counterFactory) +counter := api.NewEntityID("counter", "orders") + +client.SignalEntity(ctx, counter, "Add", api.WithSignalInput(1)) + +registry.AddOrchestratorN("read-counter", func(ctx *task.OrchestrationContext) (any, error) { + var value int + err := ctx.CallEntity(counter, "Get").Await(&value) + return value, err +}) +``` + +The SDK supports raw entity functions, state-struct dispatch, and persistent entity objects with separate durable state. Entity factories run once per operation batch, may run concurrently across batches, and can capture dependencies. A factory can return batch cleanup, and an entity object can implement `task.EntityBatchCloser`. Shared functions registered with `AddEntityN` must be thread-safe. Entity IDs use the compact JSON form `"@name@key"`. Reflected operations can use `task.OptionalEntityInput[T]` when input is optional. + +The SDK also supports scheduled signals, orchestration calls, entity-to-entity signals, queries, cleanup, and ordered critical sections across more than one entity. + +Entity names and operation names are matched case-insensitively using the same invariant rule as the .NET SDK, so a name resolves to the same entity in both SDKs. + +The DTS worker accepts the legacy `EntityBatchRequest` work item and the current `EntityRequestV2` work item. + +Full sample: [samples/entity](./samples/entity). + +## Task versions + +An orchestrator or an activity can have more than one implementation under the same logical name. The registry identity contains the name and the version. The registry ignores letter case. + +```go +registry.AddOrchestratorNVersion("orders", "v1", ordersV1) +registry.AddOrchestratorNVersion("orders", "v2", ordersV2) +registry.AddActivityNVersion("charge", "v2", chargeV2) +``` + +Dispatch obeys these rules: + +- An exact match of the name and the version has priority. +- A versioned request uses an unversioned registration only if the logical name has no versioned registrations. +- An activity uses the version of its parent orchestration. To select a different version, supply `task.WithActivityVersion`. To request the unversioned activity, supply `""`. +- A sub-orchestration uses `VersioningOptions.DefaultVersion`. You can override this value. +- `task.WithContinueAsNewVersion` moves the next execution to a different version. This change happens at a deterministic ContinueAsNew boundary. To go back to an unversioned registration, use `task.UnversionedTaskVersion`. + +Set `durabletaskscheduler.Options.Versioning` to configure the management client and the worker together. The reject and fail strategies for a version mismatch stay available for rolling deployments. + +## Data converters + +`api.DataConverter` controls how the SDK serializes your payloads. The default converter is `api.JSONDataConverter`. It keeps the existing `encoding/json` wire format. + +Set `durabletaskscheduler.Options.DataConverter` one time. The SDK gives the converter to the management client and to the worker. + +The converter applies to the typed payloads. These payloads include the orchestration, activity, entity, event, status, management, metadata, and ContinueAsNew values. + +The converter does not apply to these values: + +- The `WithRaw*` options. +- The serialized metadata fields. +- The failure metadata. +- The large-payload reference descriptors. + +The SDK does not save the identity of the converter. A new converter must continue to decode the payloads from your earlier deployments. + +## Tags, trace context, and turn limits + +Use `api.WithTags`, `task.WithActivityTags`, and `task.WithSubOrchestrationTags` to attach user tags. An activity and a sub-orchestration inherit the tags of the parent orchestration. A tag on the action has priority over an inherited tag. The completion actions carry the current tags, so ContinueAsNew keeps them. + +The client sends the sampled caller trace context when it schedules an orchestration or signals an entity. The worker adds separate action trace contexts for the service-owned activity and sub-orchestration spans. The worker does not emit duplicate local Durable Task spans. A legacy entity operation request also sends its operation trace context to the entity actions. The current DTS V2 entity request does not carry that source field. + +Use `task.OrchestrationOptions.MaxEventsPerTurn` to limit the new events in one turn. If the worker uses only part of a batch, it sets `numEventsProcessed`. DTS then keeps the remaining events for the next replay. This count obeys the DTS work-item rules. The orchestration control markers do not count against the limit. -## Distributed tracing support +To control instance-ID reuse, configure `api.OrchestrationIDReusePolicy.DedupeStatuses`: -The Durable Task Framework for Go supports publishing distributed traces to any configured [Open Telemetry](https://opentelemetry.io/)-compatible exporter. Simply use [`otel.SetTracerProvider(tp)`](https://pkg.go.dev/go.opentelemetry.io/otel#SetTracerProvider) to register a global `TracerProvider` as part of your application startup and the task hub worker will automatically use it to emit OLTP trace spans. +| Value | Result | +| --- | --- | +| A status in the list | The new start is rejected. | +| A status that is not in the list | DTS can terminate the instance and replace it. | +| A nil slice | The service default applies. | +| An empty non-nil slice | Replacement is permitted for every reusable status. | -The following example code shows how you can configure distributed trace collection with [Zipkin](https://zipkin.io/), a popular open source distributed tracing system. The example assumes Zipkin is running locally, as shown in the code. +## Large payloads + +To externalize large payloads, set `durabletaskscheduler.Options.LargePayloads` to an `*api.LargePayloadOptions` value. This value holds an `api.LargePayloadStore` that saves the bytes. It also holds an `api.LargePayloadResolver` that reads the bytes back. The SDK applies the option to the management client and to the worker. + +The [`payload`](./payload) package includes Azure Blob Storage support. It emits the same self-describing `blob:v2` tokens as the .NET SDK. + +## Distributed tracing + +The SDK sends the W3C trace context of a sampled caller when it schedules an orchestration. DTS owns the spans for the orchestrations, activities, timers, and sub-orchestrations. Your application code can use standard [OpenTelemetry](https://opentelemetry.io/) instrumentation. Use it for caller spans, custom activity spans, and outbound dependencies. + +This example sends the traces of your process to an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) over OTLP/HTTP. Configure the DTS telemetry separately for the service-owned spans. ```go -func ConfigureZipkinTracing() (*trace.TracerProvider, error) { - // Inspired by this sample: https://github.com/open-telemetry/opentelemetry-go/blob/main/example/zipkin/main.go - exp, err := zipkin.New("http://localhost:9411/api/v2/spans") +func ConfigureOTLPTracing(ctx context.Context) (*trace.TracerProvider, error) { + exp, err := otlptracehttp.New( + ctx, + otlptracehttp.WithEndpoint("localhost:4318"), + otlptracehttp.WithInsecure(), + ) if err != nil { return nil, err } - // NOTE: The simple span processor is not recommended for production. - // Instead, the batch span processor should be used for production. + // NOTE: Do not use the simple span processor in production. + // Use the batch span processor in production. processor := trace.NewSimpleSpanProcessor(exp) // processor := trace.NewBatchSpanProcessor(exp) @@ -239,133 +479,172 @@ func ConfigureZipkinTracing() (*trace.TracerProvider, error) { } ``` -You can find this code in the [distributedtracing](./samples/distributedtracing) sample. The following is a screenshot showing the trace for the sample's orchestration, which calls an activity, creates a 2-second durable timer, and uses another activity to make an HTTP request to bing.com: +The [distributed tracing sample](./samples/distributedtracing) starts a caller span before `ScheduleNewOrchestration`. DTS then joins its service-side spans to the same trace. The sample also instruments an HTTP request from an activity. -![image](https://user-images.githubusercontent.com/2704139/205171291-8d12d6fe-5d4f-40c7-9a48-2586a4c4af49.png) +## History export (preview) -Note that each orchestration is represented as a single span with activities, timers, and sub-orchestrations as child spans. The generated spans contain a variety of attributes that include information such as orchestration instance IDs, task names, task IDs, etc. +> [!WARNING] +> This package is a preview. The exported API can change without a major version. The saved shape of the `ExportJob` entity state and the names of the system tasks can also change. Extended sessions are not supported. -## Cloning this repository +The [`exporthistory`](./exporthistory) package copies terminal orchestration histories to Azure Blob Storage. The output format is gzip-compressed JSONL. -Clone the repository as you normally would: +The package uses a durable entity for the job state. It also uses an operation orchestrator, an export orchestrator, and two activities. One activity lists the instances. The other activity exports them. -```bash -git clone https://github.com/microsoft/durabletask-go +```go +store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ + ConnectionString: storageConnectionString, + ContainerName: "history-exports", +}) +err = exporthistory.Register(registry, exporthistory.WorkerOptions{ + Source: taskHubClient, // supplies ListInstanceIDs and orchestration history + Store: store, +}) +worker, err := durabletaskscheduler.NewWorker(options, registry, logger, + durabletaskclient.WithAutoWorkItemFilters(), + exporthistory.WithExportHistory(), +) + +exportClient, err := exporthistory.NewClient(taskHubClient, exporthistory.ClientOptions{ + ContainerName: "history-exports", +}) +job, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ + Mode: exporthistory.ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, +}) +description, err := job.Describe(ctx) ``` -The protocol buffer definitions used to generate the gRPC bindings are vendored under [`vendored/durabletask-protobuf/protos`](./vendored/durabletask-protobuf/protos). See [`vendored/durabletask-protobuf/README.md`](./vendored/durabletask-protobuf/README.md) for details on how to refresh them from the upstream [microsoft/durabletask-protobuf](https://github.com/microsoft/durabletask-protobuf) repository. +The package supports batch jobs and continuous jobs. It also supports durable checkpoints, terminal-status filters, and a per-batch instance limit. It collects the failures. It returns typed errors for validation, not-found, and invalid-transition conditions. -## Building the project +Each successful `Create` reserves a new generation-specific orchestration ID, exposed as `description.OrchestratorInstanceID`. Only the current generation can change job state, though older work can briefly remain in flight. Recreation retains previous generations' orchestration histories. `Delete` captures the ID of the generation it removes and cleans only that ID, so concurrent recreation is safe. Legacy export jobs and mixed .NET/older Go export-history workers are not supported. -This project requires go v1.19.x or greater. You can build a standalone executable by simply running `go build` at the project root. +Each exported object is an opaque gzip file. The name ends with `.jsonl.gz`. The content type is `application/gzip`. The object has no `Content-Encoding` header. Every reader therefore gets the bytes that the object name promises. -### Generating protobuf +For more information, read the [export history guide](./exporthistory/README.md). See also the [sample](./samples/exporthistory). -Use the following command to regenerate the protobuf bindings from the vendored proto file. Use this whenever updating the proto file under [`vendored/durabletask-protobuf/protos`](./vendored/durabletask-protobuf/protos). +## Development + +### Clone the repository ```bash -# NOTE: assumes the .proto file defines: option go_package = "/internal/protos" -protoc --go_out=. --go-grpc_out=. -I vendored/durabletask-protobuf/protos orchestrator_service.proto +git clone https://github.com/microsoft/durabletask-go ``` -### Generating mocks for testing +The protocol buffer definitions are in [`vendored/durabletask-protobuf/protos`](./vendored/durabletask-protobuf/protos). To get a new version from the upstream [microsoft/durabletask-protobuf](https://github.com/microsoft/durabletask-protobuf) repository, read [`vendored/durabletask-protobuf/README.md`](./vendored/durabletask-protobuf/README.md). + +### Build the project -Test mocks were generated using [mockery](https://github.com/vektra/mockery). Use the following command at the project root to regenerate the mocks. +This project needs Go 1.25 or later. This project is a library. To build all of the packages, run this command in the project root: ```bash -mockery --dir ./backend --name="^Backend|^Executor|^TaskWorker" --output ./tests/mocks --with-expecter +go build ./... ``` -## Running tests +### Generate the protobuf code -All automated tests are under `./tests`. A separate test package hierarchy was chosen intentionally to prioritize [black box testing](https://en.wikipedia.org/wiki/Black-box_testing). This strategy also makes it easier to catch accidental breaking API changes. +Run this command after you change the `.proto` file: + +```bash +# NOTE: The .proto file must define: option go_package = "/internal/protos" +protoc --go_out=. --go-grpc_out=. -I vendored/durabletask-protobuf/protos orchestrator_service.proto +``` -Run tests with the following command. +### Run the tests + +The package tests are beside the code that they exercise. The `./tests` directory holds [black box tests](https://en.wikipedia.org/wiki/Black-box_testing). These tests find accidental changes to the public API. The `./tests` directory holds deterministic runtime tests that use hand-built histories. The `./tests/durabletaskscheduler` directory holds the end-to-end tests for a live scheduler. + +Run the full suite with these commands: ```bash -go test ./tests/... -coverpkg ./api,./task,./client,./backend/...,./internal/helpers +go test ./... -count=1 -coverpkg=./api,./task,./client,./durabletaskscheduler,./exporthistory,./payload,./internal/contextprop,./internal/failure,./internal/grpcerrors,./internal/helpers,./internal/historyconv,./internal/largepayload,./internal/tagcodec +(cd cmd/orchestratorvet && go test ./...) +(cd samples/distributedtracing && go test ./...) ``` -## Running integration tests +The DTS tests and the Azurite tests skip themselves if their environment variables are empty. Read [Run the services locally](#run-the-services-locally) for these variables. PR validation runs the full suite against both services. It then runs the suite again with the race detector on the most recent supported Go version. + +### Test doubles + +This repository has no generated mock package. DTS is the only supported runtime, so the useful fakes are the generated gRPC surfaces. The tests use small hand-written fakes: -You can run pre-built container images to run full integration tests against the durable task host over gRPC. +- The client tests embed `protos.UnimplementedTaskHubSidecarServiceServer`. They serve it over [`bufconn`](https://pkg.go.dev/google.golang.org/grpc/test/bufconn). Other client tests embed `protos.TaskHubSidecarServiceClient` and override only the RPCs under test. `TaskHubSidecarService` is the generated name of the DTS gRPC service. The name does not imply a sidecar deployment. +- The worker tests and the executor tests implement `task.Executor` and `task.EntityExecutor`. +- The orchestration tests supply hand-built histories to `task.NewTaskExecutor`. -### .NET Durable Task client SDK tests +### Check the orchestrators for replay hazards -Use the following docker command to run tests against a running worker. +DTS replays your orchestrator code from the history on each turn. Your orchestrator must therefore be deterministic and free of side effects. + +`cmd/orchestratorvet` is a driver for the `orchestratorgo` analyzer. The driver is compatible with [`go vet`](https://pkg.go.dev/cmd/vet). The analyzer examines the orchestrators that a package registers with `task.TaskRegistry`. ```bash -docker run -e GRPC_HOST="host.docker.internal" cgillum/durabletask-dotnet-tester:0.5.0-beta +cd cmd/orchestratorvet +go build -o ../../bin/orchestratorvet . +cd ../.. +go vet -vettool=$PWD/bin/orchestratorvet ./... ``` -Note that the test assumes the gRPC server can be reached over `localhost` on port `4001` on the host machine. These values can be overridden with the following environment variables: +The analyzer starts at each `AddOrchestrator`, `AddOrchestratorN`, `AddOrchestratorVersion`, and `AddOrchestratorNVersion` call that it can resolve. It then follows the call graph. The graph includes named functions and methods in the same package. It also includes resolvable function variables and nested function literals. -* `GRPC_HOST`: Use this to change from the default `127.0.0.1` to some other value, for example `host.docker.internal`. -* `GRPC_PORT`: Set this environment variable to change the default port from `4001` to something else. +The analyzer reports these hazards: -If successful, you should see output that looks like the following: +- Wall-clock reads and host timers. +- Nondeterministic identifier sources and random sources. +- Unsafe parallelism and synchronization. +- Direct filesystem, network, process, and environment I/O. +- Replay-unsafe logging. +- Unbounded loops that it proves do not make progress. +- Task names that a complete registration set proves are missing. +- Registration forms that `task.TaskRegistry` rejects or that give an unstable name. -``` -Test run for /root/out/bin/Debug/Microsoft.DurableTask.Tests/net6.0/Microsoft.DurableTask.Tests.dll (.NETCoreApp,Version=v6.0) -Microsoft (R) Test Execution Command Line Tool Version 17.3.1 (x64) -Copyright (c) Microsoft Corporation. All rights reserved. - -Starting test execution, please wait... -A total of 1 test files matched the specified pattern. -[xUnit.net 00:00:00.00] xUnit.net VSTest Adapter v2.4.3+1b45f5407b (64-bit .NET 6.0.10) -[xUnit.net 00:00:00.82] Discovering: Microsoft.DurableTask.Tests -[xUnit.net 00:00:00.90] Discovered: Microsoft.DurableTask.Tests -[xUnit.net 00:00:00.90] Starting: Microsoft.DurableTask.Tests - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.ExternalEvents(eventCount: 100) [6 s] - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.ExternalEvents(eventCount: 1) [309 ms] - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.LongTimer [8 s] - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.SubOrchestration [1 s] - ... - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.ActivityFanOut [914 ms] -[xUnit.net 00:01:01.04] Finished: Microsoft.DurableTask.Tests - Passed Microsoft.DurableTask.Tests.OrchestrationPatterns.SingleActivity_Async [365 ms] - -Test Run Successful. -Total tests: 33 - Passed: 33 - Total time: 1.0290 Minutes -``` +The analyzer supplies an `analysis.SuggestedFix` for `time.Now()` and for `go func() { ... }()`. The fixes change them to `ctx.CurrentTimeUtc` and `ctx.Go`. Both `gopls` and `go vet -fix` can apply these fixes. The other diagnostics have no fix, because no single rewrite is always correct. + +The analyzer reports only the hazards that it proves. It stays silent for all other code. You can therefore enable it on an existing codebase without many diagnostics. -## Running locally +The analyzer ignores the files that end in `_test.go`. Tests frequently register invalid or nondeterministic orchestrators on purpose. To include these files, use the `-orchestratorgo.test-files` flag. -You can run the engine locally by pressing `F5` in [Visual Studio Code](https://code.visualstudio.com/) (the recommended editor). You can also simply run `go run main.go` to start a local Durable Task gRPC server that listens on port 4001. +The analyzer does not examine activity bodies or entity bodies. It also ignores code that no registered orchestrator reaches. + +For the full list of checks, the suggested fixes, the false-positive guards, and the limitations, read [`cmd/orchestratorvet/README.md`](cmd/orchestratorvet/README.md). + +### Run the services locally + +The DTS emulator supplies a local task hub. Use it for development and for the end-to-end tests. The blob payload tests and the history-export tests need Azurite. Start both services with any OCI runtime: ```bash -go run main.go --port 4001 --db ./test.sqlite3 +docker run -d -p 8080:8080 -p 8082:8082 \ + -e DTS_TASK_HUB_NAMES=default \ + mcr.microsoft.com/dts/dts-emulator:latest +docker run -d -p 10000:10000 mcr.microsoft.com/azure-storage/azurite:3.37.0 ``` -The following is the expected output: +The end-to-end tests read these environment variables. The tests skip themselves if the variables are empty. -``` -2022/09/14 17:26:50 backend started: sqlite::./test.sqlite3 -2022/09/14 17:26:50 server listening at 127.0.0.1:4001 -2022/09/14 17:26:50 orchestration-processor: waiting for new work items... -2022/09/14 17:26:50 activity-processor: waiting for new work items... -``` +| Variable | Description | +| --- | --- | +| `DTS_CONNECTION_STRING` | A full connection string. This variable has priority over the two variables below. | +| `DTS_EMULATOR_ENDPOINT` | The gRPC endpoint of the emulator, for example `http://127.0.0.1:8080`. | +| `DTS_TASK_HUB` | The task hub name for `DTS_EMULATOR_ENDPOINT`. The default value is `default`. | +| `AZURITE_CONNECTION_STRING` | An Azure Storage connection string for the blob payload tests and the history-export tests. | + +For `AZURITE_CONNECTION_STRING`, use the development credentials from the [Azurite documentation](https://learn.microsoft.com/azure/storage/common/storage-use-azurite#well-known-storage-account-and-key). -At this point you can use one of the [language SDKs](#language-sdks) mentioned earlier in a separate process to implement and execute durable orchestrations. Those SDKs will connect to port `4001` by default to interact with the Durable Task engine. +```bash +DTS_EMULATOR_ENDPOINT="http://127.0.0.1:8080" \ +DTS_TASK_HUB="default" \ +AZURITE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" \ +go test ./... -count=1 +``` -## Contributing +## Contributions -This project welcomes contributions and suggestions. Most contributions require you to agree to a -Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us -the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com. +This project accepts contributions and suggestions. Most contributions need a Contributor License Agreement (CLA). The CLA declares that you have the right to give us the rights to your contribution. For more information, go to . -When you submit a pull request, a CLA bot will automatically determine whether you need to provide -a CLA and decorate the PR appropriately (e.g., status check, comment). Simply follow the instructions -provided by the bot. You will only need to do this once across all repos using our CLA. +When you send a pull request, a CLA bot examines the pull request. The bot tells you if you must supply a CLA. Obey the instructions from the bot. You supply the CLA one time only for all repositories that use our CLA. -This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). -For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or -contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. +This project uses the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information, read the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/). You can also send questions to [opencode@microsoft.com](mailto:opencode@microsoft.com). ## Trademarks -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/legal/intellectualproperty/trademarks/usage/general). -Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. -Any use of third-party trademarks or logos are subject to those third-party's policies. +This project can contain trademarks or logos for projects, products, or services. You must obey [Microsoft's Trademark and Brand Guidelines](https://www.microsoft.com/legal/intellectualproperty/trademarks/usage/general) when you use Microsoft trademarks or logos. Do not use Microsoft trademarks or logos in a changed version of this project if the use causes confusion. Do not imply Microsoft sponsorship. Third-party trademarks and logos obey the policies of those third parties. diff --git a/api/context.go b/api/context.go new file mode 100644 index 00000000..ca9882f0 --- /dev/null +++ b/api/context.go @@ -0,0 +1,97 @@ +package api + +import ( + "context" + "maps" +) + +// ReservedContextFieldPrefix is reserved for Durable Task runtime identity tags. +const ReservedContextFieldPrefix = "__durabletask.context." + +// ContextFields are immutable caller-supplied values propagated into task contexts. +type ContextFields map[string]string + +// OrchestrationContextInfo identifies the orchestration associated with a task context. +type OrchestrationContextInfo struct { + InstanceID InstanceID + Name string + Version string + ParentInstanceID InstanceID +} + +// ActivityContextInfo identifies the activity associated with a task context. +type ActivityContextInfo struct { + InstanceID InstanceID + Name string + Version string + TaskID int32 +} + +// EntityContextInfo identifies the entity operation associated with a task context. +type EntityContextInfo struct { + EntityID EntityID + Operation string + RequestID string + IsSignal bool +} + +type orchestrationContextInfoKey struct{} +type activityContextInfoKey struct{} +type entityContextInfoKey struct{} +type contextFieldsKey struct{} + +// WithOrchestrationContextInfo returns a context containing orchestration identity. +func WithOrchestrationContextInfo(ctx context.Context, info OrchestrationContextInfo) context.Context { + return context.WithValue(ctx, orchestrationContextInfoKey{}, info) +} + +// OrchestrationContextInfoFromContext returns orchestration identity from ctx. +func OrchestrationContextInfoFromContext(ctx context.Context) (OrchestrationContextInfo, bool) { + info, ok := ctx.Value(orchestrationContextInfoKey{}).(OrchestrationContextInfo) + return info, ok +} + +// WithActivityContextInfo returns a context containing activity identity. +func WithActivityContextInfo(ctx context.Context, info ActivityContextInfo) context.Context { + return context.WithValue(ctx, activityContextInfoKey{}, info) +} + +// ActivityContextInfoFromContext returns activity identity from ctx. +func ActivityContextInfoFromContext(ctx context.Context) (ActivityContextInfo, bool) { + info, ok := ctx.Value(activityContextInfoKey{}).(ActivityContextInfo) + return info, ok +} + +// WithEntityContextInfo returns a context containing entity operation identity. +func WithEntityContextInfo(ctx context.Context, info EntityContextInfo) context.Context { + return context.WithValue(ctx, entityContextInfoKey{}, info) +} + +// EntityContextInfoFromContext returns entity operation identity from ctx. +func EntityContextInfoFromContext(ctx context.Context) (EntityContextInfo, bool) { + info, ok := ctx.Value(entityContextInfoKey{}).(EntityContextInfo) + return info, ok +} + +// ContextWithFields merges fields with any fields already in ctx, with fields +// taking precedence on conflicts. The stored map is a defensive copy. +func ContextWithFields(ctx context.Context, fields ContextFields) context.Context { + if len(fields) == 0 { + return ctx + } + merged := ContextFieldsFromContext(ctx) + if merged == nil { + merged = make(ContextFields, len(fields)) + } + maps.Copy(merged, fields) + return context.WithValue(ctx, contextFieldsKey{}, merged) +} + +// ContextFieldsFromContext returns a defensive copy of caller-supplied fields. +func ContextFieldsFromContext(ctx context.Context) ContextFields { + fields, ok := ctx.Value(contextFieldsKey{}).(ContextFields) + if !ok || len(fields) == 0 { + return nil + } + return maps.Clone(fields) +} diff --git a/api/context_test.go b/api/context_test.go new file mode 100644 index 00000000..c406635a --- /dev/null +++ b/api/context_test.go @@ -0,0 +1,16 @@ +package api + +import ( + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" +) + +func TestWithContextFieldsRejectsReservedKeys(t *testing.T) { + option := WithContextFields(ContextFields{ + ReservedContextFieldPrefix + "orchestration_version": "spoofed", + }) + if err := option(new(protos.CreateInstanceRequest), DefaultDataConverter()); err == nil { + t.Fatal("expected reserved context field to be rejected") + } +} diff --git a/api/converter.go b/api/converter.go new file mode 100644 index 00000000..e0fb5b73 --- /dev/null +++ b/api/converter.go @@ -0,0 +1,70 @@ +package api + +import ( + "encoding/json" + "fmt" +) + +// DataConverter serializes application payloads into the text fields used by +// the Durable Task protocol. Implementations must return valid UTF-8, be +// deterministic, and be safe for concurrent use. Because converter identity is +// not stored in the protocol, replacements must remain able to decode payloads +// written by earlier deployments. +type DataConverter interface { + Serialize(value any) (string, error) + Deserialize(payload string, target any) error +} + +// JSONDataConverter preserves the SDK's standard encoding/json payload format. +type JSONDataConverter struct{} + +func (JSONDataConverter) Serialize(value any) (string, error) { + data, err := json.Marshal(value) + if err != nil { + return "", err + } + return string(data), nil +} + +func (JSONDataConverter) Deserialize(payload string, target any) error { + if target == nil { + return nil + } + if err := json.Unmarshal([]byte(payload), target); err != nil { + return fmt.Errorf("failed to deserialize payload: %w", err) + } + return nil +} + +// DefaultDataConverter returns the converter used when none is configured. +func DefaultDataConverter() DataConverter { + return JSONDataConverter{} +} + +// NormalizeDataConverter returns the default JSON converter for nil inputs. +func NormalizeDataConverter(converter DataConverter) DataConverter { + if converter == nil { + return DefaultDataConverter() + } + return converter +} + +// SerializeData serializes a typed payload and rejects an empty encoding, which +// cannot be distinguished from an absent payload across all protocol surfaces. +func SerializeData(converter DataConverter, value any) (string, error) { + payload, err := NormalizeDataConverter(converter).Serialize(value) + if err != nil { + return "", err + } + if payload == "" { + return "", fmt.Errorf("data converter returned an empty payload") + } + return payload, nil +} + +func deserializePayload(converter DataConverter, payload string, target any) error { + if target == nil || payload == "" { + return nil + } + return NormalizeDataConverter(converter).Deserialize(payload, target) +} diff --git a/api/converter_test.go b/api/converter_test.go new file mode 100644 index 00000000..80e14a2e --- /dev/null +++ b/api/converter_test.go @@ -0,0 +1,135 @@ +package api + +import ( + "errors" + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +type recordingConverter struct { + serialized []any + deserialized []string + err error +} + +func (c *recordingConverter) Serialize(value any) (string, error) { + c.serialized = append(c.serialized, value) + if c.err != nil { + return "", c.err + } + return "converted", nil +} + +func (c *recordingConverter) Deserialize(payload string, target any) error { + c.deserialized = append(c.deserialized, payload) + if c.err != nil { + return c.err + } + if value, ok := target.(*string); ok { + *value = payload + } + return nil +} + +func TestJSONDataConverterMatchesExistingPayloadFormat(t *testing.T) { + converter := DefaultDataConverter() + payload, err := converter.Serialize(struct { + Value string `json:"value"` + }{Value: "test"}) + require.NoError(t, err) + require.Equal(t, `{"value":"test"}`, payload) + + var result struct { + Value string `json:"value"` + } + require.NoError(t, converter.Deserialize(payload, &result)) + require.Equal(t, "test", result.Value) +} + +func TestPayloadOptionsUseConfiguredConverterAndRawOptionsBypassIt(t *testing.T) { + converter := new(recordingConverter) + + create := new(protos.CreateInstanceRequest) + require.NoError(t, WithInput("value")(create, converter)) + require.Equal(t, "converted", create.Input.GetValue()) + + event := new(protos.RaiseEventRequest) + require.NoError(t, WithEventPayload("value")(event, converter)) + require.Equal(t, "converted", event.Input.GetValue()) + + terminate := new(protos.TerminateRequest) + require.NoError(t, WithOutput("value")(terminate, converter)) + require.Equal(t, "converted", terminate.Output.GetValue()) + + signal := new(protos.SignalEntityRequest) + require.NoError(t, WithSignalInput("value")(signal, converter)) + require.Equal(t, "converted", signal.Input.GetValue()) + require.Len(t, converter.serialized, 4) + + failing := &recordingConverter{err: errors.New("must not be called")} + require.NoError(t, WithRawInput("raw")(create, failing)) + require.NoError(t, WithRawEventData("raw")(event, failing)) + require.NoError(t, WithRawOutput("raw")(terminate, failing)) + require.NoError(t, WithRawSignalInput("raw")(signal, failing)) + require.Empty(t, failing.serialized) +} + +func TestEntityTypedNilInputIsAbsent(t *testing.T) { + var input *int + request := new(protos.SignalEntityRequest) + require.NoError(t, WithSignalInput(input)(request, DefaultDataConverter())) + require.Nil(t, request.Input) +} + +func TestMetadataUsesConfiguredConverter(t *testing.T) { + converter := new(recordingConverter) + metadata := &OrchestrationMetadata{ + SerializedInput: "input", + SerializedOutput: "output", + SerializedCustomStatus: "status", + Converter: converter, + } + var value string + require.NoError(t, metadata.ReadInput(&value)) + require.Equal(t, "input", value) + require.NoError(t, metadata.ReadOutput(&value)) + require.Equal(t, "output", value) + require.NoError(t, metadata.ReadCustomStatus(&value)) + require.Equal(t, "status", value) + + entity := &EntityMetadata{ + StateIncluded: true, + HasState: true, + SerializedState: "state", + Converter: converter, + } + require.NoError(t, entity.ReadState(&value)) + require.Equal(t, "state", value) + require.Equal(t, []string{"input", "output", "status", "state"}, converter.deserialized) +} + +func TestConverterErrorsAreReturnedWithoutJSONFallback(t *testing.T) { + expected := errors.New("converter failed") + converter := &recordingConverter{err: expected} + req := new(protos.CreateInstanceRequest) + require.ErrorIs(t, WithInput(make(chan int))(req, converter), expected) + require.Nil(t, req.Input) +} + +type emptyDataConverter struct{} + +func (emptyDataConverter) Serialize(any) (string, error) { + return "", nil +} + +func (emptyDataConverter) Deserialize(string, any) error { + return nil +} + +func TestEmptyTypedEncodingIsRejected(t *testing.T) { + req := new(protos.CreateInstanceRequest) + require.ErrorContains(t, WithInput("value")(req, emptyDataConverter{}), "empty payload") + require.Nil(t, req.Input) +} diff --git a/api/doc.go b/api/doc.go new file mode 100644 index 00000000..ef272b46 --- /dev/null +++ b/api/doc.go @@ -0,0 +1,3 @@ +// Package api defines the public models, options, errors, and extension +// contracts shared by Durable Task Scheduler clients, workers, and task code. +package api diff --git a/api/entity.go b/api/entity.go new file mode 100644 index 00000000..d5d7e9cf --- /dev/null +++ b/api/entity.go @@ -0,0 +1,200 @@ +package api + +import ( + "encoding/json" + "errors" + "fmt" + "reflect" + "time" + + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var ( + // ErrEntityStateNotIncluded indicates that entity state was excluded from a metadata response. + ErrEntityStateNotIncluded = errors.New("entity state was not included") + // ErrEntityHasNoState indicates that an entity metadata response contains no application state. + ErrEntityHasNoState = errors.New("entity has no state") +) + +// EntityID uniquely identifies an entity by its name and key. +type EntityID struct { + Name string + Key string +} + +// NewEntityID creates a new EntityID with the specified name and key. +func NewEntityID(name string, key string) EntityID { + if err := helpers.ValidateEntityName(name); err != nil { + panic(err) + } + return EntityID{Name: helpers.ToLowerInvariant(name), Key: key} +} + +// String returns the entity instance ID in the format "@@". +func (e EntityID) String() string { + return fmt.Sprintf("@%s@%s", helpers.ToLowerInvariant(e.Name), e.Key) +} + +// MarshalJSON serializes entity IDs using the cross-SDK compact instance ID format. +func (e EntityID) MarshalJSON() ([]byte, error) { + if err := helpers.ValidateEntityName(e.Name); err != nil { + return nil, err + } + return json.Marshal(e.String()) +} + +// UnmarshalJSON parses an entity ID from the cross-SDK compact instance ID format. +func (e *EntityID) UnmarshalJSON(data []byte) error { + if e == nil { + return fmt.Errorf("entity ID target must not be nil") + } + var value string + if err := json.Unmarshal(data, &value); err != nil { + return fmt.Errorf("entity ID must be a compact string: %w", err) + } + parsed, err := EntityIDFromString(value) + if err != nil { + return err + } + *e = parsed + return nil +} + +// EntityIDFromString parses an entity instance ID string in the format "@@". +func EntityIDFromString(s string) (EntityID, error) { + name, key, err := helpers.ParseEntityInstanceID(s) + if err != nil { + return EntityID{}, err + } + return EntityID{Name: name, Key: key}, nil +} + +// EntityMetadata contains metadata about an entity instance. +type EntityMetadata struct { + InstanceID EntityID + LastModifiedTime time.Time + BacklogQueueSize int32 + LockedBy string + StateIncluded bool + HasState bool + SerializedState string + Converter DataConverter `json:"-"` +} + +// SignalEntityOptions is a functional option type for signaling an entity. +type SignalEntityOptions func(*protos.SignalEntityRequest, DataConverter) error + +// WithSignalInput configures the input for an entity signal. +func WithSignalInput(input any) SignalEntityOptions { + return func(req *protos.SignalEntityRequest, converter DataConverter) error { + if isNilEntityValue(input) { + req.Input = nil + return nil + } + payload, err := SerializeData(converter, input) + if err != nil { + return err + } + req.Input = wrapperspb.String(payload) + return nil + } +} + +func isNilEntityValue(value any) bool { + if value == nil { + return true + } + reflected := reflect.ValueOf(value) + switch reflected.Kind() { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return reflected.IsNil() + default: + return false + } +} + +// WithRawSignalInput configures a raw string input for an entity signal. +func WithRawSignalInput(input string) SignalEntityOptions { + return func(req *protos.SignalEntityRequest, _ DataConverter) error { + req.Input = wrapperspb.String(input) + return nil + } +} + +// WithSignalScheduledTime configures a scheduled time for the entity signal. +func WithSignalScheduledTime(t time.Time) SignalEntityOptions { + return func(req *protos.SignalEntityRequest, _ DataConverter) error { + req.ScheduledTime = timestamppb.New(t) + return nil + } +} + +// ReadState deserializes entity state with the metadata converter. +func (m *EntityMetadata) ReadState(target any) error { + if !m.StateIncluded { + return ErrEntityStateNotIncluded + } + if !m.HasState { + return ErrEntityHasNoState + } + if target == nil { + return nil + } + return NormalizeDataConverter(m.Converter).Deserialize(m.SerializedState, target) +} + +// GetEntityOptions controls an entity metadata request. The zero value includes state. +type GetEntityOptions struct { + ExcludeState bool +} + +// EntityQuery defines filter criteria for querying entities. +type EntityQuery struct { + // InstanceIDStartsWith filters entities whose instance ID starts with this prefix. + InstanceIDStartsWith string + // LastModifiedFrom filters entities modified on or after this time. + LastModifiedFrom time.Time + // LastModifiedTo filters entities modified before this time. + LastModifiedTo time.Time + // ExcludeState omits entity state from the results. State is included by default. + ExcludeState bool + // IncludeTransient whether to include transient (stateless) entities. + IncludeTransient bool + // PageSize limits the number of entities returned per page. + PageSize int32 + // ContinuationToken for fetching the next page of results. + ContinuationToken string +} + +// EntityQueryResults contains the results of an entity query. +type EntityQueryResults struct { + Entities []*EntityMetadata + ContinuationToken string +} + +// CleanEntityStorageOptions controls entity storage cleanup. The zero value +// removes empty entities, releases orphaned locks, and continues to completion. +type CleanEntityStorageOptions struct { + // ContinuationToken for resuming a previous cleanup operation. + ContinuationToken string + // PreserveEmptyEntities disables removal of entities with no state and no locks. + PreserveEmptyEntities bool + // PreserveOrphanedLocks disables release of locks held by non-running orchestrations. + PreserveOrphanedLocks bool + // SinglePage returns after one backend cleanup request. + SinglePage bool +} + +// CleanEntityStorageResult contains the results of a cleanup operation. +type CleanEntityStorageResult struct { + // EmptyEntitiesRemoved is the number of empty entities removed. + EmptyEntitiesRemoved int32 + // OrphanedLocksReleased is the number of orphaned locks released. + OrphanedLocksReleased int32 + // ContinuationToken for resuming cleanup. Empty if complete. + ContinuationToken string +} diff --git a/api/entity_test.go b/api/entity_test.go new file mode 100644 index 00000000..75a4a33e --- /dev/null +++ b/api/entity_test.go @@ -0,0 +1,97 @@ +package api + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_API_EntityID_String(t *testing.T) { + id := NewEntityID("Counter", "myCounter") + assert.Equal(t, "@counter@myCounter", id.String()) +} + +func Test_API_EntityID_JSON(t *testing.T) { + type payload struct { + Entity EntityID + } + encoded, err := json.Marshal(payload{Entity: NewEntityID("Counter", "Key@Part")}) + require.NoError(t, err) + assert.JSONEq(t, `{"Entity":"@counter@Key@Part"}`, string(encoded)) + + var decoded payload + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, NewEntityID("counter", "Key@Part"), decoded.Entity) + + for _, invalid := range []string{ + `null`, + `{"Name":"counter","Key":"key"}`, + `"counter@key"`, + `"@@key"`, + } { + t.Run(invalid, func(t *testing.T) { + require.Error(t, json.Unmarshal([]byte(invalid), new(EntityID))) + }) + } +} + +func Test_API_EntityMetadata_StateAvailability(t *testing.T) { + var state int + require.ErrorIs(t, (&EntityMetadata{}).ReadState(&state), ErrEntityStateNotIncluded) + require.ErrorIs(t, (&EntityMetadata{StateIncluded: true}).ReadState(&state), ErrEntityHasNoState) + require.NoError(t, (&EntityMetadata{ + StateIncluded: true, + HasState: true, + SerializedState: "42", + }).ReadState(&state)) + assert.Equal(t, 42, state) +} + +func Test_API_EntityIDFromString(t *testing.T) { + tests := []struct { + name string + input string + want EntityID + wantErr bool + }{ + {name: "valid", input: "@counter@key1", want: EntityID{Name: "counter", Key: "key1"}}, + {name: "empty key", input: "@entity@", want: EntityID{Name: "entity", Key: ""}}, + {name: "invalid empty name", input: "@@key1", wantErr: true}, + {name: "invalid no prefix", input: "no-at-sign", wantErr: true}, + {name: "invalid no second @", input: "@onlyone", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := EntityIDFromString(tt.input) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func Test_API_NewEntityID_InvalidNamePanics(t *testing.T) { + assert.Panics(t, func() { NewEntityID("", "key") }) + assert.Panics(t, func() { NewEntityID("bad@name", "key") }) +} + +func FuzzEntityIDRoundTrip(f *testing.F) { + f.Add("Counter", "key") + f.Add("name", "key@with@separators") + f.Fuzz(func(t *testing.T, name, key string) { + if name == "" || strings.Contains(name, "@") { + t.Skip() + } + entityID := NewEntityID(name, key) + parsed, err := EntityIDFromString(entityID.String()) + require.NoError(t, err) + require.Equal(t, entityID, parsed) + }) +} diff --git a/api/failure.go b/api/failure.go new file mode 100644 index 00000000..482ba0f6 --- /dev/null +++ b/api/failure.go @@ -0,0 +1,185 @@ +package api + +import ( + "encoding/json" + "errors" + "fmt" +) + +// ErrorType is a stable cross-language durable failure identifier. +type ErrorType string + +const ( + ErrorTypeActivityTaskNotFound ErrorType = "ActivityTaskNotFound" + ErrorTypeEntityTaskNotFound ErrorType = "EntityTaskNotFound" + ErrorTypeOrchestratorTaskNotFound ErrorType = "OrchestratorTaskNotFound" + ErrorTypeTaskFailed ErrorType = "TaskFailedException" + ErrorTypeEntityOperationFailed ErrorType = "EntityOperationFailedException" + ErrorTypeVersionMismatch ErrorType = "VersionMismatch" + ErrorTypeVersionError ErrorType = "VersionError" + ErrorTypeHistoryLimitExceeded ErrorType = "HistoryLimitExceeded" + ErrorTypeOrchestratorResponseTooLarge ErrorType = "OrchestratorResponseTooLarge" + ErrorTypeActivityPanic ErrorType = "TaskActivityPanic" + ErrorTypeEntityOperationPanic ErrorType = "EntityOperationPanic" + ErrorTypeOrchestratorPanic ErrorType = "OrchestratorPanic" +) + +// MaxFailureDetailsDepth bounds failure chains in JSON and protobuf payloads. +const MaxFailureDetailsDepth = 32 + +var ( + ErrInvalidArgument = errors.New("invalid argument") + ErrTaskNotRegistered = errors.New("task is not registered") + ErrVersionMismatch = errors.New("task version is incompatible with this worker") +) + +func invalidArgument(message string) error { + return WrapInvalidArgument(errors.New(message)) +} + +// WrapInvalidArgument classifies an argument validation error without losing its cause. +func WrapInvalidArgument(err error) error { + if err == nil || errors.Is(err, ErrInvalidArgument) { + return err + } + return &invalidArgumentError{cause: err} +} + +type invalidArgumentError struct { + cause error +} + +func (e *invalidArgumentError) Error() string { + return fmt.Sprintf("%s: %v", ErrInvalidArgument, e.cause) +} + +func (e *invalidArgumentError) Unwrap() error { + return e.cause +} + +func (*invalidArgumentError) Is(target error) bool { + return target == ErrInvalidArgument +} + +// FailureDetails describes a durable failure that can cross process and language boundaries. +type FailureDetails struct { + ErrorType ErrorType `json:"type"` + ErrorMessage string `json:"message"` + StackTrace string `json:"stackTrace,omitempty"` + InnerFailure *FailureDetails `json:"innerFailure,omitempty"` + IsNonRetriable bool `json:"isNonRetriable,omitempty"` + Properties map[string]any `json:"properties,omitempty"` +} + +func (d *FailureDetails) String() string { + if d == nil { + return "" + } + if d.ErrorType == "" { + return d.ErrorMessage + } + return fmt.Sprintf("%s: %s", d.ErrorType, d.ErrorMessage) +} + +// NonRetriable reports whether this failure frame must not be retried. +func (d *FailureDetails) NonRetriable() bool { + return d != nil && d.IsNonRetriable +} + +// IsCausedBy reports whether this failure chain contains one of the supplied wire error types. +func (d *FailureDetails) IsCausedBy(errorTypes ...ErrorType) bool { + for current := d; current != nil; current = current.InnerFailure { + for _, errorType := range errorTypes { + if current.ErrorType == errorType { + return true + } + } + } + return false +} + +// Matches reports whether this failure chain belongs to a stable Go error category. +func (d *FailureDetails) Matches(target error) bool { + switch { + case errors.Is(target, ErrTaskNotRegistered): + return d.IsCausedBy( + ErrorTypeActivityTaskNotFound, + ErrorTypeEntityTaskNotFound, + ErrorTypeOrchestratorTaskNotFound, + ) + case errors.Is(target, ErrVersionMismatch): + return d.IsCausedBy(ErrorTypeVersionMismatch, ErrorTypeVersionError) + default: + return false + } +} + +func (d *FailureDetails) MarshalJSON() ([]byte, error) { + if err := d.validate(); err != nil { + return nil, err + } + type alias FailureDetails + return json.Marshal((*alias)(d)) +} + +func (d *FailureDetails) UnmarshalJSON(data []byte) error { + type alias FailureDetails + var decoded alias + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *d = FailureDetails(decoded) + return d.validate() +} + +func (d *FailureDetails) validate() error { + visited := make(map[*FailureDetails]struct{}) + for depth, current := 0, d; current != nil; depth, current = depth+1, current.InnerFailure { + if depth >= MaxFailureDetailsDepth { + return fmt.Errorf("failure details exceed the maximum depth of %d", MaxFailureDetailsDepth) + } + if _, ok := visited[current]; ok { + return errors.New("failure details contain a cycle") + } + visited[current] = struct{}{} + } + return nil +} + +// DurableTaskErrorTypeProvider supplies a stable cross-language error type. +type DurableTaskErrorTypeProvider interface { + DurableTaskErrorType() ErrorType +} + +// DurableTaskFailureDetailsProvider supplies failure details already received from durable history. +type DurableTaskFailureDetailsProvider interface { + DurableTaskFailureDetails() *FailureDetails +} + +// DurableTaskErrorPropertiesProvider supplies additional serializable failure properties. +type DurableTaskErrorPropertiesProvider interface { + DurableTaskErrorProperties() map[string]any +} + +// ErrorPropertiesProvider enriches failures without requiring application errors to implement an interface. +type ErrorPropertiesProvider interface { + ErrorProperties(error) map[string]any +} + +// ErrorPropertiesProviderFunc adapts a function into an ErrorPropertiesProvider. +type ErrorPropertiesProviderFunc func(error) map[string]any + +func (provider ErrorPropertiesProviderFunc) ErrorProperties(err error) map[string]any { + return provider(err) +} + +// DurableTaskStackTraceProvider supplies a stack trace for a failure. +type DurableTaskStackTraceProvider interface { + DurableTaskStackTrace() string +} + +// NonRetriable marks an error that must bypass retry handlers. +type NonRetriable interface { + error + NonRetriable() bool +} diff --git a/api/failure_test.go b/api/failure_test.go new file mode 100644 index 00000000..84d81ab7 --- /dev/null +++ b/api/failure_test.go @@ -0,0 +1,53 @@ +package api + +import ( + "encoding/json" + "errors" + "testing" +) + +func TestFailureDetailsHelpers(t *testing.T) { + details := &FailureDetails{ + ErrorType: "Outer", + ErrorMessage: "outer failed", + InnerFailure: &FailureDetails{ + ErrorType: "ActivityTaskNotFound", + ErrorMessage: "missing", + IsNonRetriable: true, + }, + } + + if details.String() != "Outer: outer failed" { + t.Fatalf("String() = %q", details.String()) + } + if !details.IsCausedBy(ErrorTypeActivityTaskNotFound) { + t.Fatal("IsCausedBy() did not inspect the full chain") + } + if details.NonRetriable() { + t.Fatal("outer failure should remain retriable") + } + if !details.InnerFailure.NonRetriable() { + t.Fatal("inner failure should be non-retriable") + } +} + +func TestFailureDetailsRejectsCyclesDuringJSONMarshal(t *testing.T) { + details := &FailureDetails{ErrorType: "Cycle", ErrorMessage: "cycle"} + details.InnerFailure = details + + if _, err := json.Marshal(details); err == nil { + t.Fatal("json.Marshal() succeeded for a cyclic failure chain") + } +} + +func TestWrapInvalidArgumentPreservesClassificationAndCause(t *testing.T) { + cause := errors.New("bad value") + err := WrapInvalidArgument(cause) + + if !errors.Is(err, ErrInvalidArgument) { + t.Fatal("error does not match ErrInvalidArgument") + } + if !errors.Is(err, cause) { + t.Fatal("error does not preserve its cause") + } +} diff --git a/api/history.go b/api/history.go new file mode 100644 index 00000000..70a2cffa --- /dev/null +++ b/api/history.go @@ -0,0 +1,349 @@ +package api + +import ( + "errors" + "fmt" + "time" +) + +const ( + DefaultHistoryMaxEvents = 100_000 + MaxHistoryMaxEvents = 1_000_000 + DefaultHistoryMaxBytes = 64 * 1024 * 1024 + MaxHistoryMaxBytes = 1024 * 1024 * 1024 +) + +var ErrHistoryLimitExceeded = errors.New("orchestration history exceeds the configured event limit") + +// HistoryEventType identifies the durable event represented by a [HistoryEvent]. +type HistoryEventType string + +const ( + HistoryEventUnknown HistoryEventType = "Unknown" + HistoryEventExecutionStarted HistoryEventType = "ExecutionStarted" + HistoryEventExecutionCompleted HistoryEventType = "ExecutionCompleted" + HistoryEventExecutionTerminated HistoryEventType = "ExecutionTerminated" + HistoryEventTaskScheduled HistoryEventType = "TaskScheduled" + HistoryEventTaskCompleted HistoryEventType = "TaskCompleted" + HistoryEventTaskFailed HistoryEventType = "TaskFailed" + HistoryEventSubOrchestrationInstanceCreated HistoryEventType = "SubOrchestrationInstanceCreated" + HistoryEventSubOrchestrationInstanceCompleted HistoryEventType = "SubOrchestrationInstanceCompleted" + HistoryEventSubOrchestrationInstanceFailed HistoryEventType = "SubOrchestrationInstanceFailed" + HistoryEventTimerCreated HistoryEventType = "TimerCreated" + HistoryEventTimerFired HistoryEventType = "TimerFired" + HistoryEventOrchestratorStarted HistoryEventType = "OrchestratorStarted" + HistoryEventOrchestratorCompleted HistoryEventType = "OrchestratorCompleted" + HistoryEventEventSent HistoryEventType = "EventSent" + HistoryEventEventRaised HistoryEventType = "EventRaised" + HistoryEventGeneric HistoryEventType = "Generic" + HistoryEventHistoryState HistoryEventType = "HistoryState" + HistoryEventContinueAsNew HistoryEventType = "ContinueAsNew" + HistoryEventExecutionSuspended HistoryEventType = "ExecutionSuspended" + HistoryEventExecutionResumed HistoryEventType = "ExecutionResumed" + HistoryEventEntityOperationSignaled HistoryEventType = "EntityOperationSignaled" + HistoryEventEntityOperationCalled HistoryEventType = "EntityOperationCalled" + HistoryEventEntityOperationCompleted HistoryEventType = "EntityOperationCompleted" + HistoryEventEntityOperationFailed HistoryEventType = "EntityOperationFailed" + HistoryEventEntityLockRequested HistoryEventType = "EntityLockRequested" + HistoryEventEntityLockGranted HistoryEventType = "EntityLockGranted" + HistoryEventEntityUnlockSent HistoryEventType = "EntityUnlockSent" + HistoryEventExecutionRewound HistoryEventType = "ExecutionRewound" +) + +// HistoryQuery configures an orchestration history read. +type HistoryQuery struct { + // ExecutionID selects a specific execution. Empty selects the service's + // current execution. GetOrchestrationHistory verifies a nonempty selection + // against ExecutionStarted events and rejects missing or conflicting IDs. + ExecutionID string + // MaxEvents bounds the buffered GetOrchestrationHistory result. It is + // validated but ignored by StreamOrchestrationHistory, which does not retain + // events. Zero uses DefaultHistoryMaxEvents. + MaxEvents int + // MaxBytes bounds the approximate in-memory size of the buffered + // GetOrchestrationHistory result. It is ignored by + // StreamOrchestrationHistory. Zero uses DefaultHistoryMaxBytes. + MaxBytes int +} + +// NormalizeHistoryQuery validates a history query and applies defaults. +func NormalizeHistoryQuery(query HistoryQuery) (HistoryQuery, error) { + switch { + case query.MaxEvents < 0: + return HistoryQuery{}, WrapInvalidArgument(errors.New("history event limit cannot be negative")) + case query.MaxEvents == 0: + query.MaxEvents = DefaultHistoryMaxEvents + case query.MaxEvents > MaxHistoryMaxEvents: + return HistoryQuery{}, WrapInvalidArgument( + fmt.Errorf("history event limit cannot exceed %d", MaxHistoryMaxEvents), + ) + } + switch { + case query.MaxBytes < 0: + return HistoryQuery{}, WrapInvalidArgument(errors.New("history byte limit cannot be negative")) + case query.MaxBytes == 0: + query.MaxBytes = DefaultHistoryMaxBytes + case query.MaxBytes > MaxHistoryMaxBytes: + return HistoryQuery{}, WrapInvalidArgument( + fmt.Errorf("history byte limit cannot exceed %d", MaxHistoryMaxBytes), + ) + } + return query, nil +} + +// OrchestrationHistory is a bounded snapshot of an orchestration's durable history. +type OrchestrationHistory struct { + InstanceID InstanceID `json:"instanceId"` + // ExecutionID is observed in ExecutionStarted events, never copied from the + // query. It may be empty only for an unpinned history without such events. + ExecutionID string `json:"executionId,omitempty"` + Events []*HistoryEvent `json:"events"` +} + +// HistoryEventHandler receives events in storage/service order. +type HistoryEventHandler func(*HistoryEvent) error + +// HistoryEvent is one API-owned durable history record. Exactly one detail +// field is populated for event types that carry details. +type HistoryEvent struct { + Type HistoryEventType `json:"type"` + EventID int32 `json:"eventId"` + Timestamp time.Time `json:"timestamp"` + // UnknownType identifies a wire event type that this SDK does not yet + // understand. It is diagnostic and populated only for HistoryEventUnknown. + UnknownType string `json:"unknownType,omitempty"` + + ExecutionStarted *HistoryExecutionStartedEvent `json:"executionStarted,omitempty"` + ExecutionCompleted *HistoryExecutionCompletedEvent `json:"executionCompleted,omitempty"` + ExecutionTerminated *HistoryExecutionTerminatedEvent `json:"executionTerminated,omitempty"` + TaskScheduled *HistoryTaskScheduledEvent `json:"taskScheduled,omitempty"` + TaskCompleted *HistoryTaskResultEvent `json:"taskCompleted,omitempty"` + TaskFailed *HistoryTaskFailureEvent `json:"taskFailed,omitempty"` + SubOrchestrationInstanceCreated *HistorySubOrchestrationInstanceCreatedEvent `json:"subOrchestrationInstanceCreated,omitempty"` + SubOrchestrationInstanceCompleted *HistoryTaskResultEvent `json:"subOrchestrationInstanceCompleted,omitempty"` + SubOrchestrationInstanceFailed *HistoryTaskFailureEvent `json:"subOrchestrationInstanceFailed,omitempty"` + TimerCreated *HistoryTimerEvent `json:"timerCreated,omitempty"` + TimerFired *HistoryTimerFiredEvent `json:"timerFired,omitempty"` + EventSent *HistoryExternalEvent `json:"eventSent,omitempty"` + EventRaised *HistoryExternalEvent `json:"eventRaised,omitempty"` + Generic *HistoryPayloadEvent `json:"generic,omitempty"` + HistoryState *HistoryStateEvent `json:"historyState,omitempty"` + ContinueAsNew *HistoryPayloadEvent `json:"continueAsNew,omitempty"` + ExecutionSuspended *HistoryPayloadEvent `json:"executionSuspended,omitempty"` + ExecutionResumed *HistoryPayloadEvent `json:"executionResumed,omitempty"` + Entity *HistoryEntityEvent `json:"entity,omitempty"` + ExecutionRewound *HistoryExecutionRewoundEvent `json:"executionRewound,omitempty"` + + Converter DataConverter `json:"-"` +} + +type HistoryExecutionStartedEvent struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + InstanceID InstanceID `json:"instanceId"` + ExecutionID string `json:"executionId,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + ScheduledStartAt time.Time `json:"scheduledStartAt,omitempty"` + Parent *HistoryParentInstanceInfo `json:"parent,omitempty"` + ParentTraceContext *HistoryTraceContext `json:"parentTraceContext,omitempty"` + OrchestrationSpanID string `json:"orchestrationSpanId,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ContextFields ContextFields `json:"contextFields,omitempty"` +} + +type HistoryParentInstanceInfo struct { + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + InstanceID InstanceID `json:"instanceId,omitempty"` + ExecutionID string `json:"executionId,omitempty"` + TaskScheduledID int32 `json:"taskScheduledId"` +} + +type HistoryTraceContext struct { + TraceParent string `json:"traceParent,omitempty"` + TraceState string `json:"traceState,omitempty"` + SpanID string `json:"spanId,omitempty"` +} + +type HistoryExecutionCompletedEvent struct { + RuntimeStatus OrchestrationStatus `json:"runtimeStatus"` + SerializedResult string `json:"serializedResult,omitempty"` + FailureDetails *FailureDetails `json:"failureDetails,omitempty"` +} + +type HistoryExecutionTerminatedEvent struct { + SerializedInput string `json:"serializedInput,omitempty"` + Recursive bool `json:"recursive"` +} + +type HistoryTaskScheduledEvent struct { + Name string `json:"name"` + Version string `json:"version,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + ParentTraceContext *HistoryTraceContext `json:"parentTraceContext,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ContextFields ContextFields `json:"contextFields,omitempty"` +} + +type HistoryTaskResultEvent struct { + TaskScheduledID int32 `json:"taskScheduledId"` + SerializedResult string `json:"serializedResult,omitempty"` +} + +type HistoryTaskFailureEvent struct { + TaskScheduledID int32 `json:"taskScheduledId"` + FailureDetails *FailureDetails `json:"failureDetails,omitempty"` +} + +type HistorySubOrchestrationInstanceCreatedEvent struct { + InstanceID InstanceID `json:"instanceId"` + Name string `json:"name"` + Version string `json:"version,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + ParentTraceContext *HistoryTraceContext `json:"parentTraceContext,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ContextFields ContextFields `json:"contextFields,omitempty"` +} + +type HistoryTimerEvent struct { + FireAt time.Time `json:"fireAt"` +} + +type HistoryTimerFiredEvent struct { + FireAt time.Time `json:"fireAt"` + TimerID int32 `json:"timerId"` +} + +type HistoryExternalEvent struct { + InstanceID InstanceID `json:"instanceId,omitempty"` + Name string `json:"name"` + SerializedInput string `json:"serializedInput,omitempty"` +} + +type HistoryPayloadEvent struct { + SerializedInput string `json:"serializedInput,omitempty"` +} + +type HistoryStateEvent struct { + State *OrchestrationMetadata `json:"state,omitempty"` +} + +type HistoryEntityEvent struct { + RequestID string `json:"requestId,omitempty"` + Operation string `json:"operation,omitempty"` + TargetInstanceID string `json:"targetInstanceId,omitempty"` + ParentInstanceID string `json:"parentInstanceId,omitempty"` + ParentExecutionID string `json:"parentExecutionId,omitempty"` + ScheduledAt time.Time `json:"scheduledAt,omitempty"` + CriticalSectionID string `json:"criticalSectionId,omitempty"` + LockSet []string `json:"lockSet,omitempty"` + Position int32 `json:"position,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + SerializedOutput string `json:"serializedOutput,omitempty"` + FailureDetails *FailureDetails `json:"failureDetails,omitempty"` +} + +type HistoryExecutionRewoundEvent struct { + Reason string `json:"reason,omitempty"` + Name string `json:"name,omitempty"` + Version string `json:"version,omitempty"` + InstanceID InstanceID `json:"instanceId,omitempty"` + ParentExecutionID string `json:"parentExecutionId,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + Parent *HistoryParentInstanceInfo `json:"parent,omitempty"` + ParentTraceContext *HistoryTraceContext `json:"parentTraceContext,omitempty"` + Tags map[string]string `json:"tags,omitempty"` + ContextFields ContextFields `json:"contextFields,omitempty"` +} + +// ReadInput deserializes the input-like payload carried by this event. +// Events without an input payload leave target unchanged and return nil. +func (e *HistoryEvent) ReadInput(target any) error { + if e == nil { + return nil + } + var payload string + switch e.Type { + case HistoryEventExecutionStarted: + if e.ExecutionStarted != nil { + payload = e.ExecutionStarted.SerializedInput + } + case HistoryEventExecutionTerminated: + if e.ExecutionTerminated != nil { + payload = e.ExecutionTerminated.SerializedInput + } + case HistoryEventTaskScheduled: + if e.TaskScheduled != nil { + payload = e.TaskScheduled.SerializedInput + } + case HistoryEventSubOrchestrationInstanceCreated: + if e.SubOrchestrationInstanceCreated != nil { + payload = e.SubOrchestrationInstanceCreated.SerializedInput + } + case HistoryEventEventSent: + if e.EventSent != nil { + payload = e.EventSent.SerializedInput + } + case HistoryEventEventRaised: + if e.EventRaised != nil { + payload = e.EventRaised.SerializedInput + } + case HistoryEventContinueAsNew: + if e.ContinueAsNew != nil { + payload = e.ContinueAsNew.SerializedInput + } + case HistoryEventExecutionSuspended: + if e.ExecutionSuspended != nil { + payload = e.ExecutionSuspended.SerializedInput + } + case HistoryEventExecutionResumed: + if e.ExecutionResumed != nil { + payload = e.ExecutionResumed.SerializedInput + } + case HistoryEventExecutionRewound: + if e.ExecutionRewound != nil { + payload = e.ExecutionRewound.SerializedInput + } + case HistoryEventEntityOperationSignaled, HistoryEventEntityOperationCalled: + if e.Entity != nil { + payload = e.Entity.SerializedInput + } + } + return deserializePayload(e.Converter, payload, target) +} + +// ReadResult deserializes the result/output payload carried by this event. +// Events without a result payload leave target unchanged and return nil. +func (e *HistoryEvent) ReadResult(target any) error { + if e == nil { + return nil + } + var payload string + switch e.Type { + case HistoryEventExecutionCompleted: + if e.ExecutionCompleted != nil { + payload = e.ExecutionCompleted.SerializedResult + } + case HistoryEventTaskCompleted: + if e.TaskCompleted != nil { + payload = e.TaskCompleted.SerializedResult + } + case HistoryEventSubOrchestrationInstanceCompleted: + if e.SubOrchestrationInstanceCompleted != nil { + payload = e.SubOrchestrationInstanceCompleted.SerializedResult + } + case HistoryEventEntityOperationCompleted: + if e.Entity != nil { + payload = e.Entity.SerializedOutput + } + } + return deserializePayload(e.Converter, payload, target) +} + +// ReadData deserializes a generic event payload. +func (e *HistoryEvent) ReadData(target any) error { + if e == nil || e.Type != HistoryEventGeneric || e.Generic == nil { + return nil + } + return deserializePayload(e.Converter, e.Generic.SerializedInput, target) +} diff --git a/api/history_test.go b/api/history_test.go new file mode 100644 index 00000000..e74b1cae --- /dev/null +++ b/api/history_test.go @@ -0,0 +1,69 @@ +package api + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeHistoryQuery(t *testing.T) { + normalized, err := NormalizeHistoryQuery(HistoryQuery{}) + require.NoError(t, err) + require.Equal(t, DefaultHistoryMaxEvents, normalized.MaxEvents) + + _, err = NormalizeHistoryQuery(HistoryQuery{MaxEvents: -1}) + require.ErrorIs(t, err, ErrInvalidArgument) + + _, err = NormalizeHistoryQuery(HistoryQuery{MaxEvents: MaxHistoryMaxEvents + 1}) + require.ErrorIs(t, err, ErrInvalidArgument) + + _, err = NormalizeHistoryQuery(HistoryQuery{MaxBytes: MaxHistoryMaxBytes + 1}) + require.ErrorIs(t, err, ErrInvalidArgument) +} + +func TestHistoryEventTypedPayloadReaders(t *testing.T) { + converter := historyTestConverter{} + input := &HistoryEvent{ + Type: HistoryEventExecutionStarted, + ExecutionStarted: &HistoryExecutionStartedEvent{SerializedInput: "input"}, + Converter: converter, + } + var decoded string + require.NoError(t, input.ReadInput(&decoded)) + require.Equal(t, "decoded:input", decoded) + + result := &HistoryEvent{ + Type: HistoryEventEntityOperationCompleted, + Entity: &HistoryEntityEvent{SerializedOutput: "output"}, + Converter: converter, + } + require.NoError(t, result.ReadResult(&decoded)) + require.Equal(t, "decoded:output", decoded) + + generic := &HistoryEvent{ + Type: HistoryEventGeneric, + Generic: &HistoryPayloadEvent{SerializedInput: "data"}, + Converter: converter, + } + require.NoError(t, generic.ReadData(&decoded)) + require.Equal(t, "decoded:data", decoded) + + require.NoError(t, (&HistoryEvent{}).ReadInput(&decoded)) + require.NoError(t, (*HistoryEvent)(nil).ReadResult(&decoded)) +} + +type historyTestConverter struct{} + +func (historyTestConverter) Serialize(any) (string, error) { + return "", errors.New("not implemented") +} + +func (historyTestConverter) Deserialize(payload string, target any) error { + value, ok := target.(*string) + if !ok { + return errors.New("expected *string") + } + *value = "decoded:" + payload + return nil +} diff --git a/api/large_payload.go b/api/large_payload.go new file mode 100644 index 00000000..58f35c5f --- /dev/null +++ b/api/large_payload.go @@ -0,0 +1,138 @@ +package api + +import ( + "context" + "errors" + "strings" +) + +const ( + DefaultLargePayloadThresholdBytes = 64 * 1024 + DefaultLargePayloadMaxBytes = 64 * 1024 * 1024 + DefaultAzureBlobPayloadThresholdBytes = 256 * 1024 + DefaultAzureBlobPayloadMaxBytes = 10 * 1024 * 1024 + DurableTaskPayloadReferencePrefix = "durabletask-payload:v1:" + AzureBlobPayloadReferencePrefixV1 = "blob:v1:" + AzureBlobPayloadReferencePrefixV2 = "blob:v2:" +) + +var ( + ErrLargePayloadTooLarge = errors.New("large payload exceeds the configured size limit") + ErrLargePayloadReference = errors.New("invalid large payload reference") + ErrLargePayloadIntegrity = errors.New("large payload integrity check failed") +) + +// IsLargePayloadReference reports whether value uses a reserved Durable Task +// large-payload reference prefix. Recognized values must be validated and +// resolved rather than treated as application data. +func IsLargePayloadReference(value string) bool { + return strings.HasPrefix(value, DurableTaskPayloadReferencePrefix) || + strings.HasPrefix(value, AzureBlobPayloadReferencePrefixV1) || + strings.HasPrefix(value, AzureBlobPayloadReferencePrefixV2) +} + +// LargePayloadStore externalizes payload bytes and returns an opaque location. +type LargePayloadStore interface { + Store(context.Context, []byte) (string, error) +} + +// LargePayloadResolver resolves payload bytes from an opaque, untrusted location. +// Implementations must validate schemes, accounts, paths, and other allow-list +// constraints before reading data. +type LargePayloadResolver interface { + Resolve(context.Context, string) ([]byte, error) +} + +// LargePayloadTokenStore supports a native payload token format. Native token +// stores are useful when a backing service already has an interoperable token +// representation, such as Azure Blob Storage's blob:v2 URLs. +type LargePayloadTokenStore interface { + StoreToken(context.Context, []byte) (string, error) + ResolveToken(context.Context, string) ([]byte, error) + IsLargePayloadToken(string) bool +} + +// LargePayloadTokenValidator validates recognized native tokens before they +// are preserved or resolved. +type LargePayloadTokenValidator interface { + ValidateLargePayloadToken(string) error +} + +// LargePayloadDefaults supplies storage-specific defaults to use when an +// option is omitted. It is intentionally optional to preserve legacy defaults. +type LargePayloadDefaults interface { + LargePayloadDefaults() (thresholdBytes, maxPayloadBytes int) +} + +// InclusiveLargePayloadThreshold identifies stores whose threshold includes +// payloads exactly equal to the configured threshold. +type InclusiveLargePayloadThreshold interface { + UsesInclusiveLargePayloadThreshold() bool +} + +// LargePayloadOptionsValidator performs store-specific option validation after +// defaults are applied. +type LargePayloadOptionsValidator interface { + ValidateLargePayloadOptions(thresholdBytes, maxPayloadBytes int) error +} + +// LargePayloadOptions configures payload externalization and hydration. +type LargePayloadOptions struct { + Store LargePayloadStore + Resolver LargePayloadResolver + TokenStore LargePayloadTokenStore + // ThresholdBytes uses storage-specific defaults when zero. + ThresholdBytes int + // MaxPayloadBytes uses storage-specific defaults when zero. + MaxPayloadBytes int +} + +// NormalizeLargePayloadOptions validates options and applies generic or +// store-specific defaults. A nil input returns nil, nil. +func NormalizeLargePayloadOptions(options *LargePayloadOptions) (*LargePayloadOptions, error) { + if options == nil { + return nil, nil + } + normalized := *options + if normalized.Store == nil { + return nil, WrapInvalidArgument(errors.New("large payload store is required")) + } + if normalized.Resolver == nil { + return nil, WrapInvalidArgument(errors.New("large payload resolver is required")) + } + if normalized.TokenStore == nil { + if tokenStore, ok := normalized.Store.(LargePayloadTokenStore); ok { + normalized.TokenStore = tokenStore + } + } + defaultThreshold, defaultMax := DefaultLargePayloadThresholdBytes, DefaultLargePayloadMaxBytes + if defaults, ok := normalized.Store.(LargePayloadDefaults); ok { + defaultThreshold, defaultMax = defaults.LargePayloadDefaults() + } + if normalized.ThresholdBytes == 0 { + normalized.ThresholdBytes = defaultThreshold + } + if normalized.MaxPayloadBytes == 0 { + normalized.MaxPayloadBytes = defaultMax + } + if normalized.ThresholdBytes < 0 { + return nil, WrapInvalidArgument(errors.New("large payload threshold cannot be negative")) + } + if normalized.MaxPayloadBytes <= 0 { + return nil, WrapInvalidArgument(errors.New("large payload maximum must be greater than zero")) + } + if normalized.ThresholdBytes > normalized.MaxPayloadBytes { + return nil, WrapInvalidArgument( + errors.New("large payload threshold cannot exceed maximum payload size"), + ) + } + if validator, ok := normalized.Store.(LargePayloadOptionsValidator); ok { + if err := validator.ValidateLargePayloadOptions( + normalized.ThresholdBytes, + normalized.MaxPayloadBytes, + ); err != nil { + return nil, WrapInvalidArgument(err) + } + } + return &normalized, nil +} diff --git a/backend/logger.go b/api/logger.go similarity index 71% rename from backend/logger.go rename to api/logger.go index f759358b..d726204a 100644 --- a/backend/logger.go +++ b/api/logger.go @@ -1,4 +1,4 @@ -package backend +package api import ( "fmt" @@ -6,6 +6,7 @@ import ( "log/slog" ) +// Logger receives diagnostic messages from DTS clients and workers. type Logger interface { // Debug logs a message at level Debug. Debug(v ...any) @@ -39,46 +40,40 @@ var defaultLogger = &logger{ debugLogger: log.New(log.Writer(), "DEBUG: ", log.Flags()), } -// Debug implements Logger -func (log *logger) Debug(v ...any) { - log.debugLogger.Print(v...) +func (l *logger) Debug(v ...any) { + l.debugLogger.Print(v...) } -// Debugf implements Logger -func (log *logger) Debugf(format string, v ...any) { - log.debugLogger.Printf(format, v...) +func (l *logger) Debugf(format string, v ...any) { + l.debugLogger.Printf(format, v...) } -// Error implements Logger -func (log *logger) Error(v ...any) { - log.errorLogger.Print(v...) +func (l *logger) Error(v ...any) { + l.errorLogger.Print(v...) } -// Errorf implements Logger -func (log *logger) Errorf(format string, v ...any) { - log.errorLogger.Printf(format, v...) +func (l *logger) Errorf(format string, v ...any) { + l.errorLogger.Printf(format, v...) } -// Info implements Logger -func (log *logger) Info(v ...any) { - log.infoLogger.Print(v...) +func (l *logger) Info(v ...any) { + l.infoLogger.Print(v...) } -// Infof implements Logger -func (log *logger) Infof(format string, v ...any) { - log.infoLogger.Printf(format, v...) +func (l *logger) Infof(format string, v ...any) { + l.infoLogger.Printf(format, v...) } -// Warn implements Logger -func (log *logger) Warn(v ...any) { - log.warningLogger.Print(v...) +func (l *logger) Warn(v ...any) { + l.warningLogger.Print(v...) } -// Warnf implements Logger -func (log *logger) Warnf(format string, v ...any) { - log.warningLogger.Printf(format, v...) +func (l *logger) Warnf(format string, v ...any) { + l.warningLogger.Printf(format, v...) } +// DefaultLogger returns the process-wide logger used when a caller does not +// supply one. func DefaultLogger() Logger { return defaultLogger } diff --git a/api/logger_test.go b/api/logger_test.go new file mode 100644 index 00000000..8d9f7c6f --- /dev/null +++ b/api/logger_test.go @@ -0,0 +1,77 @@ +package api + +import ( + "bytes" + "context" + "log/slog" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestNewSlogLoggerRoutesEveryLevel pins the adapter applications use to route +// SDK logging into their own slog handler: every Logger method must reach slog +// at the matching level, and the formatting variants must apply the format. +func TestNewSlogLoggerRoutesEveryLevel(t *testing.T) { + for _, test := range []struct { + name string + level slog.Level + log func(Logger) + want string + }{ + {"Debug", slog.LevelDebug, func(l Logger) { l.Debug("a", "b") }, "ab"}, + {"Debugf", slog.LevelDebug, func(l Logger) { l.Debugf("a=%d", 1) }, "a=1"}, + {"Info", slog.LevelInfo, func(l Logger) { l.Info("a", "b") }, "ab"}, + {"Infof", slog.LevelInfo, func(l Logger) { l.Infof("a=%d", 1) }, "a=1"}, + {"Warn", slog.LevelWarn, func(l Logger) { l.Warn("a", "b") }, "ab"}, + {"Warnf", slog.LevelWarn, func(l Logger) { l.Warnf("a=%d", 1) }, "a=1"}, + {"Error", slog.LevelError, func(l Logger) { l.Error("a", "b") }, "ab"}, + {"Errorf", slog.LevelError, func(l Logger) { l.Errorf("a=%d", 1) }, "a=1"}, + } { + t.Run(test.name, func(t *testing.T) { + var buf bytes.Buffer + var recorded []slog.Record + handler := &recordingHandler{buf: &buf, records: &recorded} + test.log(NewSlogLogger(slog.New(handler))) + + require.Len(t, recorded, 1) + assert.Equal(t, test.level, recorded[0].Level) + assert.Equal(t, test.want, recorded[0].Message) + }) + } +} + +// TestDefaultLoggerImplementsEveryLevel keeps the default logger usable as the +// zero-configuration option callers pass to NewClient and NewWorker. +func TestDefaultLoggerImplementsEveryLevel(t *testing.T) { + log := DefaultLogger() + require.NotNil(t, log) + assert.NotPanics(t, func() { + log.Debug("d") + log.Debugf("d=%d", 1) + log.Info("i") + log.Infof("i=%d", 1) + log.Warn("w") + log.Warnf("w=%d", 1) + log.Error("e") + log.Errorf("e=%d", 1) + }) +} + +type recordingHandler struct { + buf *bytes.Buffer + records *[]slog.Record +} + +func (h *recordingHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (h *recordingHandler) Handle(_ context.Context, record slog.Record) error { + *h.records = append(*h.records, record) + h.buf.WriteString(record.Message) + return nil +} + +func (h *recordingHandler) WithAttrs([]slog.Attr) slog.Handler { return h } + +func (h *recordingHandler) WithGroup(string) slog.Handler { return h } diff --git a/api/management.go b/api/management.go new file mode 100644 index 00000000..79e4ad4a --- /dev/null +++ b/api/management.go @@ -0,0 +1,146 @@ +package api + +import ( + "errors" + "fmt" + "time" + + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const ( + DefaultInstanceQueryPageSize = 100 + MaxInstanceQueryPageSize = 1000 + MaxRemoteTagFilterScanPages = 100 + MaxInstanceBatchSize = 500 + DefaultPurgePollInterval = 100 * time.Millisecond +) + +var ErrFeatureNotSupported = errors.New("feature is not supported by Durable Task Scheduler") + +// OrchestrationQuery describes a bounded query for orchestration instances. +// ContinuationToken values are opaque and must only be reused with the same query. +type OrchestrationQuery struct { + RuntimeStatus []OrchestrationStatus + CreatedTimeFrom time.Time + CreatedTimeTo time.Time + TaskHubNames []string + PageSize int + ContinuationToken string + InstanceIDPrefix string + FetchInputsAndOutputs bool + // Tags are exact-match filters. The gRPC wire contract has no tag filter, so + // the client scans a bounded number of service pages locally and can return a + // partial page with a continuation token to resume from. + Tags map[string]string +} + +type OrchestrationQueryResult struct { + Orchestrations []*OrchestrationMetadata + ContinuationToken string +} + +// InstanceIDQuery describes a bounded query that returns only instance IDs. +type InstanceIDQuery struct { + RuntimeStatus []OrchestrationStatus + CompletedTimeFrom time.Time + CompletedTimeTo time.Time + PageSize int + ContinuationToken string +} + +type InstanceIDQueryResult struct { + InstanceIDs []InstanceID + ContinuationToken string +} + +type RestartOptions func(*protos.RestartInstanceRequest) error + +func WithRestartNewInstanceID(restartWithNewInstanceID bool) RestartOptions { + return func(req *protos.RestartInstanceRequest) error { + req.RestartWithNewInstanceId = restartWithNewInstanceID + return nil + } +} + +type RewindOptions func(*protos.RewindInstanceRequest) error + +func WithRewindReason(reason string) RewindOptions { + return func(req *protos.RewindInstanceRequest) error { + req.Reason = wrapperspb.String(reason) + return nil + } +} + +type PurgeInstanceFilter struct { + CreatedTimeFrom time.Time + CreatedTimeTo time.Time + RuntimeStatus []OrchestrationStatus + Timeout time.Duration +} + +// PurgeInstancesRequest selects either a bounded list of instance IDs or a filter. +// Filter requests are polled until the service reports completion. +type PurgeInstancesRequest struct { + InstanceIDs []InstanceID + Filter *PurgeInstanceFilter + Recursive bool + PollInterval time.Duration +} + +type PurgeInstancesResult struct { + DeletedInstanceCount int + IsComplete bool +} + +func (request PurgeInstancesRequest) Validate() error { + hasInstanceIDs := len(request.InstanceIDs) > 0 + hasFilter := request.Filter != nil + if hasInstanceIDs == hasFilter { + return invalidArgument("purge request must specify exactly one of instance IDs or a filter") + } + for _, id := range request.InstanceIDs { + if id == EmptyInstanceID { + return invalidArgument("purge instance ID cannot be empty") + } + } + if request.Filter != nil { + if err := ValidateTimeRange(request.Filter.CreatedTimeFrom, request.Filter.CreatedTimeTo); err != nil { + return fmt.Errorf("invalid purge filter: %w", err) + } + if request.Filter.Timeout < 0 { + return invalidArgument("purge timeout cannot be negative") + } + } + return nil +} + +type CreateTaskHubOptions func(*protos.CreateTaskHubRequest) error + +func WithRecreateTaskHub(recreateIfExists bool) CreateTaskHubOptions { + return func(req *protos.CreateTaskHubRequest) error { + req.RecreateIfExists = recreateIfExists + return nil + } +} + +func NormalizeInstanceQueryPageSize(pageSize int) (int, error) { + switch { + case pageSize < 0: + return 0, invalidArgument("page size cannot be negative") + case pageSize == 0: + return DefaultInstanceQueryPageSize, nil + case pageSize > MaxInstanceQueryPageSize: + return 0, invalidArgument(fmt.Sprintf("page size cannot exceed %d", MaxInstanceQueryPageSize)) + default: + return pageSize, nil + } +} + +func ValidateTimeRange(from, to time.Time) error { + if !from.IsZero() && !to.IsZero() && from.After(to) { + return invalidArgument("start time must not be after end time") + } + return nil +} diff --git a/api/management_test.go b/api/management_test.go new file mode 100644 index 00000000..df5e32b0 --- /dev/null +++ b/api/management_test.go @@ -0,0 +1,43 @@ +package api + +import ( + "context" + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" + "github.com/stretchr/testify/require" +) + +func TestWithTagsMergesAndRejectsReservedKeys(t *testing.T) { + req := &protos.CreateInstanceRequest{Tags: map[string]string{"existing": "value"}} + require.NoError(t, WithTags(map[string]string{"team": "durable"})(req, DefaultDataConverter())) + require.Equal(t, "value", req.Tags["existing"]) + require.Equal(t, "durable", req.Tags["team"]) + require.Equal(t, "1", req.Tags[tagcodec.ContextEncodingTag]) + require.Error(t, WithTags(map[string]string{ReservedContextFieldPrefix + "name": "invalid"})(req, DefaultDataConverter())) +} + +func TestNormalizeLargePayloadOptions(t *testing.T) { + store := testPayloadStore{} + normalized, err := NormalizeLargePayloadOptions(&LargePayloadOptions{ + Store: store, + Resolver: store, + }) + require.NoError(t, err) + require.Equal(t, DefaultLargePayloadThresholdBytes, normalized.ThresholdBytes) + require.Equal(t, DefaultLargePayloadMaxBytes, normalized.MaxPayloadBytes) + + _, err = NormalizeLargePayloadOptions(&LargePayloadOptions{}) + require.ErrorIs(t, err, ErrInvalidArgument) +} + +type testPayloadStore struct{} + +func (testPayloadStore) Store(context.Context, []byte) (string, error) { + return "test", nil +} + +func (testPayloadStore) Resolve(context.Context, string) ([]byte, error) { + return nil, nil +} diff --git a/api/orchestration.go b/api/orchestration.go index 12bc9ef3..d8031b31 100644 --- a/api/orchestration.go +++ b/api/orchestration.go @@ -4,33 +4,27 @@ import ( "encoding/json" "errors" "fmt" + "maps" + "slices" + "strings" "time" "github.com/microsoft/durabletask-go/internal/helpers" "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" ) var ( ErrInstanceNotFound = errors.New("no such instance exists") - ErrNotStarted = errors.New("orchestration has not started") ErrNotCompleted = errors.New("orchestration has not yet completed") - ErrNoFailures = errors.New("orchestration did not report failure details") ErrDuplicateInstance = errors.New("orchestration instance already exists") - ErrIgnoreInstance = errors.New("ignore creating orchestration instance") + ErrInvalidState = errors.New("orchestration is not in a valid state for this operation") EmptyInstanceID = InstanceID("") ) -type CreateOrchestrationAction = protos.CreateOrchestrationAction - -const ( - REUSE_ID_ACTION_ERROR CreateOrchestrationAction = protos.CreateOrchestrationAction_ERROR - REUSE_ID_ACTION_IGNORE CreateOrchestrationAction = protos.CreateOrchestrationAction_IGNORE - REUSE_ID_ACTION_TERMINATE CreateOrchestrationAction = protos.CreateOrchestrationAction_TERMINATE -) - type OrchestrationStatus = protos.OrchestrationStatus const ( @@ -44,7 +38,17 @@ const ( RUNTIME_STATUS_SUSPENDED OrchestrationStatus = protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED ) -type OrchestrationIdReusePolicy = protos.OrchestrationIdReusePolicy +// OrchestrationIDReusePolicy controls deduplication when an orchestration with +// the requested instance ID already exists. +// +// A nil DedupeStatuses slice uses the service default. An empty, non-nil slice +// allows every configurable status to be replaced. Otherwise, an existing +// instance whose status appears in DedupeStatuses causes scheduling to fail. +// ContinuedAsNew is not configurable because it is an execution transition +// rather than a stable instance status. +type OrchestrationIDReusePolicy struct { + DedupeStatuses []OrchestrationStatus +} // InstanceID is a unique identifier for an orchestration instance. type InstanceID string @@ -52,26 +56,33 @@ type InstanceID string type OrchestrationMetadata struct { InstanceID InstanceID Name string + Version string + ExecutionID string + ParentInstanceID InstanceID RuntimeStatus protos.OrchestrationStatus + ScheduledStartAt time.Time CreatedAt time.Time LastUpdatedAt time.Time + CompletedAt time.Time SerializedInput string SerializedOutput string SerializedCustomStatus string - FailureDetails *protos.TaskFailureDetails + FailureDetails *FailureDetails + Tags map[string]string + Converter DataConverter `json:"-"` } // NewOrchestrationOptions configures options for starting a new orchestration. -type NewOrchestrationOptions func(*protos.CreateInstanceRequest) error +type NewOrchestrationOptions func(*protos.CreateInstanceRequest, DataConverter) error -// GetOrchestrationMetadataOptions is a set of options for fetching orchestration metadata. +// FetchOrchestrationMetadataOptions configures orchestration metadata retrieval. type FetchOrchestrationMetadataOptions func(*protos.GetInstanceRequest) // RaiseEventOptions is a set of options for raising an orchestration event. -type RaiseEventOptions func(*protos.RaiseEventRequest) error +type RaiseEventOptions func(*protos.RaiseEventRequest, DataConverter) error // TerminateOptions is a set of options for terminating an orchestration. -type TerminateOptions func(*protos.TerminateRequest) error +type TerminateOptions func(*protos.TerminateRequest, DataConverter) error // PurgeOptions is a set of options for purging an orchestration. type PurgeOptions func(*protos.PurgeInstancesRequest) error @@ -79,39 +90,69 @@ type PurgeOptions func(*protos.PurgeInstancesRequest) error // WithInstanceID configures an explicit orchestration instance ID. If not specified, // a random UUID value will be used for the orchestration instance ID. func WithInstanceID(id InstanceID) NewOrchestrationOptions { - return func(req *protos.CreateInstanceRequest) error { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { + if err := helpers.ValidateOrchestrationInstanceID(string(id)); err != nil { + return err + } req.InstanceId = string(id) return nil } } -// WithOrchestrationIdReusePolicy configures Orchestration ID reuse policy. -func WithOrchestrationIdReusePolicy(policy *protos.OrchestrationIdReusePolicy) NewOrchestrationOptions { - return func(req *protos.CreateInstanceRequest) error { - // initialize CreateInstanceOption +// WithOrchestrationIDReusePolicy configures orchestration instance-ID reuse. +func WithOrchestrationIDReusePolicy(policy *OrchestrationIDReusePolicy) NewOrchestrationOptions { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { + if policy == nil || policy.DedupeStatuses == nil { + req.OrchestrationIdReusePolicy = nil + return nil + } + for _, status := range policy.DedupeStatuses { + if !isReusableOrchestrationStatus(status) { + return invalidArgument(fmt.Sprintf("invalid orchestration dedupe status: %s", status)) + } + } + replaceable := make([]protos.OrchestrationStatus, 0, len(reusableOrchestrationStatuses)) + for _, status := range reusableOrchestrationStatuses { + if !slices.Contains(policy.DedupeStatuses, status) { + replaceable = append(replaceable, status) + } + } req.OrchestrationIdReusePolicy = &protos.OrchestrationIdReusePolicy{ - Action: policy.Action, - OperationStatus: policy.OperationStatus, + ReplaceableStatus: replaceable, } return nil } } +var reusableOrchestrationStatuses = [...]OrchestrationStatus{ + RUNTIME_STATUS_RUNNING, + RUNTIME_STATUS_COMPLETED, + RUNTIME_STATUS_FAILED, + RUNTIME_STATUS_CANCELED, + RUNTIME_STATUS_TERMINATED, + RUNTIME_STATUS_PENDING, + RUNTIME_STATUS_SUSPENDED, +} + +func isReusableOrchestrationStatus(status OrchestrationStatus) bool { + return slices.Contains(reusableOrchestrationStatuses[:], status) +} + // WithInput configures an input for the orchestration. The specified input must be serializable. func WithInput(input any) NewOrchestrationOptions { - return func(req *protos.CreateInstanceRequest) error { - bytes, err := json.Marshal(input) + return func(req *protos.CreateInstanceRequest, converter DataConverter) error { + payload, err := SerializeData(converter, input) if err != nil { return err } - req.Input = wrapperspb.String(string(bytes)) + req.Input = wrapperspb.String(payload) return nil } } // WithRawInput configures an input for the orchestration. The specified input must be a string. func WithRawInput(rawInput string) NewOrchestrationOptions { - return func(req *protos.CreateInstanceRequest) error { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { req.Input = wrapperspb.String(rawInput) return nil } @@ -121,12 +162,56 @@ func WithRawInput(rawInput string) NewOrchestrationOptions { // Note that the actual start time could be later than the specified start time if the // task hub is under load or if the app is not running at the specified start time. func WithStartTime(startTime time.Time) NewOrchestrationOptions { - return func(req *protos.CreateInstanceRequest) error { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { req.ScheduledStartTimestamp = timestamppb.New(startTime) return nil } } +// WithVersion configures the orchestration version. +func WithVersion(version string) NewOrchestrationOptions { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { + req.Version = wrapperspb.String(version) + return nil + } +} + +// WithContextFields configures immutable fields propagated into orchestration +// and activity contexts. The fields are persisted with orchestration history. +func WithContextFields(fields ContextFields) NewOrchestrationOptions { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { + for key := range fields { + if strings.HasPrefix(key, ReservedContextFieldPrefix) { + return invalidArgument(fmt.Sprintf("context field %q uses reserved prefix %q", key, ReservedContextFieldPrefix)) + } + if strings.HasPrefix(key, tagcodec.UserTagPrefix) { + return invalidArgument(fmt.Sprintf("context field %q uses reserved prefix %q", key, tagcodec.UserTagPrefix)) + } + } + req.Tags = tagcodec.Merge(req.Tags, tagcodec.EncodeContextFields(fields)) + return nil + } +} + +// WithTags configures orchestration tags that are persisted and returned by metadata queries. +func WithTags(tags map[string]string) NewOrchestrationOptions { + return func(req *protos.CreateInstanceRequest, _ DataConverter) error { + for key := range tags { + if key == "" { + return invalidArgument("tag key cannot be empty") + } + if strings.HasPrefix(key, ReservedContextFieldPrefix) { + return invalidArgument(fmt.Sprintf("tag %q uses reserved prefix %q", key, ReservedContextFieldPrefix)) + } + if strings.HasPrefix(key, tagcodec.UserTagPrefix) { + return invalidArgument(fmt.Sprintf("tag %q uses reserved prefix %q", key, tagcodec.UserTagPrefix)) + } + } + req.Tags = tagcodec.Merge(req.Tags, tagcodec.EncodeUserTags(tags)) + return nil + } +} + // WithFetchPayloads configures whether to load orchestration inputs, outputs, and custom status values, which could be large. func WithFetchPayloads(fetchPayloads bool) FetchOrchestrationMetadataOptions { return func(req *protos.GetInstanceRequest) { @@ -136,19 +221,19 @@ func WithFetchPayloads(fetchPayloads bool) FetchOrchestrationMetadataOptions { // WithEventPayload configures an event payload. The specified payload must be serializable. func WithEventPayload(data any) RaiseEventOptions { - return func(req *protos.RaiseEventRequest) error { - bytes, err := json.Marshal(data) + return func(req *protos.RaiseEventRequest, converter DataConverter) error { + payload, err := SerializeData(converter, data) if err != nil { return err } - req.Input = wrapperspb.String(string(bytes)) + req.Input = wrapperspb.String(payload) return nil } } // WithRawEventData configures an event payload that is a raw, unprocessed string (e.g. JSON data). func WithRawEventData(data string) RaiseEventOptions { - return func(req *protos.RaiseEventRequest) error { + return func(req *protos.RaiseEventRequest, _ DataConverter) error { req.Input = wrapperspb.String(data) return nil } @@ -156,19 +241,19 @@ func WithRawEventData(data string) RaiseEventOptions { // WithOutput configures an output for the terminated orchestration. The specified output must be serializable. func WithOutput(data any) TerminateOptions { - return func(req *protos.TerminateRequest) error { - bytes, err := json.Marshal(data) + return func(req *protos.TerminateRequest, converter DataConverter) error { + payload, err := SerializeData(converter, data) if err != nil { return err } - req.Output = wrapperspb.String(string(bytes)) + req.Output = wrapperspb.String(payload) return nil } } // WithRawOutput configures a raw, unprocessed output (i.e. pre-serialized) for the terminated orchestration. func WithRawOutput(data string) TerminateOptions { - return func(req *protos.TerminateRequest) error { + return func(req *protos.TerminateRequest, _ DataConverter) error { req.Output = wrapperspb.String(data) return nil } @@ -176,12 +261,27 @@ func WithRawOutput(data string) TerminateOptions { // WithRecursiveTerminate configures whether to terminate all sub-orchestrations created by the target orchestration. func WithRecursiveTerminate(recursive bool) TerminateOptions { - return func(req *protos.TerminateRequest) error { + return func(req *protos.TerminateRequest, _ DataConverter) error { req.Recursive = recursive return nil } } +// ReadInput deserializes the orchestration input with the metadata converter. +func (m *OrchestrationMetadata) ReadInput(target any) error { + return deserializePayload(m.Converter, m.SerializedInput, target) +} + +// ReadOutput deserializes the orchestration output with the metadata converter. +func (m *OrchestrationMetadata) ReadOutput(target any) error { + return deserializePayload(m.Converter, m.SerializedOutput, target) +} + +// ReadCustomStatus deserializes custom status with the metadata converter. +func (m *OrchestrationMetadata) ReadCustomStatus(target any) error { + return deserializePayload(m.Converter, m.SerializedCustomStatus, target) +} + // WithRecursivePurge configures whether to purge all sub-orchestrations created by the target orchestration. func WithRecursivePurge(recursive bool) PurgeOptions { return func(req *protos.PurgeInstancesRequest) error { @@ -199,7 +299,7 @@ func NewOrchestrationMetadata( serializedInput string, serializedOutput string, serializedCustomStatus string, - failureDetails *protos.TaskFailureDetails, + failureDetails *FailureDetails, ) *OrchestrationMetadata { return &OrchestrationMetadata{ InstanceID: iid, @@ -214,135 +314,88 @@ func NewOrchestrationMetadata( } } +type orchestrationMetadataJSON struct { + InstanceID *InstanceID `json:"id"` + Name *string `json:"name"` + Status *string `json:"status"` + CreatedAt *time.Time `json:"createdAt"` + LastUpdatedAt *time.Time `json:"lastUpdatedAt"` + Version string `json:"version,omitempty"` + ExecutionID string `json:"executionId,omitempty"` + ParentInstanceID InstanceID `json:"parentInstanceId,omitempty"` + ScheduledStartAt *time.Time `json:"scheduledStartAt,omitempty"` + CompletedAt *time.Time `json:"completedAt,omitempty"` + SerializedInput string `json:"serializedInput,omitempty"` + SerializedOutput string `json:"serializedOutput,omitempty"` + SerializedCustomStatus string `json:"serializedCustomStatus,omitempty"` + FailureDetails *FailureDetails `json:"failureDetails,omitempty"` + Tags map[string]string `json:"tags,omitempty"` +} + func (m *OrchestrationMetadata) MarshalJSON() ([]byte, error) { - obj := make(map[string]any, 16) - - // Required values - obj["id"] = m.InstanceID - obj["name"] = m.Name - obj["status"] = helpers.ToRuntimeStatusString(m.RuntimeStatus) - obj["createdAt"] = m.CreatedAt - obj["lastUpdatedAt"] = m.LastUpdatedAt - - // Optional values - if m.SerializedInput != "" { - obj["serializedInput"] = m.SerializedInput + status := helpers.ToRuntimeStatusString(m.RuntimeStatus) + payload := orchestrationMetadataJSON{ + InstanceID: &m.InstanceID, + Name: &m.Name, + Status: &status, + CreatedAt: &m.CreatedAt, + LastUpdatedAt: &m.LastUpdatedAt, + Version: m.Version, + ExecutionID: m.ExecutionID, + ParentInstanceID: m.ParentInstanceID, + SerializedInput: m.SerializedInput, + SerializedOutput: m.SerializedOutput, + SerializedCustomStatus: m.SerializedCustomStatus, + FailureDetails: m.FailureDetails, + Tags: maps.Clone(m.Tags), } - if m.SerializedOutput != "" { - obj["serializedOutput"] = m.SerializedOutput + if !m.ScheduledStartAt.IsZero() { + payload.ScheduledStartAt = &m.ScheduledStartAt } - if m.SerializedCustomStatus != "" { - obj["serializedCustomStatus"] = m.SerializedCustomStatus - } - - // Optional failure details (recursive) - if m.FailureDetails != nil { - const fieldCount = 4 - root := make(map[string]any, fieldCount) - current := root - f := m.FailureDetails - for { - current["type"] = f.ErrorType - current["message"] = f.ErrorMessage - if f.StackTrace != nil { - current["stackTrace"] = f.StackTrace.GetValue() - } - if f.InnerFailure == nil { - // base case - break - } - // recursive case - f = f.InnerFailure - inner := make(map[string]any, fieldCount) - current["innerFailure"] = inner - current = inner - } - obj["failureDetails"] = root + if !m.CompletedAt.IsZero() { + payload.CompletedAt = &m.CompletedAt } - return json.Marshal(obj) + return json.Marshal(payload) } -func (m *OrchestrationMetadata) UnmarshalJSON(data []byte) (err error) { - defer func() { - if r := recover(); r != nil { - if rerr, ok := r.(error); ok { - err = fmt.Errorf("failed to unmarshal the JSON payload: %w", rerr) - } else { - err = errors.New("failed to unmarshal the JSON payload") - } - } - }() - - var obj map[string]any - if err := json.Unmarshal(data, &obj); err != nil { +func (m *OrchestrationMetadata) UnmarshalJSON(data []byte) error { + var payload orchestrationMetadataJSON + if err := json.Unmarshal(data, &payload); err != nil { return fmt.Errorf("failed to unmarshal orchestration metadata json: %w", err) } - - if id, ok := obj["id"]; ok { - m.InstanceID = InstanceID(id.(string)) - } else { + if payload.InstanceID == nil { return errors.New("missing 'id' field") } - if name, ok := obj["name"]; ok { - m.Name = name.(string) - } else { + if payload.Name == nil { return errors.New("missing 'name' field") } - if status, ok := obj["status"]; ok { - m.RuntimeStatus = helpers.FromRuntimeStatusString(status.(string)) - } else { - return errors.New("missing 'name' field") + if payload.Status == nil { + return errors.New("missing 'status' field") } - if createdAt, ok := obj["createdAt"]; ok { - if time, err := time.Parse(time.RFC3339, createdAt.(string)); err == nil { - m.CreatedAt = time - } else { - return errors.New("invalid 'createdAt' field: must be RFC3339 format") - } - } else { + if payload.CreatedAt == nil { return errors.New("missing 'createdAt' field") } - if lastUpdatedAt, ok := obj["lastUpdatedAt"]; ok { - if time, err := time.Parse(time.RFC3339, lastUpdatedAt.(string)); err == nil { - m.LastUpdatedAt = time - } else { - return errors.New("invalid 'lastUpdatedAt' field: must be RFC3339 format") - } - } else { + if payload.LastUpdatedAt == nil { return errors.New("missing 'lastUpdatedAt' field") } - if input, ok := obj["serializedInput"]; ok { - m.SerializedInput = input.(string) - } - if output, ok := obj["serializedOutput"]; ok { - m.SerializedOutput = output.(string) + m.InstanceID = *payload.InstanceID + m.Name = *payload.Name + m.RuntimeStatus = helpers.FromRuntimeStatusString(*payload.Status) + m.CreatedAt = *payload.CreatedAt + m.LastUpdatedAt = *payload.LastUpdatedAt + m.Version = payload.Version + m.ExecutionID = payload.ExecutionID + m.ParentInstanceID = payload.ParentInstanceID + m.SerializedInput = payload.SerializedInput + m.SerializedOutput = payload.SerializedOutput + m.SerializedCustomStatus = payload.SerializedCustomStatus + m.FailureDetails = payload.FailureDetails + m.Tags = maps.Clone(payload.Tags) + if payload.ScheduledStartAt != nil { + m.ScheduledStartAt = *payload.ScheduledStartAt } - if output, ok := obj["serializedCustomStatus"]; ok { - m.SerializedCustomStatus = output.(string) - } - - failureDetails, ok := obj["failureDetails"] - if ok { - m.FailureDetails = &protos.TaskFailureDetails{} - current := m.FailureDetails - obj = failureDetails.(map[string]any) - for { - current.ErrorType = obj["type"].(string) - current.ErrorMessage = obj["message"].(string) - if stackTrace, ok := obj["stackTrace"]; ok { - current.StackTrace = wrapperspb.String(stackTrace.(string)) - } - if innerFailure, ok := obj["innerFailure"]; ok { - // recursive case - next := &protos.TaskFailureDetails{} - current.InnerFailure = next - current = next - obj = innerFailure.(map[string]any) - } else { - // base case - break - } - } + if payload.CompletedAt != nil { + m.CompletedAt = *payload.CompletedAt } return nil } diff --git a/api/orchestration_test.go b/api/orchestration_test.go new file mode 100644 index 00000000..4a82afd6 --- /dev/null +++ b/api/orchestration_test.go @@ -0,0 +1,116 @@ +package api + +import ( + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +func Test_API_WithInstanceID_RejectsEntityFormat(t *testing.T) { + req := &protos.CreateInstanceRequest{} + + err := WithInstanceID(InstanceID("@counter@key"))(req, DefaultDataConverter()) + require.Error(t, err) +} + +func Test_API_WithInstanceID_AllowsNormalValue(t *testing.T) { + req := &protos.CreateInstanceRequest{} + + err := WithInstanceID(InstanceID("my-instance"))(req, DefaultDataConverter()) + require.NoError(t, err) + require.Equal(t, "my-instance", req.InstanceId) +} + +func Test_API_WithOrchestrationIDReusePolicy_RejectsInvalidStatus(t *testing.T) { + req := &protos.CreateInstanceRequest{} + err := WithOrchestrationIDReusePolicy(&OrchestrationIDReusePolicy{ + DedupeStatuses: []OrchestrationStatus{OrchestrationStatus(99)}, + })(req, DefaultDataConverter()) + require.ErrorContains(t, err, "invalid orchestration dedupe status") +} + +func Test_API_WithOrchestrationIDReusePolicy_DeduplicatesStatuses(t *testing.T) { + req := &protos.CreateInstanceRequest{} + err := WithOrchestrationIDReusePolicy(&OrchestrationIDReusePolicy{ + DedupeStatuses: []OrchestrationStatus{ + RUNTIME_STATUS_RUNNING, + RUNTIME_STATUS_RUNNING, + }, + })(req, DefaultDataConverter()) + require.NoError(t, err) + require.NotContains( + t, + req.OrchestrationIdReusePolicy.ReplaceableStatus, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING, + ) + require.Len(t, req.OrchestrationIdReusePolicy.ReplaceableStatus, len(reusableOrchestrationStatuses)-1) +} + +func Test_API_WithOrchestrationIDReusePolicy_AllStatusesReplaceNothing(t *testing.T) { + req := &protos.CreateInstanceRequest{} + err := WithOrchestrationIDReusePolicy(&OrchestrationIDReusePolicy{ + DedupeStatuses: append([]OrchestrationStatus(nil), reusableOrchestrationStatuses[:]...), + })(req, DefaultDataConverter()) + require.NoError(t, err) + require.NotNil(t, req.OrchestrationIdReusePolicy) + require.Empty(t, req.OrchestrationIdReusePolicy.ReplaceableStatus) +} + +func Test_API_ReusableOrchestrationStatusesTrackStableProtoStatuses(t *testing.T) { + require.Len(t, reusableOrchestrationStatuses, len(protos.OrchestrationStatus_name)-1) + for status := range protos.OrchestrationStatus_name { + value := OrchestrationStatus(status) + if value == RUNTIME_STATUS_CONTINUED_AS_NEW { + require.False(t, isReusableOrchestrationStatus(value)) + continue + } + require.True(t, isReusableOrchestrationStatus(value), value.String()) + } +} + +// WithRecursiveTerminate and WithRecursivePurge are the only way callers reach +// the DTS-side recursive flags, so both directions must survive on the wire. +func Test_API_WithRecursiveTerminate_SetsWireFlag(t *testing.T) { + for _, recursive := range []bool{true, false} { + req := &protos.TerminateRequest{Recursive: !recursive} + require.NoError(t, WithRecursiveTerminate(recursive)(req, DefaultDataConverter())) + require.Equal(t, recursive, req.Recursive) + } +} + +func Test_API_WithRecursivePurge_SetsWireFlag(t *testing.T) { + for _, recursive := range []bool{true, false} { + req := &protos.PurgeInstancesRequest{Recursive: !recursive} + require.NoError(t, WithRecursivePurge(recursive)(req)) + require.Equal(t, recursive, req.Recursive) + } +} + +// IsComplete decides when DTS client wait loops stop polling, so every terminal +// status must be treated as complete and every non-terminal one as running. +func Test_API_OrchestrationMetadata_RunningAndCompleteStatuses(t *testing.T) { + complete := []protos.OrchestrationStatus{ + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CANCELED, + } + running := []protos.OrchestrationStatus{ + protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_PENDING, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, + } + + for _, status := range complete { + metadata := &OrchestrationMetadata{RuntimeStatus: status} + require.Truef(t, metadata.IsComplete(), "status %v should be complete", status) + require.Falsef(t, metadata.IsRunning(), "status %v should not be running", status) + } + for _, status := range running { + metadata := &OrchestrationMetadata{RuntimeStatus: status} + require.Falsef(t, metadata.IsComplete(), "status %v should not be complete", status) + require.Truef(t, metadata.IsRunning(), "status %v should be running", status) + } +} diff --git a/backend/activity.go b/backend/activity.go deleted file mode 100644 index 8f3d8df7..00000000 --- a/backend/activity.go +++ /dev/null @@ -1,102 +0,0 @@ -package backend - -import ( - "context" - "fmt" - - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" -) - -type activityProcessor struct { - be Backend - executor ActivityExecutor -} - -type ActivityExecutor interface { - ExecuteActivity(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) -} - -func NewActivityTaskWorker(be Backend, executor ActivityExecutor, logger Logger, opts ...NewTaskWorkerOptions) TaskWorker { - processor := newActivityProcessor(be, executor) - return NewTaskWorker(processor, logger, opts...) -} - -func newActivityProcessor(be Backend, executor ActivityExecutor) TaskProcessor { - return &activityProcessor{ - be: be, - executor: executor, - } -} - -// Name implements TaskProcessor -func (*activityProcessor) Name() string { - return "activity-processor" -} - -// FetchWorkItem implements TaskDispatcher -func (ap *activityProcessor) FetchWorkItem(ctx context.Context) (WorkItem, error) { - return ap.be.GetActivityWorkItem(ctx) -} - -// ProcessWorkItem implements TaskDispatcher -func (p *activityProcessor) ProcessWorkItem(ctx context.Context, wi WorkItem) error { - awi := wi.(*ActivityWorkItem) - - ts := awi.NewEvent.GetTaskScheduled() - if ts == nil { - return fmt.Errorf("%v: invalid TaskScheduled event", awi.InstanceID) - } - - // Create span as child of spanContext found in TaskScheduledEvent - ctx, err := helpers.ContextFromTraceContext(ctx, ts.ParentTraceContext) - if err != nil { - return fmt.Errorf("%v: failed to parse activity trace context: %w", awi.InstanceID, err) - } - var span trace.Span - ctx, span = helpers.StartNewActivitySpan(ctx, ts.Name, ts.Version.GetValue(), string(awi.InstanceID), awi.NewEvent.EventId) - if span != nil { - defer func() { - if r := recover(); r != nil { - span.SetStatus(codes.Error, fmt.Sprintf("%v", r)) - } - span.End() - }() - } - - // Execute the activity and get its result - result, err := p.executor.ExecuteActivity(ctx, awi.InstanceID, awi.NewEvent) - if err != nil { - if span != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - } - return err - } - - awi.Result = result - return nil -} - -// CompleteWorkItem implements TaskDispatcher -func (ap *activityProcessor) CompleteWorkItem(ctx context.Context, wi WorkItem) error { - awi := wi.(*ActivityWorkItem) - if awi.Result == nil { - return fmt.Errorf("can't complete work item '%s' with nil result", wi) - } - if awi.Result.GetTaskCompleted() == nil && awi.Result.GetTaskFailed() == nil { - return fmt.Errorf("can't complete work item '%s', which isn't TaskCompleted or TaskFailed", wi) - } - - return ap.be.CompleteActivityWorkItem(ctx, awi) -} - -// AbandonWorkItem implements TaskDispatcher -func (ap *activityProcessor) AbandonWorkItem(ctx context.Context, wi WorkItem) error { - awi := wi.(*ActivityWorkItem) - return ap.be.AbandonActivityWorkItem(ctx, awi) -} diff --git a/backend/backend.go b/backend/backend.go deleted file mode 100644 index f5c77758..00000000 --- a/backend/backend.go +++ /dev/null @@ -1,203 +0,0 @@ -package backend - -import ( - "context" - "errors" - "fmt" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "google.golang.org/protobuf/proto" -) - -var ( - ErrTaskHubExists = errors.New("task hub already exists") - ErrTaskHubNotFound = errors.New("task hub not found") - ErrNotInitialized = errors.New("backend not initialized") - ErrWorkItemLockLost = errors.New("lock on work-item was lost") - ErrBackendAlreadyStarted = errors.New("backend is already started") - ErrOperationAborted = errors.New("operation aborted") - ErrNilHistoryEvent = errors.New("HistoryEvent must be non-nil") - ErrNilEventTimestamp = errors.New("HistoryEvent must have a non-nil timestamp") - ErrNotExecutionStarted = errors.New("HistoryEvent must be an ExecutionStartedEvent") -) - -type ( - HistoryEvent = protos.HistoryEvent - TaskFailureDetails = protos.TaskFailureDetails -) - -type OrchestrationIdReusePolicyOptions func(*protos.OrchestrationIdReusePolicy) error - -func WithOrchestrationIdReusePolicy(policy *protos.OrchestrationIdReusePolicy) OrchestrationIdReusePolicyOptions { - return func(po *protos.OrchestrationIdReusePolicy) error { - if policy != nil { - po.Action = policy.Action - po.OperationStatus = policy.OperationStatus - } - return nil - } -} - -type Backend interface { - // CreateTaskHub creates a new task hub for the current backend. Task hub creation must be idempotent. - // - // If the task hub for this backend already exists, an error of type [ErrTaskHubExists] is returned. - CreateTaskHub(context.Context) error - - // DeleteTaskHub deletes an existing task hub configured for the current backend. It's up to the backend - // implementation to determine how the task hub data is deleted. - // - // If the task hub for this backend doesn't exist, an error of type [ErrTaskHubNotFound] is returned. - DeleteTaskHub(context.Context) error - - // Start starts any background processing done by this backend. - Start(context.Context) error - - // Stop stops any background processing done by this backend. - Stop(context.Context) error - - // CreateOrchestrationInstance creates a new orchestration instance with a history event that - // wraps a ExecutionStarted event. - CreateOrchestrationInstance(context.Context, *HistoryEvent, ...OrchestrationIdReusePolicyOptions) error - - // AddNewEvent adds a new orchestration event to the specified orchestration instance. - AddNewOrchestrationEvent(context.Context, api.InstanceID, *HistoryEvent) error - - // GetOrchestrationWorkItem gets a pending work item from the task hub or returns [ErrNoOrchWorkItems] - // if there are no pending work items. - GetOrchestrationWorkItem(context.Context) (*OrchestrationWorkItem, error) - - // GetOrchestrationRuntimeState gets the runtime state of an orchestration instance. - GetOrchestrationRuntimeState(context.Context, *OrchestrationWorkItem) (*OrchestrationRuntimeState, error) - - // GetOrchestrationMetadata gets the metadata associated with the given orchestration instance ID. - // - // Returns [api.ErrInstanceNotFound] if the orchestration instance doesn't exist. - GetOrchestrationMetadata(context.Context, api.InstanceID) (*api.OrchestrationMetadata, error) - - // CompleteOrchestrationWorkItem completes a work item by saving the updated runtime state to durable storage. - // - // Returns [ErrWorkItemLockLost] if the work-item couldn't be completed due to a lock-lost conflict (e.g., split-brain). - CompleteOrchestrationWorkItem(context.Context, *OrchestrationWorkItem) error - - // AbandonOrchestrationWorkItem undoes any state changes and returns the work item to the work item queue. - // - // This is called if an internal failure happens in the processing of an orchestration work item. It is - // not called if the orchestration work item is processed successfully (note that an orchestration that - // completes with a failure is still considered a successfully processed work item). - AbandonOrchestrationWorkItem(context.Context, *OrchestrationWorkItem) error - - // GetActivityWorkItem gets a pending activity work item from the task hub or returns [ErrNoWorkItems] - // if there are no pending activity work items. - GetActivityWorkItem(context.Context) (*ActivityWorkItem, error) - - // CompleteActivityWorkItem sends a message to the parent orchestration indicating activity completion. - // - // Returns [ErrWorkItemLockLost] if the work-item couldn't be completed due to a lock-lost conflict (e.g., split-brain). - CompleteActivityWorkItem(context.Context, *ActivityWorkItem) error - - // AbandonActivityWorkItem returns the work-item back to the queue without committing any other chances. - // - // This is called when an internal failure occurs during activity work-item processing. - AbandonActivityWorkItem(context.Context, *ActivityWorkItem) error - - // PurgeOrchestrationState deletes all saved state for the specified orchestration instance. - // - // [api.ErrInstanceNotFound] is returned if the specified orchestration instance doesn't exist. - // [api.ErrNotCompleted] is returned if the specified orchestration instance is still running. - PurgeOrchestrationState(context.Context, api.InstanceID) error -} - -// MarshalHistoryEvent serializes the [HistoryEvent] into a protobuf byte array. -func MarshalHistoryEvent(e *HistoryEvent) ([]byte, error) { - if bytes, err := proto.Marshal(e); err != nil { - return nil, fmt.Errorf("failed to marshal history event: %w", err) - } else { - return bytes, nil - } -} - -// UnmarshalHistoryEvent deserializes a [HistoryEvent] from a protobuf byte array. -func UnmarshalHistoryEvent(bytes []byte) (*HistoryEvent, error) { - e := &protos.HistoryEvent{} - if err := proto.Unmarshal(bytes, e); err != nil { - return nil, fmt.Errorf("unreadable history event payload: %w", err) - } - return e, nil -} - -// purgeOrchestrationState purges the orchestration state, including sub-orchestrations if [recursive] is true. -// Returns (deletedInstanceCount, error), where deletedInstanceCount is the number of instances deleted. -func purgeOrchestrationState(ctx context.Context, be Backend, iid api.InstanceID, recursive bool) (int, error) { - deletedInstanceCount := 0 - if recursive { - owi := &OrchestrationWorkItem{ - InstanceID: iid, - } - state, err := be.GetOrchestrationRuntimeState(ctx, owi) - if err != nil { - return 0, fmt.Errorf("failed to fetch orchestration state: %w", err) - } - if len(state.NewEvents())+len(state.oldEvents) == 0 { - // If there are no events, the orchestration instance doesn't exist - return 0, api.ErrInstanceNotFound - } - if !state.IsCompleted() { - // Orchestration must be completed before purging its state - return 0, api.ErrNotCompleted - } - subOrchestrationInstances := getSubOrchestrationInstances(state.OldEvents(), state.NewEvents()) - for _, subOrchestrationInstance := range subOrchestrationInstances { - // Recursively purge sub-orchestrations - count, err := purgeOrchestrationState(ctx, be, subOrchestrationInstance, recursive) - // `count` sub-orchestrations have been successfully purged (even in case of error) - deletedInstanceCount += count - if err != nil { - return deletedInstanceCount, fmt.Errorf("failed to purge sub-orchestration: %w", err) - } - } - } - // Purging root orchestration - if err := be.PurgeOrchestrationState(ctx, iid); err != nil { - return deletedInstanceCount, err - } - return deletedInstanceCount + 1, nil -} - -// terminateSubOrchestrationInstances submits termination requests to sub-orchestrations if [et.Recurse] is true. -func terminateSubOrchestrationInstances(ctx context.Context, be Backend, iid api.InstanceID, state *OrchestrationRuntimeState, et *protos.ExecutionTerminatedEvent) error { - if !et.Recurse { - return nil - } - subOrchestrationInstances := getSubOrchestrationInstances(state.OldEvents(), state.NewEvents()) - for _, subOrchestrationInstance := range subOrchestrationInstances { - e := helpers.NewExecutionTerminatedEvent(et.Input, et.Recurse) - // Adding terminate event to sub-orchestration instance - if err := be.AddNewOrchestrationEvent(ctx, subOrchestrationInstance, e); err != nil { - return fmt.Errorf("failed to submit termination request to sub-orchestration: %w", err) - } - } - return nil -} - -// getSubOrchestrationInstances returns the instance IDs of all sub-orchestrations in the specified events. -func getSubOrchestrationInstances(oldEvents []*HistoryEvent, newEvents []*HistoryEvent) []api.InstanceID { - subOrchestrationInstancesMap := make(map[api.InstanceID]struct{}, len(oldEvents)+len(newEvents)) - for _, e := range oldEvents { - if created := e.GetSubOrchestrationInstanceCreated(); created != nil { - subOrchestrationInstancesMap[api.InstanceID(created.InstanceId)] = struct{}{} - } - } - for _, e := range newEvents { - if created := e.GetSubOrchestrationInstanceCreated(); created != nil { - subOrchestrationInstancesMap[api.InstanceID(created.InstanceId)] = struct{}{} - } - } - subOrchestrationInstances := make([]api.InstanceID, 0, len(subOrchestrationInstancesMap)) - for orch := range subOrchestrationInstancesMap { - subOrchestrationInstances = append(subOrchestrationInstances, orch) - } - return subOrchestrationInstances -} diff --git a/backend/client.go b/backend/client.go deleted file mode 100644 index 51f27d97..00000000 --- a/backend/client.go +++ /dev/null @@ -1,207 +0,0 @@ -package backend - -import ( - "context" - "fmt" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/google/uuid" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" -) - -type TaskHubClient interface { - ScheduleNewOrchestration(ctx context.Context, orchestrator any, opts ...api.NewOrchestrationOptions) (api.InstanceID, error) - FetchOrchestrationMetadata(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) - WaitForOrchestrationStart(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) - WaitForOrchestrationCompletion(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) - TerminateOrchestration(ctx context.Context, id api.InstanceID, opts ...api.TerminateOptions) error - RaiseEvent(ctx context.Context, id api.InstanceID, eventName string, opts ...api.RaiseEventOptions) error - SuspendOrchestration(ctx context.Context, id api.InstanceID, reason string) error - ResumeOrchestration(ctx context.Context, id api.InstanceID, reason string) error - PurgeOrchestrationState(ctx context.Context, id api.InstanceID, opts ...api.PurgeOptions) error -} - -type backendClient struct { - be Backend -} - -func NewTaskHubClient(be Backend) TaskHubClient { - return &backendClient{ - be: be, - } -} - -func (c *backendClient) ScheduleNewOrchestration(ctx context.Context, orchestrator any, opts ...api.NewOrchestrationOptions) (api.InstanceID, error) { - name := helpers.GetTaskFunctionName(orchestrator) - req := &protos.CreateInstanceRequest{Name: name} - for _, configure := range opts { - if err := configure(req); err != nil { - return api.EmptyInstanceID, fmt.Errorf("failed to configure create instance request: %w", err) - } - } - if req.InstanceId == "" { - u, err := uuid.NewV7() - if err != nil { - return api.EmptyInstanceID, fmt.Errorf("failed to generate instance ID: %w", err) - } - req.InstanceId = u.String() - } - - var span trace.Span - ctx, span = helpers.StartNewCreateOrchestrationSpan(ctx, req.Name, req.Version.GetValue(), req.InstanceId) - defer span.End() - - tc := helpers.TraceContextFromSpan(span) - e := helpers.NewExecutionStartedEvent(req.Name, req.InstanceId, req.Input, nil, tc, req.ScheduledStartTimestamp) - if err := c.be.CreateOrchestrationInstance(ctx, e, WithOrchestrationIdReusePolicy(req.OrchestrationIdReusePolicy)); err != nil { - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) - return api.EmptyInstanceID, fmt.Errorf("failed to start orchestration: %w", err) - } - return api.InstanceID(req.InstanceId), nil -} - -// FetchOrchestrationMetadata fetches metadata for the specified orchestration from the configured task hub. -// -// ErrInstanceNotFound is returned when the specified orchestration doesn't exist. -func (c *backendClient) FetchOrchestrationMetadata(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) { - metadata, err := c.be.GetOrchestrationMetadata(ctx, id) - if err != nil { - return nil, fmt.Errorf("failed to fetch orchestration metadata: %w", err) - } - return metadata, nil -} - -// WaitForOrchestrationStart waits for an orchestration to start running and returns an [OrchestrationMetadata] object that contains -// metadata about the started instance. -// -// ErrInstanceNotFound is returned when the specified orchestration doesn't exist. -func (c *backendClient) WaitForOrchestrationStart(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) { - return c.waitForOrchestrationCondition(ctx, id, func(metadata *api.OrchestrationMetadata) bool { - return metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_PENDING - }) -} - -// WaitForOrchestrationCompletion waits for an orchestration to complete and returns an [OrchestrationMetadata] object that contains -// metadata about the completed instance. -// -// ErrInstanceNotFound is returned when the specified orchestration doesn't exist. -func (c *backendClient) WaitForOrchestrationCompletion(ctx context.Context, id api.InstanceID) (*api.OrchestrationMetadata, error) { - return c.waitForOrchestrationCondition(ctx, id, func(metadata *api.OrchestrationMetadata) bool { - return metadata.IsComplete() - }) -} - -func (c *backendClient) waitForOrchestrationCondition(ctx context.Context, id api.InstanceID, condition func(metadata *api.OrchestrationMetadata) bool) (*api.OrchestrationMetadata, error) { - b := backoff.ExponentialBackOff{ - InitialInterval: 100 * time.Millisecond, - MaxInterval: 10 * time.Second, - Multiplier: 1.5, - RandomizationFactor: 0.05, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - } - b.Reset() - - for { - t := time.NewTimer(b.NextBackOff()) - select { - case <-ctx.Done(): - if !t.Stop() { - <-t.C - } - return nil, ctx.Err() - case <-t.C: - metadata, err := c.FetchOrchestrationMetadata(ctx, id) - if err != nil { - return nil, err - } - if metadata != nil && condition(metadata) { - return metadata, nil - } - } - } -} - -// TerminateOrchestration enqueues a message to terminate a running orchestration, causing it to stop receiving new events and -// go directly into the TERMINATED state. This operation is asynchronous. An orchestration worker must -// dequeue the termination event before the orchestration will be terminated. -func (c *backendClient) TerminateOrchestration(ctx context.Context, id api.InstanceID, opts ...api.TerminateOptions) error { - req := &protos.TerminateRequest{InstanceId: string(id), Recursive: true} - for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure termination request: %w", err) - } - } - e := helpers.NewExecutionTerminatedEvent(req.Output, req.Recursive) - if err := c.be.AddNewOrchestrationEvent(ctx, id, e); err != nil { - return fmt.Errorf("failed to submit termination request:: %w", err) - } - return nil -} - -// RaiseEvent implements TaskHubClient and sends an asynchronous event notification to a waiting orchestration. -// -// In order to handle the event, the target orchestration instance must be waiting for an event named [eventName] -// using the [WaitForSingleEvent] method of the orchestration context parameter. If the target orchestration instance -// is not yet waiting for an event named [eventName], then the event will be bufferred in memory until a task -// subscribing to that event name is created. -// -// Raised events for a completed or non-existent orchestration instance will be silently discarded. -func (c *backendClient) RaiseEvent(ctx context.Context, id api.InstanceID, eventName string, opts ...api.RaiseEventOptions) error { - req := &protos.RaiseEventRequest{InstanceId: string(id), Name: eventName} - for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure raise event request: %w", err) - } - } - - e := helpers.NewEventRaisedEvent(req.Name, req.Input) - if err := c.be.AddNewOrchestrationEvent(ctx, id, e); err != nil { - return fmt.Errorf("failed to raise event: %w", err) - } - return nil -} - -// SuspendOrchestration suspends an orchestration instance, halting processing of its events until a "resume" operation resumes it. -// -// Note that suspended orchestrations are still considered to be "running" even though they will not process events. -func (c *backendClient) SuspendOrchestration(ctx context.Context, id api.InstanceID, reason string) error { - e := helpers.NewSuspendOrchestrationEvent(reason) - if err := c.be.AddNewOrchestrationEvent(ctx, id, e); err != nil { - return fmt.Errorf("failed to suspend orchestration: %w", err) - } - return nil -} - -// ResumeOrchestration resumes an orchestration instance that was previously suspended. -func (c *backendClient) ResumeOrchestration(ctx context.Context, id api.InstanceID, reason string) error { - e := helpers.NewResumeOrchestrationEvent(reason) - if err := c.be.AddNewOrchestrationEvent(ctx, id, e); err != nil { - return fmt.Errorf("failed to resume orchestration: %w", err) - } - return nil -} - -// PurgeOrchestrationState deletes the state of the specified orchestration instance. -// -// [api.ErrInstanceNotFound] is returned if the specified orchestration instance doesn't exist. -// [api.ErrNotCompleted] is returned if the specified orchestration instance is still running. -func (c *backendClient) PurgeOrchestrationState(ctx context.Context, id api.InstanceID, opts ...api.PurgeOptions) error { - req := &protos.PurgeInstancesRequest{Request: &protos.PurgeInstancesRequest_InstanceId{InstanceId: string(id)}, Recursive: true} - for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure purge request: %w", err) - } - } - if _, err := purgeOrchestrationState(ctx, c.be, id, req.Recursive); err != nil { - return fmt.Errorf("failed to purge orchestration state: %w", err) - } - return nil -} diff --git a/backend/executor.go b/backend/executor.go deleted file mode 100644 index 570988cd..00000000 --- a/backend/executor.go +++ /dev/null @@ -1,534 +0,0 @@ -package backend - -import ( - context "context" - "errors" - "fmt" - "strconv" - "strings" - "sync" - "time" - - "github.com/cenkalti/backoff/v4" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/metadata" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/emptypb" - "google.golang.org/protobuf/types/known/timestamppb" - "google.golang.org/protobuf/types/known/wrapperspb" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" -) - -var emptyCompleteTaskResponse = &protos.CompleteTaskResponse{} - -var errShuttingDown error = status.Error(codes.Canceled, "shutting down") - -type ExecutionResults struct { - Response *protos.OrchestratorResponse - complete chan struct{} - pending chan string -} - -type activityExecutionResult struct { - response *protos.ActivityResponse - complete chan struct{} - pending chan string -} - -type Executor interface { - ExecuteOrchestrator(ctx context.Context, iid api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent) (*ExecutionResults, error) - ExecuteActivity(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) - Shutdown(ctx context.Context) error -} - -type grpcExecutor struct { - protos.UnimplementedTaskHubSidecarServiceServer - workItemQueue chan *protos.WorkItem - pendingOrchestrators *sync.Map // map[api.InstanceID]*ExecutionResults - pendingActivities *sync.Map // map[string]*activityExecutionResult - backend Backend - logger Logger - onWorkItemConnection func(context.Context) error - streamShutdownChan <-chan any -} - -type grpcExecutorOptions func(g *grpcExecutor) - -// IsDurableTaskGrpcRequest returns true if the specified gRPC method name represents an operation -// that is compatible with the gRPC executor. -func IsDurableTaskGrpcRequest(fullMethodName string) bool { - return strings.HasPrefix(fullMethodName, "/TaskHubSidecarService/") -} - -// WithOnGetWorkItemsConnectionCallback allows the caller to get a notification when an external process -// connects over gRPC and invokes the GetWorkItems operation. This can be useful for doing things like -// lazily auto-starting the task hub worker only when necessary. -func WithOnGetWorkItemsConnectionCallback(callback func(context.Context) error) grpcExecutorOptions { - return func(g *grpcExecutor) { - g.onWorkItemConnection = callback - } -} - -func WithStreamShutdownChannel(c <-chan any) grpcExecutorOptions { - return func(g *grpcExecutor) { - g.streamShutdownChan = c - } -} - -// NewGrpcExecutor returns the Executor object and a method to invoke to register the gRPC server in the executor. -func NewGrpcExecutor(be Backend, logger Logger, opts ...grpcExecutorOptions) (executor Executor, registerServerFn func(grpcServer grpc.ServiceRegistrar)) { - grpcExecutor := &grpcExecutor{ - workItemQueue: make(chan *protos.WorkItem), - backend: be, - logger: logger, - pendingOrchestrators: &sync.Map{}, - pendingActivities: &sync.Map{}, - } - - for _, opt := range opts { - opt(grpcExecutor) - } - - return grpcExecutor, func(grpcServer grpc.ServiceRegistrar) { - protos.RegisterTaskHubSidecarServiceServer(grpcServer, grpcExecutor) - } -} - -// ExecuteOrchestrator implements Executor -func (executor *grpcExecutor) ExecuteOrchestrator(ctx context.Context, iid api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent) (*ExecutionResults, error) { - result := &ExecutionResults{complete: make(chan struct{})} - executor.pendingOrchestrators.Store(iid, result) - - workItem := &protos.WorkItem{ - Request: &protos.WorkItem_OrchestratorRequest{ - OrchestratorRequest: &protos.OrchestratorRequest{ - InstanceId: string(iid), - ExecutionId: nil, - PastEvents: oldEvents, - NewEvents: newEvents, - }, - }, - } - - // Send the orchestration execution work-item to the connected worker. - // This will block if the worker isn't listening for work items. - select { - case <-ctx.Done(): - executor.logger.Warnf("%s: context canceled before dispatching orchestrator work item", iid) - return nil, ctx.Err() - case executor.workItemQueue <- workItem: - } - - // Wait for the connected worker to signal that it's done executing the work-item - select { - case <-ctx.Done(): - executor.logger.Warnf("%s: context canceled before receiving orchestrator result", iid) - return nil, ctx.Err() - case <-result.complete: - executor.logger.Debugf("%s: orchestrator got result", iid) - if result.Response == nil { - return nil, ErrOperationAborted - } - } - - return result, nil -} - -// ExecuteActivity implements Executor -func (executor *grpcExecutor) ExecuteActivity(ctx context.Context, iid api.InstanceID, e *protos.HistoryEvent) (*protos.HistoryEvent, error) { - key := getActivityExecutionKey(string(iid), e.EventId) - result := &activityExecutionResult{complete: make(chan struct{})} - executor.pendingActivities.Store(key, result) - - task := e.GetTaskScheduled() - workItem := &protos.WorkItem{ - Request: &protos.WorkItem_ActivityRequest{ - ActivityRequest: &protos.ActivityRequest{ - Name: task.Name, - Version: task.Version, - Input: task.Input, - OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: string(iid)}, - TaskId: e.EventId, - }, - }, - } - - // Send the activity execution work-item to the connected worker. - // This will block if the worker isn't listening for work items. - select { - case <-ctx.Done(): - executor.logger.Warnf("%s/%s#%d: context canceled before dispatching activity work item", iid, task.Name, e.EventId) - return nil, ctx.Err() - case executor.workItemQueue <- workItem: - } - - // Wait for the connected worker to signal that it's done executing the work-item - select { - case <-ctx.Done(): - executor.logger.Warnf("%s/%s#%d: context canceled before receiving activity result", iid, task.Name, e.EventId) - return nil, ctx.Err() - case <-result.complete: - executor.logger.Debugf("%s: activity got result", key) - if result.response == nil { - return nil, ErrOperationAborted - } - } - - var responseEvent *protos.HistoryEvent - if failureDetails := result.response.GetFailureDetails(); failureDetails != nil { - responseEvent = helpers.NewTaskFailedEvent(result.response.TaskId, result.response.FailureDetails) - } else { - responseEvent = helpers.NewTaskCompletedEvent(result.response.TaskId, result.response.Result) - } - - return responseEvent, nil -} - -// Shutdown implements Executor -func (g *grpcExecutor) Shutdown(ctx context.Context) error { - // closing the work item queue is a signal for shutdown - close(g.workItemQueue) - - // Iterate through all pending items and close them to unblock the goroutines waiting on this - g.pendingActivities.Range(func(_, value any) bool { - p, ok := value.(*activityExecutionResult) - if ok { - close(p.complete) - } - return true - }) - g.pendingOrchestrators.Range(func(_, value any) bool { - p, ok := value.(*ExecutionResults) - if ok { - close(p.complete) - } - return true - }) - - return nil -} - -// Hello implements protos.TaskHubSidecarServiceServer -func (grpcExecutor) Hello(ctx context.Context, empty *emptypb.Empty) (*emptypb.Empty, error) { - return empty, nil -} - -// GetWorkItems implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) GetWorkItems(req *protos.GetWorkItemsRequest, stream protos.TaskHubSidecarService_GetWorkItemsServer) error { - if md, ok := metadata.FromIncomingContext(stream.Context()); ok { - g.logger.Infof("work item stream established by user-agent: %v", md.Get("user-agent")) - } - - // There are some cases where the app may need to be notified when a client connects to fetch work items, like - // for auto-starting the worker. The app also has an opportunity to set itself as unavailable by returning an error. - callback := g.onWorkItemConnection - if callback != nil { - if err := callback(stream.Context()); err != nil { - message := "unable to establish work item stream at this time: " + err.Error() - g.logger.Warn(message) - return status.Errorf(codes.Unavailable, message) - } - } - - // Collect all pending activities on this stream - // Note: we don't need sync.Map's here because access is only on this thread - pendingActivities := make(map[string]struct{}) - pendingActivityCh := make(chan string, 1) - pendingOrchestrators := make(map[string]struct{}) - pendingOrchestratorCh := make(chan string, 1) - defer func() { - // If there's any pending activity left, remove them - for key := range pendingActivities { - g.logger.Debugf("cleaning up pending activity: %s", key) - p, ok := g.pendingActivities.LoadAndDelete(key) - if ok { - pending := p.(*activityExecutionResult) - close(pending.complete) - } - } - for key := range pendingOrchestrators { - g.logger.Debugf("cleaning up pending orchestrator: %s", key) - p, ok := g.pendingOrchestrators.LoadAndDelete(api.InstanceID(key)) - if ok { - pending := p.(*ExecutionResults) - close(pending.complete) - } - } - }() - - // The worker client invokes this method, which streams back work-items as they arrive. - for { - select { - case <-stream.Context().Done(): - g.logger.Info("work item stream closed") - return nil - case wi, ok := <-g.workItemQueue: - if !ok { - continue - } - switch x := wi.Request.(type) { - case *protos.WorkItem_OrchestratorRequest: - key := x.OrchestratorRequest.GetInstanceId() - pendingOrchestrators[key] = struct{}{} - p, ok := g.pendingOrchestrators.Load(api.InstanceID(key)) - if ok { - p.(*ExecutionResults).pending = pendingOrchestratorCh - } - case *protos.WorkItem_ActivityRequest: - key := getActivityExecutionKey(x.ActivityRequest.GetOrchestrationInstance().GetInstanceId(), x.ActivityRequest.GetTaskId()) - pendingActivities[key] = struct{}{} - p, ok := g.pendingActivities.Load(key) - if ok { - p.(*activityExecutionResult).pending = pendingActivityCh - } - } - - if err := stream.Send(wi); err != nil { - g.logger.Errorf("encountered an error while sending work item: %v", err) - return err - } - case key := <-pendingActivityCh: - delete(pendingActivities, key) - case key := <-pendingOrchestratorCh: - delete(pendingOrchestrators, key) - case <-g.streamShutdownChan: - return errShuttingDown - } - } -} - -// CompleteOrchestratorTask implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) CompleteOrchestratorTask(ctx context.Context, res *protos.OrchestratorResponse) (*protos.CompleteTaskResponse, error) { - iid := api.InstanceID(res.InstanceId) - if g.deletePendingOrchestrator(iid, res) { - return emptyCompleteTaskResponse, nil - } - - return emptyCompleteTaskResponse, fmt.Errorf("unknown instance ID: %s", res.InstanceId) -} - -func (g *grpcExecutor) deletePendingOrchestrator(iid api.InstanceID, res *protos.OrchestratorResponse) bool { - p, ok := g.pendingOrchestrators.LoadAndDelete(iid) - if !ok { - return false - } - - // Note that res can be nil in case of certain failures - pending := p.(*ExecutionResults) - pending.Response = res - if pending.pending != nil { - pending.pending <- string(iid) - } - close(pending.complete) - return true -} - -// CompleteActivityTask implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) CompleteActivityTask(ctx context.Context, res *protos.ActivityResponse) (*protos.CompleteTaskResponse, error) { - key := getActivityExecutionKey(res.InstanceId, res.TaskId) - if g.deletePendingActivityTask(key, res) { - return emptyCompleteTaskResponse, nil - } - - return emptyCompleteTaskResponse, fmt.Errorf("unknown instance ID/task ID combo: %s", key) -} - -func (g *grpcExecutor) deletePendingActivityTask(key string, res *protos.ActivityResponse) bool { - p, ok := g.pendingActivities.LoadAndDelete(key) - if !ok { - return false - } - - // Note that res can be nil in case of certain failures - pending := p.(*activityExecutionResult) - pending.response = res - if pending.pending != nil { - pending.pending <- key - } - close(pending.complete) - return true -} - -func getActivityExecutionKey(iid string, taskID int32) string { - return iid + "/" + strconv.FormatInt(int64(taskID), 10) -} - -// CreateTaskHub implements protos.TaskHubSidecarServiceServer -func (grpcExecutor) CreateTaskHub(context.Context, *protos.CreateTaskHubRequest) (*protos.CreateTaskHubResponse, error) { - return nil, status.Error(codes.Unimplemented, "CreateTaskHub is not implemented") -} - -// DeleteTaskHub implements protos.TaskHubSidecarServiceServer -func (grpcExecutor) DeleteTaskHub(context.Context, *protos.DeleteTaskHubRequest) (*protos.DeleteTaskHubResponse, error) { - return nil, status.Error(codes.Unimplemented, "DeleteTaskHub is not implemented") -} - -// GetInstance implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) GetInstance(ctx context.Context, req *protos.GetInstanceRequest) (*protos.GetInstanceResponse, error) { - metadata, err := g.backend.GetOrchestrationMetadata(ctx, api.InstanceID(req.InstanceId)) - if err != nil { - if errors.Is(err, api.ErrInstanceNotFound) { - return &protos.GetInstanceResponse{Exists: false}, nil - } - return nil, err - } - - if metadata == nil { - return &protos.GetInstanceResponse{Exists: false}, nil - } - - return createGetInstanceResponse(req, metadata), nil -} - -// PurgeInstances implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) PurgeInstances(ctx context.Context, req *protos.PurgeInstancesRequest) (*protos.PurgeInstancesResponse, error) { - if req.GetPurgeInstanceFilter() != nil { - return nil, status.Error(codes.Unimplemented, "multi-instance purge is not yet implemented") - } - count, err := purgeOrchestrationState(ctx, g.backend, api.InstanceID(req.GetInstanceId()), req.Recursive) - resp := &protos.PurgeInstancesResponse{DeletedInstanceCount: int32(count)} - if err != nil { - return resp, fmt.Errorf("failed to purge orchestration state: %w", err) - } - return resp, nil -} - -// QueryInstances implements protos.TaskHubSidecarServiceServer -func (grpcExecutor) QueryInstances(context.Context, *protos.QueryInstancesRequest) (*protos.QueryInstancesResponse, error) { - return nil, status.Error(codes.Unimplemented, "QueryInstances is not implemented") -} - -// RaiseEvent implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) RaiseEvent(ctx context.Context, req *protos.RaiseEventRequest) (*protos.RaiseEventResponse, error) { - e := helpers.NewEventRaisedEvent(req.Name, req.Input) - if err := g.backend.AddNewOrchestrationEvent(ctx, api.InstanceID(req.InstanceId), e); err != nil { - return nil, err - } - - return &protos.RaiseEventResponse{}, nil -} - -// StartInstance implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) StartInstance(ctx context.Context, req *protos.CreateInstanceRequest) (*protos.CreateInstanceResponse, error) { - instanceID := req.InstanceId - ctx, span := helpers.StartNewCreateOrchestrationSpan(ctx, req.Name, req.Version.GetValue(), instanceID) - defer span.End() - - e := helpers.NewExecutionStartedEvent(req.Name, instanceID, req.Input, nil, helpers.TraceContextFromSpan(span), req.ScheduledStartTimestamp) - if err := g.backend.CreateOrchestrationInstance(ctx, e, WithOrchestrationIdReusePolicy(req.OrchestrationIdReusePolicy)); err != nil { - return nil, err - } - - return &protos.CreateInstanceResponse{InstanceId: instanceID}, nil -} - -// TerminateInstance implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) TerminateInstance(ctx context.Context, req *protos.TerminateRequest) (*protos.TerminateResponse, error) { - e := helpers.NewExecutionTerminatedEvent(req.Output, req.Recursive) - if err := g.backend.AddNewOrchestrationEvent(ctx, api.InstanceID(req.InstanceId), e); err != nil { - return nil, fmt.Errorf("failed to submit termination request: %w", err) - } - return &protos.TerminateResponse{}, nil -} - -// SuspendInstance implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) SuspendInstance(ctx context.Context, req *protos.SuspendRequest) (*protos.SuspendResponse, error) { - e := helpers.NewSuspendOrchestrationEvent(req.Reason.GetValue()) - if err := g.backend.AddNewOrchestrationEvent(ctx, api.InstanceID(req.InstanceId), e); err != nil { - return nil, err - } - - return &protos.SuspendResponse{}, nil -} - -// ResumeInstance implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) ResumeInstance(ctx context.Context, req *protos.ResumeRequest) (*protos.ResumeResponse, error) { - e := helpers.NewResumeOrchestrationEvent(req.Reason.GetValue()) - if err := g.backend.AddNewOrchestrationEvent(ctx, api.InstanceID(req.InstanceId), e); err != nil { - return nil, err - } - - return &protos.ResumeResponse{}, nil -} - -// WaitForInstanceCompletion implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) WaitForInstanceCompletion(ctx context.Context, req *protos.GetInstanceRequest) (*protos.GetInstanceResponse, error) { - return g.waitForInstance(ctx, req, func(m *api.OrchestrationMetadata) bool { - return m.IsComplete() - }) -} - -// WaitForInstanceStart implements protos.TaskHubSidecarServiceServer -func (g *grpcExecutor) WaitForInstanceStart(ctx context.Context, req *protos.GetInstanceRequest) (*protos.GetInstanceResponse, error) { - return g.waitForInstance(ctx, req, func(m *api.OrchestrationMetadata) bool { - return m.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_PENDING - }) -} - -func (g *grpcExecutor) waitForInstance(ctx context.Context, req *protos.GetInstanceRequest, condition func(*api.OrchestrationMetadata) bool) (*protos.GetInstanceResponse, error) { - iid := api.InstanceID(req.InstanceId) - - var b backoff.BackOff = &backoff.ExponentialBackOff{ - InitialInterval: 1 * time.Millisecond, - MaxInterval: 3 * time.Second, - Multiplier: 1.5, - RandomizationFactor: 0.5, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - } - b = backoff.WithContext(b, ctx) - b.Reset() - -loop: - for { - t := time.NewTimer(b.NextBackOff()) - select { - case <-ctx.Done(): - if !t.Stop() { - <-t.C - } - break loop - - case <-t.C: - metadata, err := g.backend.GetOrchestrationMetadata(ctx, iid) - if err != nil { - return nil, err - } - if metadata == nil { - return &protos.GetInstanceResponse{Exists: false}, nil - } - if condition(metadata) { - return createGetInstanceResponse(req, metadata), nil - } - } - } - - return nil, status.Errorf(codes.Canceled, "instance hasn't completed") -} - -// mustEmbedUnimplementedTaskHubSidecarServiceServer implements protos.TaskHubSidecarServiceServer -func (grpcExecutor) mustEmbedUnimplementedTaskHubSidecarServiceServer() { //nolint:unused -} - -func createGetInstanceResponse(req *protos.GetInstanceRequest, metadata *api.OrchestrationMetadata) *protos.GetInstanceResponse { - state := &protos.OrchestrationState{ - InstanceId: req.InstanceId, - Name: metadata.Name, - OrchestrationStatus: metadata.RuntimeStatus, - CreatedTimestamp: timestamppb.New(metadata.CreatedAt), - LastUpdatedTimestamp: timestamppb.New(metadata.LastUpdatedAt), - } - - if req.GetInputsAndOutputs { - state.Input = wrapperspb.String(metadata.SerializedInput) - state.CustomStatus = wrapperspb.String(metadata.SerializedCustomStatus) - state.Output = wrapperspb.String(metadata.SerializedOutput) - state.FailureDetails = metadata.FailureDetails - } - - return &protos.GetInstanceResponse{Exists: true, OrchestrationState: state} -} diff --git a/backend/orchestration.go b/backend/orchestration.go deleted file mode 100644 index 567108c5..00000000 --- a/backend/orchestration.go +++ /dev/null @@ -1,326 +0,0 @@ -package backend - -import ( - "context" - "errors" - "fmt" - "time" - - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" - "go.opentelemetry.io/otel/trace" - "google.golang.org/protobuf/types/known/wrapperspb" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" -) - -type OrchestratorExecutor interface { - ExecuteOrchestrator( - ctx context.Context, - iid api.InstanceID, - oldEvents []*protos.HistoryEvent, - newEvents []*protos.HistoryEvent) (*ExecutionResults, error) -} - -type orchestratorProcessor struct { - be Backend - executor OrchestratorExecutor - logger Logger -} - -func NewOrchestrationWorker(be Backend, executor OrchestratorExecutor, logger Logger, opts ...NewTaskWorkerOptions) TaskWorker { - processor := &orchestratorProcessor{ - be: be, - executor: executor, - logger: logger, - } - return NewTaskWorker(processor, logger, opts...) -} - -// Name implements TaskProcessor -func (*orchestratorProcessor) Name() string { - return "orchestration-processor" -} - -// FetchWorkItem implements TaskProcessor -func (p *orchestratorProcessor) FetchWorkItem(ctx context.Context) (WorkItem, error) { - return p.be.GetOrchestrationWorkItem(ctx) -} - -// ProcessWorkItem implements TaskProcessor -func (w *orchestratorProcessor) ProcessWorkItem(ctx context.Context, cwi WorkItem) error { - wi := cwi.(*OrchestrationWorkItem) - w.logger.Debugf("%v: received work item with %d new event(s): %v", wi.InstanceID, len(wi.NewEvents), helpers.HistoryListSummary(wi.NewEvents)) - - // TODO: Caching - // In the fullness of time, we should consider caching executors and runtime state - // so that we can skip the loading of state and/or the creation of executors. A cached - // executor should allow us to 1) skip runtime state loading and 2) execute only new events. - if wi.State == nil { - if state, err := w.be.GetOrchestrationRuntimeState(ctx, wi); err != nil { - return fmt.Errorf("failed to load orchestration state: %w", err) - } else { - wi.State = state - } - } - w.logger.Debugf("%v: got orchestration runtime state: %s", wi.InstanceID, getOrchestrationStateDescription(wi)) - - var terminateEvent *protos.ExecutionTerminatedEvent = nil - for _, e := range wi.NewEvents { - if et := e.GetExecutionTerminated(); et != nil { - terminateEvent = et - break - } - } - if ctx, span, ok := w.applyWorkItem(ctx, wi); ok { - defer func() { - // Note that the span and ctx references may be updated inside the continue-as-new loop. - w.endOrchestratorSpan(ctx, wi, span, false) - }() - - for continueAsNewCount := 0; ; continueAsNewCount++ { - if continueAsNewCount > 0 { - w.logger.Debugf("%v: continuing-as-new with %d event(s): %s", wi.InstanceID, len(wi.State.NewEvents()), helpers.HistoryListSummary(wi.State.NewEvents())) - } else { - w.logger.Debugf("%v: invoking orchestrator", wi.InstanceID) - } - - // Run the user orchestrator code, providing the old history and new events together. - results, err := w.executor.ExecuteOrchestrator(ctx, wi.InstanceID, wi.State.OldEvents(), wi.State.NewEvents()) - if err != nil { - return fmt.Errorf("error executing orchestrator: %w", err) - } - w.logger.Debugf("%v: orchestrator returned %d action(s): %s", wi.InstanceID, len(results.Response.Actions), helpers.ActionListSummary(results.Response.Actions)) - - // Apply the orchestrator outputs to the orchestration state. - continuedAsNew, err := wi.State.ApplyActions(results.Response.Actions, helpers.TraceContextFromSpan(span)) - if err != nil { - return fmt.Errorf("failed to apply the execution result actions: %w", err) - } - wi.State.CustomStatus = results.Response.CustomStatus - - // When continuing-as-new, we re-execute the orchestrator from the beginning with a truncated state in a tight loop - // until the orchestrator performs some non-continue-as-new action. - if continuedAsNew { - const MaxContinueAsNewCount = 20 - if continueAsNewCount >= MaxContinueAsNewCount { - return fmt.Errorf("exceeded tight-loop continue-as-new limit of %d iterations", MaxContinueAsNewCount) - } - - // We create a new trace span for every continue-as-new - w.endOrchestratorSpan(ctx, wi, span, true) - ctx, span = w.startOrResumeOrchestratorSpan(ctx, wi) - continue - } - - if wi.State.IsCompleted() { - name, _ := wi.State.Name() - w.logger.Infof("%v: '%s' completed with a %s status.", wi.InstanceID, name, helpers.ToRuntimeStatusString(wi.State.RuntimeStatus())) - } - break - } - } - if terminateEvent != nil && wi.State.IsCompleted() { - if err := terminateSubOrchestrationInstances(ctx, w.be, wi.InstanceID, wi.State, terminateEvent); err != nil { - return err - } - } - return nil -} - -// CompleteWorkItem implements TaskProcessor -func (p *orchestratorProcessor) CompleteWorkItem(ctx context.Context, wi WorkItem) error { - owi := wi.(*OrchestrationWorkItem) - return p.be.CompleteOrchestrationWorkItem(ctx, owi) -} - -// AbandonWorkItem implements TaskProcessor -func (p *orchestratorProcessor) AbandonWorkItem(ctx context.Context, wi WorkItem) error { - owi := wi.(*OrchestrationWorkItem) - return p.be.AbandonOrchestrationWorkItem(ctx, owi) -} - -func (w *orchestratorProcessor) applyWorkItem(ctx context.Context, wi *OrchestrationWorkItem) (context.Context, trace.Span, bool) { - // Ignore work items for orchestrations that are completed or are in a corrupted state. - switch { - case !wi.State.IsValid(): - w.logger.Warnf("%v: orchestration state is invalid; dropping work item", wi.InstanceID) - return nil, nil, false - case wi.State.IsCompleted(): - w.logger.Warnf("%v: orchestration already completed; dropping work item", wi.InstanceID) - return nil, nil, false - case len(wi.NewEvents) == 0: - w.logger.Warnf("%v: the work item had no events!", wi.InstanceID) - } - - // The orchestrator started event is used primarily for updating the current time as reported - // by the orchestration context APIs. - if err := wi.State.AddEvent(helpers.NewOrchestratorStartedEvent()); err != nil { - w.logger.Warnf("%v: failed to add orchestrator started event: %v", wi.InstanceID, err) - } - - // Each orchestration instancegets its own distributed tracing span. However, the implementation of - // endOrchestratorSpan will "cancel" the span mark the span as "unsampled" if the orchestration isn't - // complete. This is part of the strategy for producing one span for the entire orchestration execution, - // which isn't something that's natively supported by OTel today. - ctx, span := w.startOrResumeOrchestratorSpan(ctx, wi) - - // New events from the work item are appended to the orchestration state, with duplicates automatically - // filtered out. If all events are filtered out, return false so that the caller knows not to execute - // the orchestration logic for an empty set of events. - added := 0 - for _, e := range wi.NewEvents { - if err := wi.State.AddEvent(e); err != nil { - if errors.Is(err, ErrDuplicateEvent) { - w.logger.Warnf("%v: dropping duplicate event: %v", wi.InstanceID, e) - } else { - w.logger.Warnf("%v: dropping event: %v, %v", wi.InstanceID, e, err) - } - } else { - added++ - } - - // Special case logic for specific event types - if es := e.GetExecutionStarted(); es != nil { - w.logger.Infof("%v: starting new '%s' instance with ID = '%s'.", wi.InstanceID, es.Name, es.OrchestrationInstance.InstanceId) - } else if timerFired := e.GetTimerFired(); timerFired != nil { - // Timer spans are created and completed once the TimerFired event is received. - // TODO: Ideally we don't emit spans for cancelled timers. Is there a way to support this? - if err := helpers.StartAndEndNewTimerSpan(ctx, timerFired, e.Timestamp.AsTime(), string(wi.InstanceID)); err != nil { - w.logger.Warnf("%v: failed to generate distributed trace span for durable timer: %v", wi.InstanceID, err) - } - } - } - - if added == 0 { - w.logger.Warnf("%v: all new events were dropped", wi.InstanceID) - return ctx, span, false - } - - return ctx, span, true -} - -func getOrchestrationStateDescription(wi *OrchestrationWorkItem) string { - name, err := wi.State.Name() - if err != nil { - if len(wi.NewEvents) > 0 { - name = wi.NewEvents[0].GetExecutionStarted().GetName() - } - } - if name == "" { - name = "(unknown)" - } - - ageStr := "(new)" - createdAt, err := wi.State.CreatedTime() - if err == nil { - age := time.Since(createdAt) - - if age > 0 { - ageStr = age.Round(time.Second).String() - } - } - status := helpers.ToRuntimeStatusString(wi.State.RuntimeStatus()) - return fmt.Sprintf("name=%s, status=%s, events=%d, age=%s", name, status, len(wi.State.OldEvents()), ageStr) -} - -func (w *orchestratorProcessor) startOrResumeOrchestratorSpan(ctx context.Context, wi *OrchestrationWorkItem) (context.Context, trace.Span) { - // Get the trace context from the ExecutionStarted history event - var ptc *protos.TraceContext - var es *protos.ExecutionStartedEvent - if es = wi.State.startEvent; es != nil { - ptc = wi.State.startEvent.ParentTraceContext - } else { - for _, e := range wi.NewEvents { - if es = e.GetExecutionStarted(); es != nil { - ptc = es.ParentTraceContext - break - } - } - } - - if ptc == nil { - return ctx, helpers.NoopSpan() - } - - ctx, err := helpers.ContextFromTraceContext(ctx, ptc) - if err != nil { - w.logger.Warnf("%v: failed to parse trace context: %v", wi.InstanceID, err) - return ctx, helpers.NoopSpan() - } - - // start a new span from the updated go context - var span trace.Span - ctx, span = helpers.StartNewRunOrchestrationSpan(ctx, es, wi.State.getStartedTime()) - - // Assign or rehydrate the long-running orchestration span ID - if es.OrchestrationSpanID == nil { - // On the initial execution, assign the orchestration span ID to be the - // randomly generated span ID value. This will be persisted in the orchestration history - // and referenced on the next replay. - es.OrchestrationSpanID = wrapperspb.String(span.SpanContext().SpanID().String()) - } else { - // On subsequent executions, replace the auto-generated span ID with the orchestration - // span ID. This allows us to have one long-running span that survives multiple replays - // and process failures. - if orchestratorSpanID, err := trace.SpanIDFromHex(es.OrchestrationSpanID.Value); err == nil { - helpers.ChangeSpanID(span, orchestratorSpanID) - } - } - - return ctx, span -} - -func (w *orchestratorProcessor) endOrchestratorSpan(ctx context.Context, wi *OrchestrationWorkItem, span trace.Span, continuedAsNew bool) { - switch { - case wi.State.IsCompleted(): - if fd, err := wi.State.FailureDetails(); err == nil { - span.SetStatus(codes.Error, fd.ErrorMessage) - } - span.SetAttributes(attribute.KeyValue{ - Key: "durabletask.runtime_status", - Value: attribute.StringValue(helpers.ToRuntimeStatusString(wi.State.RuntimeStatus())), - }) - addNotableEventsToSpan(wi.State.OldEvents(), span) - addNotableEventsToSpan(wi.State.NewEvents(), span) - case continuedAsNew: - span.SetAttributes(attribute.KeyValue{ - Key: "durabletask.runtime_status", - Value: attribute.StringValue(helpers.ToRuntimeStatusString(protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW)), - }) - default: - // Cancel the span - we want to publish it only when an orchestration - // completes or when it continue-as-new's. - helpers.CancelSpan(span) - } - - // We must always call End() on a span to ensure we don't leak resources. - // See https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/api.md#span-creation - span.End() -} - -// Adds notable events to the span that are interesting to the user. -// More info: https://opentelemetry.io/docs/instrumentation/go/manual/#events -func addNotableEventsToSpan(events []*protos.HistoryEvent, span trace.Span) { - for _, e := range events { - if eventRaised := e.GetEventRaised(); eventRaised != nil { - eventByteCount := len(eventRaised.Input.GetValue()) - span.AddEvent( - "Received external event", - trace.WithTimestamp(e.Timestamp.AsTime()), - trace.WithAttributes(attribute.String("name", eventRaised.Name), attribute.Int("size", eventByteCount))) - } else if suspended := e.GetExecutionSuspended(); suspended != nil { - span.AddEvent( - "Execution suspended", - trace.WithTimestamp(e.Timestamp.AsTime()), - trace.WithAttributes(attribute.String("reason", suspended.Input.GetValue()))) - } else if resumed := e.GetExecutionResumed(); resumed != nil { - span.AddEvent( - "Execution resumed", - trace.WithTimestamp(e.Timestamp.AsTime()), - trace.WithAttributes(attribute.String("reason", resumed.Input.GetValue()))) - } - } -} diff --git a/backend/postgres/README.md b/backend/postgres/README.md deleted file mode 100644 index 70282847..00000000 --- a/backend/postgres/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Postgres Backend -### Testing -By default, the postgres tests are skipped. To run the tests, set the environment variable `POSTGRES_ENABLED` to `true` before running the tests and have a postgres server running on `localhost:5432` with a database named `postgres` and a user `postgres` with password `postgres`. diff --git a/backend/postgres/postgres.go b/backend/postgres/postgres.go deleted file mode 100644 index e0066ddd..00000000 --- a/backend/postgres/postgres.go +++ /dev/null @@ -1,1053 +0,0 @@ -package postgres - -import ( - "context" - _ "embed" - "errors" - "fmt" - "os" - "strings" - "time" - - "github.com/google/uuid" - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "google.golang.org/protobuf/proto" - - "github.com/jackc/pgx/v5" - "github.com/jackc/pgx/v5/pgxpool" -) - -//go:embed schema.sql -var schema string - -var emptyString string = "" - -type PostgresOptions struct { - PgOptions *pgxpool.Config - OrchestrationLockTimeout time.Duration - ActivityLockTimeout time.Duration -} - -type postgresBackend struct { - db *pgxpool.Pool - workerName string - logger backend.Logger - options *PostgresOptions -} - -// NewPostgresOptions creates a new options object for the postgres backend provider. -func NewPostgresOptions(host string, port uint16, database string, user string, password string) *PostgresOptions { - conf, err := pgxpool.ParseConfig(fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", user, password, host, port, database)) - if err != nil { - panic(fmt.Errorf("failed to parse the postgres connection string: %w", err)) - } - conf.ConnConfig.ConnectTimeout = 2 * time.Minute - conf.MaxConnLifetime = 2 * time.Minute - conf.MaxConnIdleTime = 2 * time.Minute - conf.MaxConns = 1 - - return &PostgresOptions{ - PgOptions: conf, - OrchestrationLockTimeout: 2 * time.Minute, - ActivityLockTimeout: 2 * time.Minute, - } -} - -// NewPostgresBackend creates a new postgres-based Backend object. -func NewPostgresBackend(opts *PostgresOptions, logger backend.Logger) backend.Backend { - hostname, err := os.Hostname() - if err != nil { - hostname = "unknown" - } - - pid := os.Getpid() - u, err := uuid.NewV7() - if err != nil { - u = uuid.New() - } - uuidStr := u.String() - - if opts == nil { - opts = NewPostgresOptions("localhost", 5432, "postgres", "postgres", "postgres") - } - - return &postgresBackend{ - db: nil, - workerName: fmt.Sprintf("%s,%d,%s", hostname, pid, uuidStr), - options: opts, - logger: logger, - } -} - -// CreateTaskHub creates the postgres database and applies the schema -func (be *postgresBackend) CreateTaskHub(ctx context.Context) error { - if err := be.Start(ctx); err != nil { - be.logger.Error("CreateTaskHub", "failed to start the backend", err) - return fmt.Errorf("failed to start the backend: %w", err) - } - - // Initialize database - if _, err := be.db.Exec(ctx, schema); err != nil { - be.logger.Error("CreateTaskHub", "failed to initialize the database", err) - return fmt.Errorf("failed to initialize the database: %w", err) - } - - return nil -} - -func (be *postgresBackend) DeleteTaskHub(ctx context.Context) error { - if be.db == nil { - return nil - } - - _, err := be.db.Exec(ctx, "DROP TABLE IF EXISTS Instances CASCADE") - if err != nil { - be.logger.Error("DeleteTaskHub", "failed to drop Instances table", err) - return fmt.Errorf("failed to drop Instances table: %w", err) - } - _, err = be.db.Exec(ctx, "DROP TABLE IF EXISTS History CASCADE") - if err != nil { - be.logger.Error("DeleteTaskHub", "failed to drop History table", err) - return fmt.Errorf("failed to drop History table: %w", err) - } - _, err = be.db.Exec(ctx, "DROP TABLE IF EXISTS NewEvents CASCADE") - if err != nil { - be.logger.Error("DeleteTaskHub", "failed to drop NewEvents table", err) - return fmt.Errorf("failed to drop NewEvents table: %w", err) - } - _, err = be.db.Exec(ctx, "DROP TABLE IF EXISTS NewTasks CASCADE") - if err != nil { - be.logger.Error("DeleteTaskHub", "failed to drop NewTasks table", err) - return fmt.Errorf("failed to drop NewTasks table: %w", err) - } - - if err := be.Stop(ctx); err != nil { - be.logger.Error("DeleteTaskHub", "failed to stop the backend", err) - return fmt.Errorf("failed to stop the backend: %w", err) - } - - return nil -} - -// AbandonOrchestrationWorkItem implements backend.Backend -func (be *postgresBackend) AbandonOrchestrationWorkItem(ctx context.Context, wi *backend.OrchestrationWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return err - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - var visibleTime *time.Time = nil - if delay := wi.GetAbandonDelay(); delay > 0 { - t := time.Now().UTC().Add(delay) - visibleTime = &t - } - - dbResult, err := tx.Exec( - ctx, - "UPDATE NewEvents SET LockedBy = NULL, VisibleTime = $1 WHERE InstanceID = $2 AND LockedBy = $3", - visibleTime, - string(wi.InstanceID), - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to update NewEvents table: %w", err) - } - - rowsAffected := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by UPDATE NewEvents statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - dbResult, err = tx.Exec( - ctx, - "UPDATE Instances SET LockedBy = NULL, LockExpiration = NULL WHERE InstanceID = $1 AND LockedBy = $2", - string(wi.InstanceID), - wi.LockedBy, - ) - - if err != nil { - return fmt.Errorf("failed to update Instances table: %w", err) - } - - rowsAffected = dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by UPDATE Instances statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err = tx.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -// CompleteOrchestrationWorkItem implements backend.Backend -func (be *postgresBackend) CompleteOrchestrationWorkItem(ctx context.Context, wi *backend.OrchestrationWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return err - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - now := time.Now().UTC() - - // Dynamically generate the UPDATE statement for the Instances table - var sqlSB strings.Builder - sqlSB.WriteString("UPDATE Instances SET ") - - sqlUpdateArgs := make([]any, 0, 10) - isCreated := false - isCompleted := false - - currIndex := 1 - for _, e := range wi.State.NewEvents() { - if es := e.GetExecutionStarted(); es != nil { - if isCreated { - // TODO: Log warning about duplicate start event - continue - } - isCreated = true - fmt.Fprintf(&sqlSB, "CreatedTime = $%d, Input = $%d, ", currIndex, currIndex+1) - currIndex += 2 - sqlUpdateArgs = append(sqlUpdateArgs, e.Timestamp.AsTime()) - sqlUpdateArgs = append(sqlUpdateArgs, es.Input.GetValue()) - } else if ec := e.GetExecutionCompleted(); ec != nil { - if isCompleted { - // TODO: Log warning about duplicate completion event - continue - } - isCompleted = true - fmt.Fprintf(&sqlSB, "CompletedTime = $%d, Output = $%d, FailureDetails = $%d, ", currIndex, currIndex+1, currIndex+2) - currIndex += 3 - sqlUpdateArgs = append(sqlUpdateArgs, now) - sqlUpdateArgs = append(sqlUpdateArgs, ec.Result.GetValue()) - if ec.FailureDetails != nil { - bytes, err := proto.Marshal(ec.FailureDetails) - if err != nil { - return fmt.Errorf("failed to marshal FailureDetails: %w", err) - } - sqlUpdateArgs = append(sqlUpdateArgs, &bytes) - } else { - sqlUpdateArgs = append(sqlUpdateArgs, nil) - } - } - // TODO: Execution suspended & resumed - } - - if wi.State.CustomStatus != nil { - fmt.Fprintf(&sqlSB, "CustomStatus = $%d, ", currIndex) - currIndex++ - sqlUpdateArgs = append(sqlUpdateArgs, wi.State.CustomStatus.Value) - } - - // TODO: Support for stickiness, which would extend the LockExpiration - fmt.Fprintf(&sqlSB, "RuntimeStatus = $%d, LastUpdatedTime = $%d, LockExpiration = NULL WHERE InstanceID = $%d AND LockedBy = $%d", currIndex, currIndex+1, currIndex+2, currIndex+3) - sqlUpdateArgs = append(sqlUpdateArgs, helpers.ToRuntimeStatusString(wi.State.RuntimeStatus()), now, string(wi.InstanceID), wi.LockedBy) - - result, err := tx.Exec(ctx, sqlSB.String(), sqlUpdateArgs...) - if err != nil { - return fmt.Errorf("failed to update Instances table: %w", err) - } - - count := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get the number of rows affected by the Instance table update: %w", err) - } else if count == 0 { - return fmt.Errorf("instance '%s' no longer exists or was locked by a different worker", string(wi.InstanceID)) - } - - // If continue-as-new, delete all existing history - if wi.State.ContinuedAsNew() { - if _, err := tx.Exec(ctx, "DELETE FROM History WHERE InstanceID = $1", string(wi.InstanceID)); err != nil { - return fmt.Errorf("failed to delete from History table: %w", err) - } - } - - // Save new history events - newHistoryCount := len(wi.State.NewEvents()) - if newHistoryCount > 0 { - builder := strings.Builder{} - builder.WriteString("INSERT INTO History (InstanceID, SequenceNumber, EventPayload) VALUES ") - for i := 0; i < newHistoryCount; i++ { - fmt.Fprintf(&builder, "($%d, $%d, $%d)", 3*i+1, 3*i+2, 3*i+3) - if i < newHistoryCount-1 { - builder.WriteString(", ") - } - } - query := builder.String() - - args := make([]any, 0, newHistoryCount*3) - nextSequenceNumber := len(wi.State.OldEvents()) - for _, e := range wi.State.NewEvents() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - args = append(args, string(wi.InstanceID), nextSequenceNumber, eventPayload) - nextSequenceNumber++ - } - - _, err = tx.Exec(ctx, query, args...) - if err != nil { - return fmt.Errorf("failed to insert into the History table: %w", err) - } - } - - // Save outbound activity tasks - newActivityCount := len(wi.State.PendingTasks()) - if newActivityCount > 0 { - builder := strings.Builder{} - builder.WriteString("INSERT INTO NewTasks (InstanceID, EventPayload) VALUES ") - for i := 0; i < newActivityCount; i++ { - fmt.Fprintf(&builder, "($%d, $%d)", 2*i+1, 2*i+2) - if i < newActivityCount-1 { - builder.WriteString(", ") - } - } - insertSql := builder.String() - - sqlInsertArgs := make([]any, 0, newActivityCount*2) - for _, e := range wi.State.PendingTasks() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - sqlInsertArgs = append(sqlInsertArgs, string(wi.InstanceID), eventPayload) - } - - _, err = tx.Exec(ctx, insertSql, sqlInsertArgs...) - if err != nil { - return fmt.Errorf("failed to insert into the NewTasks table: %w", err) - } - } - - // Save outbound orchestrator events - newEventCount := len(wi.State.PendingTimers()) + len(wi.State.PendingMessages()) - if newEventCount > 0 { - builder := strings.Builder{} - builder.WriteString("INSERT INTO NewEvents (InstanceID, EventPayload, VisibleTime) VALUES ") - for i := 0; i < newEventCount; i++ { - fmt.Fprintf(&builder, "($%d, $%d, $%d)", 3*i+1, 3*i+2, 3*i+3) - if i < newEventCount-1 { - builder.WriteString(", ") - } - } - insertSql := builder.String() - - sqlInsertArgs := make([]any, 0, newEventCount*3) - for _, e := range wi.State.PendingTimers() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - visibileTime := e.GetTimerFired().GetFireAt().AsTime() - sqlInsertArgs = append(sqlInsertArgs, string(wi.InstanceID), eventPayload, visibileTime) - } - - for _, msg := range wi.State.PendingMessages() { - if es := msg.HistoryEvent.GetExecutionStarted(); es != nil { - // Need to insert a new row into the DB - if _, err := be.createOrchestrationInstanceInternal(ctx, msg.HistoryEvent, tx, backend.WithOrchestrationIdReusePolicy(&protos.OrchestrationIdReusePolicy{ - OperationStatus: []protos.OrchestrationStatus{protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED}, - Action: api.REUSE_ID_ACTION_TERMINATE, - })); err != nil { - if errors.Is(err, backend.ErrDuplicateEvent) { - be.logger.Warnf( - "%v: dropping sub-orchestration creation event because an instance with the target ID (%v) already exists.", - wi.InstanceID, - es.OrchestrationInstance.InstanceId) - } else { - return err - } - } - } - - eventPayload, err := backend.MarshalHistoryEvent(msg.HistoryEvent) - if err != nil { - return err - } - - sqlInsertArgs = append(sqlInsertArgs, msg.TargetInstanceID, eventPayload, nil) - } - - _, err = tx.Exec(ctx, insertSql, sqlInsertArgs...) - if err != nil { - return fmt.Errorf("failed to insert into the NewEvents table: %w", err) - } - } - - // Delete inbound events - dbResult, err := tx.Exec( - ctx, - "DELETE FROM NewEvents WHERE InstanceID = $1 AND LockedBy = $2", - string(wi.InstanceID), - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to delete from NewEvents table: %w", err) - } - - rowsAffected := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by delete statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err != nil { - return fmt.Errorf("failed to delete from the NewEvents table: %w", err) - } - - if err = tx.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -// CreateOrchestrationInstance implements backend.Backend -func (be *postgresBackend) CreateOrchestrationInstance(ctx context.Context, e *backend.HistoryEvent, opts ...backend.OrchestrationIdReusePolicyOptions) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return fmt.Errorf("failed to start transaction: %w", err) - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - var instanceID string - if instanceID, err = be.createOrchestrationInstanceInternal(ctx, e, tx, opts...); errors.Is(err, api.ErrIgnoreInstance) { - // choose to ignore, do nothing - return nil - } else if err != nil { - return err - } - - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - _, err = tx.Exec( - ctx, - `INSERT INTO NewEvents (InstanceID, EventPayload) VALUES ($1, $2)`, - instanceID, - eventPayload, - ) - - if err != nil { - return fmt.Errorf("failed to insert row into NewEvents table: %w", err) - } - - if err = tx.Commit(ctx); err != nil { - return fmt.Errorf("failed to create orchestration: %w", err) - } - - return nil -} - -func (be *postgresBackend) createOrchestrationInstanceInternal(ctx context.Context, e *backend.HistoryEvent, tx pgx.Tx, opts ...backend.OrchestrationIdReusePolicyOptions) (string, error) { - if e == nil { - return "", backend.ErrNilHistoryEvent - } else if e.Timestamp == nil { - return "", backend.ErrNilEventTimestamp - } - - startEvent := e.GetExecutionStarted() - if startEvent == nil { - return "", backend.ErrNotExecutionStarted - } - instanceID := startEvent.OrchestrationInstance.InstanceId - - policy := &protos.OrchestrationIdReusePolicy{} - - for _, opt := range opts { - if err := opt(policy); err != nil { - return "", err - } - } - - rows, err := insertOrIgnoreInstanceTableInternal(ctx, tx, e, startEvent) - if err != nil { - return "", err - } - - // instance with same ID already exists - if rows <= 0 { - return instanceID, be.handleInstanceExists(ctx, tx, startEvent, policy, e) - } - return instanceID, nil -} - -func insertOrIgnoreInstanceTableInternal(ctx context.Context, tx pgx.Tx, e *backend.HistoryEvent, startEvent *protos.ExecutionStartedEvent) (int64, error) { - var parentInstanceID *string - if pi := startEvent.GetParentInstance(); pi != nil { - if instanceID := pi.GetOrchestrationInstance().GetInstanceId(); instanceID != "" { - parentInstanceID = &instanceID - } - } - res, err := tx.Exec( - ctx, - `INSERT INTO Instances ( - Name, - Version, - InstanceID, - ExecutionID, - Input, - RuntimeStatus, - CreatedTime, - ParentInstanceID - ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) ON CONFLICT DO NOTHING`, - startEvent.Name, - startEvent.Version.GetValue(), - startEvent.OrchestrationInstance.InstanceId, - startEvent.OrchestrationInstance.ExecutionId.GetValue(), - startEvent.Input.GetValue(), - "PENDING", - e.Timestamp.AsTime(), - parentInstanceID, - ) - if err != nil { - return -1, fmt.Errorf("failed to insert into Instances table: %w", err) - } - - rows := res.RowsAffected() - if err != nil { - return -1, fmt.Errorf("failed to count the rows affected: %w", err) - } - return rows, nil -} - -func (be *postgresBackend) handleInstanceExists(ctx context.Context, tx pgx.Tx, startEvent *protos.ExecutionStartedEvent, policy *protos.OrchestrationIdReusePolicy, e *backend.HistoryEvent) error { - // query RuntimeStatus for the existing instance - queryRow := tx.QueryRow( - ctx, - `SELECT RuntimeStatus FROM Instances WHERE InstanceID = $1`, - startEvent.OrchestrationInstance.InstanceId, - ) - var runtimeStatus *string - err := queryRow.Scan(&runtimeStatus) - if errors.Is(err, pgx.ErrNoRows) { - return api.ErrInstanceNotFound - } else if err != nil { - return fmt.Errorf("failed to scan the Instances table result: %w", err) - } - - // status not match, return instance duplicate error - if !isStatusMatch(policy.OperationStatus, helpers.FromRuntimeStatusString(*runtimeStatus)) { - return api.ErrDuplicateInstance - } - - // status match - switch policy.Action { - case protos.CreateOrchestrationAction_IGNORE: - // Log an warning message and ignore creating new instance - be.logger.Warnf("An instance with ID '%s' already exists; dropping duplicate create request", startEvent.OrchestrationInstance.InstanceId) - return api.ErrIgnoreInstance - case protos.CreateOrchestrationAction_TERMINATE: - // terminate existing instance - if err := be.cleanupOrchestrationStateInternal(ctx, tx, api.InstanceID(startEvent.OrchestrationInstance.InstanceId), false); err != nil { - return fmt.Errorf("failed to cleanup orchestration status: %w", err) - } - // create a new instance - var rows int64 - if rows, err = insertOrIgnoreInstanceTableInternal(ctx, tx, e, startEvent); err != nil { - return err - } - - // should never happen, because we clean up instance before create new one - if rows <= 0 { - return fmt.Errorf("failed to insert into Instances table because entry already exists") - } - return nil - } - // default behavior - return api.ErrDuplicateInstance -} - -func isStatusMatch(statuses []protos.OrchestrationStatus, runtimeStatus protos.OrchestrationStatus) bool { - for _, status := range statuses { - if status == runtimeStatus { - return true - } - } - return false -} - -func (be *postgresBackend) cleanupOrchestrationStateInternal(ctx context.Context, tx pgx.Tx, id api.InstanceID, requireCompleted bool) error { - row := tx.QueryRow(ctx, "SELECT 1 FROM Instances WHERE InstanceID = $1", string(id)) - var unused int - if err := row.Scan(&unused); errors.Is(err, pgx.ErrNoRows) { - return api.ErrInstanceNotFound - } else if err != nil { - return fmt.Errorf("failed to scan instance existence: %w", err) - } - - if requireCompleted { - // purge orchestration in ['COMPLETED', 'FAILED', 'TERMINATED'] - dbResult, err := tx.Exec(ctx, "DELETE FROM Instances WHERE InstanceID = $1 AND RuntimeStatus IN ('COMPLETED', 'FAILED', 'TERMINATED')", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from the Instances table: %w", err) - } - - rowsAffected := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected in Instances delete operation: %w", err) - } - if rowsAffected == 0 { - return api.ErrNotCompleted - } - } else { - // clean up orchestration in all RuntimeStatus - _, err := tx.Exec(ctx, "DELETE FROM Instances WHERE InstanceID = $1", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from the Instances table: %w", err) - } - } - - _, err := tx.Exec(ctx, "DELETE FROM History WHERE InstanceID = $1", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from History table: %w", err) - } - - _, err = tx.Exec(ctx, "DELETE FROM NewEvents WHERE InstanceID = $1", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from NewEvents table: %w", err) - } - - _, err = tx.Exec(ctx, "DELETE FROM NewTasks WHERE InstanceID = $1", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from NewTasks table: %w", err) - } - return nil -} - -func (be *postgresBackend) AddNewOrchestrationEvent(ctx context.Context, iid api.InstanceID, e *backend.HistoryEvent) error { - if e == nil { - return backend.ErrNilHistoryEvent - } else if e.Timestamp == nil { - return backend.ErrNilEventTimestamp - } - - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - _, err = be.db.Exec( - ctx, - `INSERT INTO NewEvents (InstanceID, EventPayload) VALUES ($1, $2)`, - string(iid), - eventPayload, - ) - - if err != nil { - return fmt.Errorf("failed to insert row into NewEvents table: %w", err) - } - - return nil -} - -// GetOrchestrationMetadata implements backend.Backend -func (be *postgresBackend) GetOrchestrationMetadata(ctx context.Context, iid api.InstanceID) (*api.OrchestrationMetadata, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - row := be.db.QueryRow( - ctx, - `SELECT InstanceID, Name, RuntimeStatus, CreatedTime, LastUpdatedTime, Input, Output, CustomStatus, FailureDetails - FROM Instances WHERE InstanceID = $1`, - string(iid), - ) - - var instanceID *string - var name *string - var runtimeStatus *string - var createdAt *time.Time - var lastUpdatedAt *time.Time - var input *string - var output *string - var customStatus *string - var failureDetails *protos.TaskFailureDetails - - var failureDetailsPayload []byte - err := row.Scan(&instanceID, &name, &runtimeStatus, &createdAt, &lastUpdatedAt, &input, &output, &customStatus, &failureDetailsPayload) - if errors.Is(err, pgx.ErrNoRows) { - return nil, api.ErrInstanceNotFound - } else if err != nil { - return nil, fmt.Errorf("failed to scan the Instances table result: %w", err) - } - - if input == nil { - input = &emptyString - } - - if output == nil { - output = &emptyString - } - - if customStatus == nil { - customStatus = &emptyString - } - - if len(failureDetailsPayload) > 0 { - failureDetails = new(protos.TaskFailureDetails) - if err := proto.Unmarshal(failureDetailsPayload, failureDetails); err != nil { - return nil, fmt.Errorf("failed to unmarshal failure details: %w", err) - } - } - - metadata := api.NewOrchestrationMetadata( - iid, - *name, - helpers.FromRuntimeStatusString(*runtimeStatus), - *createdAt, - *lastUpdatedAt, - *input, - *output, - *customStatus, - failureDetails, - ) - return metadata, nil -} - -// GetOrchestrationRuntimeState implements backend.Backend -func (be *postgresBackend) GetOrchestrationRuntimeState(ctx context.Context, wi *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - rows, err := be.db.Query( - ctx, - "SELECT EventPayload FROM History WHERE InstanceID = $1 ORDER BY SequenceNumber ASC", - string(wi.InstanceID), - ) - if err != nil { - return nil, err - } - defer rows.Close() - - existingEvents := make([]*protos.HistoryEvent, 0, 50) - for rows.Next() { - var eventPayload []byte - if err := rows.Scan(&eventPayload); err != nil { - return nil, fmt.Errorf("failed to read history event: %w", err) - } - - e, err := backend.UnmarshalHistoryEvent(eventPayload) - if err != nil { - return nil, err - } - - existingEvents = append(existingEvents, e) - } - - state := backend.NewOrchestrationRuntimeState(wi.InstanceID, existingEvents) - return state, nil -} - -// GetOrchestrationWorkItem implements backend.Backend -func (be *postgresBackend) GetOrchestrationWorkItem(ctx context.Context) (*backend.OrchestrationWorkItem, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return nil, err - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - now := time.Now().UTC() - newLockExpiration := now.Add(be.options.OrchestrationLockTimeout) - - // Place a lock on an orchestration instance that has new events that are ready to be executed. - row := tx.QueryRow( - ctx, - `UPDATE Instances SET LockedBy = $1, LockExpiration = $2 - WHERE SequenceNumber = ( - SELECT SequenceNumber FROM Instances I - WHERE (I.LockExpiration IS NULL OR I.LockExpiration < $3) AND EXISTS ( - SELECT 1 FROM NewEvents E - WHERE E.InstanceID = I.InstanceID AND (E.VisibleTime IS NULL OR E.VisibleTime < $4) - ) - ORDER BY I.InstanceID, I.SequenceNumber ASC - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) RETURNING InstanceID`, - be.workerName, // LockedBy for Instances table - newLockExpiration, // Updated LockExpiration for Instances table - now, // LockExpiration for Instances table - now, // VisibleTime for NewEvents table - ) - - var instanceID string - if err := row.Scan(&instanceID); err != nil { - if errors.Is(err, pgx.ErrNoRows) { - // No new events to process - return nil, backend.ErrNoWorkItems - } - - return nil, fmt.Errorf("failed to scan the orchestration work-item: %w", err) - } - - // TODO: Get all the unprocessed events associated with the locked instance - events, err := tx.Query( - ctx, - `UPDATE NewEvents SET DequeueCount = DequeueCount + 1, LockedBy = $1 WHERE SequenceNumber IN ( - SELECT SequenceNumber FROM NewEvents - WHERE InstanceID = $2 AND (VisibleTime IS NULL OR VisibleTime <= $3) - LIMIT 1000 - ) - RETURNING EventPayload, DequeueCount`, - be.workerName, - instanceID, - now, - ) - if err != nil { - return nil, fmt.Errorf("failed to query for orchestration work-items: %w", err) - } - defer events.Close() - - type rawEvent struct { - payload []byte - dequeue int32 - } - - rawEvents := []rawEvent{} - for events.Next() { - var eventPayload []byte - var dequeueCount int32 - if err := events.Scan(&eventPayload, &dequeueCount); err != nil { - return nil, fmt.Errorf("failed to read history event: %w", err) - } - rawEvents = append(rawEvents, rawEvent{ - payload: eventPayload, - dequeue: dequeueCount, - }) - } - events.Close() - - if err = tx.Commit(ctx); err != nil { - return nil, fmt.Errorf("failed to update orchestration work-item: %w", err) - } - - maxDequeueCount := int32(0) - newEvents := make([]*protos.HistoryEvent, 0, len(rawEvents)) - for _, e := range rawEvents { - if e.dequeue > maxDequeueCount { - maxDequeueCount = e.dequeue - } - - evt, err := backend.UnmarshalHistoryEvent(e.payload) - if err != nil { - return nil, err - } - - newEvents = append(newEvents, evt) - } - - wi := &backend.OrchestrationWorkItem{ - InstanceID: api.InstanceID(instanceID), - NewEvents: newEvents, - LockedBy: be.workerName, - RetryCount: maxDequeueCount - 1, - } - - return wi, nil -} - -func (be *postgresBackend) GetActivityWorkItem(ctx context.Context) (*backend.ActivityWorkItem, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - now := time.Now().UTC() - newLockExpiration := now.Add(be.options.OrchestrationLockTimeout) - - row := be.db.QueryRow( - ctx, - `UPDATE NewTasks SET LockedBy = $1, LockExpiration = $2, DequeueCount = DequeueCount + 1 - WHERE SequenceNumber = ( - SELECT SequenceNumber FROM NewTasks T - WHERE T.LockExpiration IS NULL OR T.LockExpiration < $3 - ORDER BY T.InstanceID, T.SequenceNumber ASC - LIMIT 1 - FOR UPDATE SKIP LOCKED - ) RETURNING SequenceNumber, InstanceID, EventPayload`, - be.workerName, - newLockExpiration, - now, - ) - - var sequenceNumber int64 - var instanceID string - var eventPayload []byte - - if err := row.Scan(&sequenceNumber, &instanceID, &eventPayload); err != nil { - if errors.Is(err, pgx.ErrNoRows) { - // No new activity tasks to process - return nil, backend.ErrNoWorkItems - } - - return nil, fmt.Errorf("failed to scan the activity work-item: %w", err) - } - - e, err := backend.UnmarshalHistoryEvent(eventPayload) - if err != nil { - return nil, err - } - - wi := &backend.ActivityWorkItem{ - SequenceNumber: sequenceNumber, - InstanceID: api.InstanceID(instanceID), - NewEvent: e, - LockedBy: be.workerName, - } - return wi, nil -} - -func (be *postgresBackend) CompleteActivityWorkItem(ctx context.Context, wi *backend.ActivityWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return err - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - bytes, err := backend.MarshalHistoryEvent(wi.Result) - if err != nil { - return err - } - - _, err = tx.Exec(ctx, "INSERT INTO NewEvents (InstanceID, EventPayload) VALUES ($1, $2)", string(wi.InstanceID), bytes) - if err != nil { - return fmt.Errorf("failed to insert into NewEvents table: %w", err) - } - - dbResult, err := tx.Exec(ctx, "DELETE FROM NewTasks WHERE SequenceNumber = $1 AND LockedBy = $2", wi.SequenceNumber, wi.LockedBy) - if err != nil { - return fmt.Errorf("failed to delete from NewTasks table: %w", err) - } - - rowsAffected := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by delete statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err = tx.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -func (be *postgresBackend) AbandonActivityWorkItem(ctx context.Context, wi *backend.ActivityWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - dbResult, err := be.db.Exec( - ctx, - "UPDATE NewTasks SET LockedBy = NULL, LockExpiration = NULL WHERE SequenceNumber = $1 AND LockedBy = $2", - wi.SequenceNumber, - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to update the NewTasks table for abandon: %w", err) - } - - rowsAffected := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by update statement for abandon: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - return nil -} - -func (be *postgresBackend) PurgeOrchestrationState(ctx context.Context, id api.InstanceID) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, pgx.TxOptions{}) - if err != nil { - return err - } - defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op - - if err := be.cleanupOrchestrationStateInternal(ctx, tx, id, true); err != nil { - return err - } - - if err = tx.Commit(ctx); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - return nil -} - -// Start implements backend.Backend -func (be *postgresBackend) Start(ctx context.Context) error { - if be.db == nil { - pool, err := pgxpool.NewWithConfig(ctx, be.options.PgOptions) - if err != nil { - be.logger.Error("Start", "failed to create a new postgres pool", err) - return fmt.Errorf("failed to create a new postgres pool %w", err) - } - be.db = pool - } - - return nil -} - -// Stop implements backend.Backend -func (be *postgresBackend) Stop(context.Context) error { - if be.db != nil { - be.db.Close() - be.db = nil - } - - return nil -} - -func (be *postgresBackend) ensureDB() error { - if be.db == nil { - return backend.ErrNotInitialized - } - return nil -} - -func (be *postgresBackend) String() string { - maskedPassword := strings.Repeat("*", len(be.options.PgOptions.ConnConfig.Password)) - connectionURI := fmt.Sprintf("postgresql://%s:%s@%s:%d/%s", be.options.PgOptions.ConnConfig.User, maskedPassword, be.options.PgOptions.ConnConfig.Host, be.options.PgOptions.ConnConfig.Port, be.options.PgOptions.ConnConfig.Database) - return connectionURI -} diff --git a/backend/postgres/schema.sql b/backend/postgres/schema.sql deleted file mode 100644 index 65ac01b4..00000000 --- a/backend/postgres/schema.sql +++ /dev/null @@ -1,63 +0,0 @@ -CREATE TABLE IF NOT EXISTS Instances ( - SequenceNumber SERIAL, - - InstanceID TEXT PRIMARY KEY NOT NULL, - ExecutionID TEXT NOT NULL, - Name TEXT NOT NULL, -- the type name of the orchestration or entity - Version TEXT NULL, -- the version of the orchestration (optional) - RuntimeStatus TEXT NOT NULL, - CreatedTime TIMESTAMP NOT NULL DEFAULT NOW(), - LastUpdatedTime TIMESTAMP NOT NULL DEFAULT NOW(), - CompletedTime TIMESTAMP NULL, - LockedBy TEXT NULL, - LockExpiration TIMESTAMP NULL, - Input TEXT NULL, - Output TEXT NULL, - CustomStatus TEXT NULL, - FailureDetails BYTEA NULL, - ParentInstanceID TEXT NULL -); - --- This index is used to improve queries with ORDER BY Instances.SequenceNumber -CREATE INDEX IF NOT EXISTS IX_Instances_SequenceNumber ON Instances(SequenceNumber); - --- This index is used by LockNext and Purge logic -CREATE INDEX IF NOT EXISTS IX_Instances_RuntimeStatus ON Instances(RuntimeStatus); - --- This index is intended to help the performance of multi-instance query -CREATE INDEX IF NOT EXISTS IX_Instances_CreatedTime ON Instances(CreatedTime); - --- This index is used to improve queries that use Instances.ParentInstanceID -CREATE INDEX IF NOT EXISTS IX_Instances_ParentInstanceID ON Instances(ParentInstanceID); - -CREATE TABLE IF NOT EXISTS History ( - InstanceID TEXT NOT NULL, - SequenceNumber SERIAL NOT NULL, - EventPayload BYTEA NOT NULL, - - PRIMARY KEY (InstanceID, SequenceNumber) -); - -CREATE TABLE IF NOT EXISTS NewEvents ( - SequenceNumber SERIAL PRIMARY KEY, -- order is important for FIFO - InstanceID TEXT NOT NULL, - ExecutionID TEXT NULL, - Timestamp TIMESTAMP NOT NULL DEFAULT NOW(), - VisibleTime TIMESTAMP NULL, -- for scheduled or abandoned messages - DequeueCount INTEGER NOT NULL DEFAULT 0, - LockedBy TEXT NULL, - EventPayload BYTEA NOT NULL, - - UNIQUE (InstanceID, SequenceNumber) -); - -CREATE TABLE IF NOT EXISTS NewTasks ( - SequenceNumber SERIAL PRIMARY KEY, -- order is important for FIFO - InstanceID TEXT NOT NULL, - ExecutionID TEXT NULL, - Timestamp TIMESTAMP NOT NULL DEFAULT NOW(), - DequeueCount INTEGER NOT NULL DEFAULT 0, - LockedBy TEXT NULL, - LockExpiration TIMESTAMP NULL, - EventPayload BYTEA NOT NULL -); \ No newline at end of file diff --git a/backend/runtimestate.go b/backend/runtimestate.go deleted file mode 100644 index 8dabe3ab..00000000 --- a/backend/runtimestate.go +++ /dev/null @@ -1,345 +0,0 @@ -package backend - -import ( - "errors" - "fmt" - "time" - - "google.golang.org/protobuf/types/known/timestamppb" - "google.golang.org/protobuf/types/known/wrapperspb" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" -) - -var ErrDuplicateEvent = errors.New("duplicate event") - -type OrchestrationRuntimeState struct { - instanceID api.InstanceID - newEvents []*protos.HistoryEvent - oldEvents []*protos.HistoryEvent - pendingTasks []*protos.HistoryEvent - pendingTimers []*protos.HistoryEvent - pendingMessages []OrchestratorMessage - - startEvent *protos.ExecutionStartedEvent - completedEvent *protos.ExecutionCompletedEvent - createdTime time.Time - lastUpdatedTime time.Time - completedTime time.Time - continuedAsNew bool - isSuspended bool - - CustomStatus *wrapperspb.StringValue -} - -type OrchestratorMessage struct { - HistoryEvent *HistoryEvent - TargetInstanceID string -} - -func NewOrchestrationRuntimeState(instanceID api.InstanceID, existingHistory []*HistoryEvent) *OrchestrationRuntimeState { - s := &OrchestrationRuntimeState{ - instanceID: instanceID, - oldEvents: make([]*HistoryEvent, 0, len(existingHistory)), - newEvents: make([]*HistoryEvent, 0, 10), - } - - for _, e := range existingHistory { - _ = s.addEvent(e, false) - } - - return s -} - -// AddEvent appends a new history event to the orchestration history -func (s *OrchestrationRuntimeState) AddEvent(e *HistoryEvent) error { - return s.addEvent(e, true) -} - -func (s *OrchestrationRuntimeState) addEvent(e *HistoryEvent, isNew bool) error { - if startEvent := e.GetExecutionStarted(); startEvent != nil { - if s.startEvent != nil { - return ErrDuplicateEvent - } - s.startEvent = startEvent - s.createdTime = e.Timestamp.AsTime() - } else if completedEvent := e.GetExecutionCompleted(); completedEvent != nil { - if s.completedEvent != nil { - return ErrDuplicateEvent - } - s.completedEvent = completedEvent - s.completedTime = e.Timestamp.AsTime() - } else if e.GetExecutionSuspended() != nil { - s.isSuspended = true - } else if e.GetExecutionResumed() != nil { - s.isSuspended = false - } - - if isNew { - s.newEvents = append(s.newEvents, e) - } else { - s.oldEvents = append(s.oldEvents, e) - } - - s.lastUpdatedTime = e.Timestamp.AsTime() - return nil -} - -func (s *OrchestrationRuntimeState) IsValid() bool { - if len(s.oldEvents) == 0 && len(s.newEvents) == 0 { - // empty orchestration state - return true - } else if s.startEvent != nil { - // orchestration history has a start event - return true - } - return false -} - -// ApplyActions takes a set of actions and updates its internal state, including populating the outbox. -func (s *OrchestrationRuntimeState) ApplyActions(actions []*protos.OrchestratorAction, currentTraceContext *protos.TraceContext) (bool, error) { - for _, action := range actions { - if completedAction := action.GetCompleteOrchestration(); completedAction != nil { - if completedAction.OrchestrationStatus == protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW { - newState := NewOrchestrationRuntimeState(s.instanceID, []*protos.HistoryEvent{}) - newState.continuedAsNew = true - if err := newState.AddEvent(helpers.NewOrchestratorStartedEvent()); err != nil { - return false, fmt.Errorf("failed to add orchestrator started event: %w", err) - } - - // Duplicate the start event info, updating just the input - if err := newState.AddEvent( - helpers.NewExecutionStartedEvent( - s.startEvent.Name, - string(s.instanceID), - completedAction.Result, - s.startEvent.ParentInstance, - s.startEvent.ParentTraceContext, - nil, - ), - ); err != nil { - return false, fmt.Errorf("failed to add execution started event: %w", err) - } - - // Unprocessed "carryover" events - for _, e := range completedAction.CarryoverEvents { - if err := newState.AddEvent(e); err != nil { - return false, fmt.Errorf("failed to add carryover event: %w", err) - } - } - - // Overwrite the current state object with a new one - *s = *newState - - // ignore all remaining actions - return true, nil - } else { - if err := s.AddEvent(helpers.NewExecutionCompletedEvent(action.Id, completedAction.OrchestrationStatus, completedAction.Result, completedAction.FailureDetails)); err != nil { - return false, fmt.Errorf("failed to add execution completed event: %w", err) - } - if s.startEvent.GetParentInstance() != nil { - msg := OrchestratorMessage{ - HistoryEvent: &protos.HistoryEvent{EventId: -1, Timestamp: timestamppb.Now()}, - TargetInstanceID: s.startEvent.GetParentInstance().OrchestrationInstance.GetInstanceId(), - } - if completedAction.OrchestrationStatus == protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { - msg.HistoryEvent.EventType = &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ - SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{ - TaskScheduledId: s.startEvent.ParentInstance.TaskScheduledId, - Result: completedAction.Result, - }, - } - } else { - // TODO: What is the expected result for termination? - msg.HistoryEvent.EventType = &protos.HistoryEvent_SubOrchestrationInstanceFailed{ - SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{ - TaskScheduledId: s.startEvent.ParentInstance.TaskScheduledId, - FailureDetails: completedAction.FailureDetails, - }, - } - } - s.pendingMessages = append(s.pendingMessages, msg) - } - } - } else if createtimer := action.GetCreateTimer(); createtimer != nil { - if err := s.AddEvent(helpers.NewTimerCreatedEvent(action.Id, createtimer.FireAt)); err != nil { - return false, fmt.Errorf("failed to add timer created event: %w", err) - } - s.pendingTimers = append(s.pendingTimers, helpers.NewTimerFiredEvent(action.Id, createtimer.FireAt, currentTraceContext)) - } else if scheduleTask := action.GetScheduleTask(); scheduleTask != nil { - scheduledEvent := helpers.NewTaskScheduledEvent( - action.Id, - scheduleTask.Name, - scheduleTask.Version, - scheduleTask.Input, - currentTraceContext, - ) - if err := s.AddEvent(scheduledEvent); err != nil { - return false, fmt.Errorf("failed to add task scheduled event: %w", err) - } - s.pendingTasks= append(s.pendingTasks, scheduledEvent) - } else if createSO := action.GetCreateSubOrchestration(); createSO != nil { - // Autogenerate an instance ID for the sub-orchestration if none is provided, using a - // deterministic algorithm based on the parent instance ID to help enable de-duplication. - if createSO.InstanceId == "" { - createSO.InstanceId = fmt.Sprintf("%s:%04x", s.instanceID, action.Id) - } - if err := s.AddEvent(helpers.NewSubOrchestrationCreatedEvent( - action.Id, - createSO.Name, - createSO.Version, - createSO.Input, - createSO.InstanceId, - currentTraceContext)); err != nil { - return false, fmt.Errorf("failed to add sub-orchestration created event: %w", err) - } - startEvent := helpers.NewExecutionStartedEvent( - createSO.Name, - createSO.InstanceId, - createSO.Input, - helpers.NewParentInfo(action.Id, s.startEvent.Name, string(s.instanceID)), - currentTraceContext, - nil, - ) - s.pendingMessages = append(s.pendingMessages, OrchestratorMessage{HistoryEvent: startEvent, TargetInstanceID: createSO.InstanceId}) - } else if sendEvent := action.GetSendEvent(); sendEvent != nil { - e := helpers.NewSendEventEvent(action.Id, sendEvent.Instance.InstanceId, sendEvent.Name, sendEvent.Data) - if err := s.AddEvent(e); err != nil { - return false, fmt.Errorf("failed to add send event: %w", err) - } - s.pendingMessages = append(s.pendingMessages, OrchestratorMessage{HistoryEvent: e, TargetInstanceID: sendEvent.Instance.InstanceId}) - } else if terminate := action.GetTerminateOrchestration(); terminate != nil { - // Send a message to terminate the target orchestration - msg := OrchestratorMessage{ - TargetInstanceID: terminate.InstanceId, - HistoryEvent: helpers.NewExecutionTerminatedEvent(terminate.Reason, terminate.Recurse), - } - s.pendingMessages = append(s.pendingMessages, msg) - } else { - return false, fmt.Errorf("unknown action type: %v", action) - } - } - - return false, nil -} - -func (s *OrchestrationRuntimeState) InstanceID() api.InstanceID { - return s.instanceID -} - -func (s *OrchestrationRuntimeState) Name() (string, error) { - if s.startEvent == nil { - return "", api.ErrNotStarted - } - - return s.startEvent.Name, nil -} - -func (s *OrchestrationRuntimeState) Input() (string, error) { - if s.startEvent == nil { - return "", api.ErrNotStarted - } - - // REVIEW: Should we distinguish between no input and the empty string? - return s.startEvent.Input.GetValue(), nil -} - -func (s *OrchestrationRuntimeState) Output() (string, error) { - if s.completedEvent == nil { - return "", api.ErrNotCompleted - } - - // REVIEW: Should we distinguish between no output and the empty string? - return s.completedEvent.Result.GetValue(), nil -} - -func (s *OrchestrationRuntimeState) RuntimeStatus() protos.OrchestrationStatus { - switch { - case s.startEvent == nil: - return protos.OrchestrationStatus_ORCHESTRATION_STATUS_PENDING - case s.isSuspended: - return protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED - case s.completedEvent != nil: - return s.completedEvent.GetOrchestrationStatus() - } - - return protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING -} - -func (s *OrchestrationRuntimeState) CreatedTime() (time.Time, error) { - if s.startEvent == nil { - return time.Time{}, api.ErrNotStarted - } - - return s.createdTime, nil -} - -func (s *OrchestrationRuntimeState) LastUpdatedTime() (time.Time, error) { - if s.startEvent == nil { - return time.Time{}, api.ErrNotStarted - } - - return s.lastUpdatedTime, nil -} - -func (s *OrchestrationRuntimeState) CompletedTime() (time.Time, error) { - if s.completedEvent == nil { - return time.Time{}, api.ErrNotCompleted - } - - return s.completedTime, nil -} - -func (s *OrchestrationRuntimeState) IsCompleted() bool { - return s.completedEvent != nil -} - -func (s *OrchestrationRuntimeState) OldEvents() []*HistoryEvent { - return s.oldEvents -} - -func (s *OrchestrationRuntimeState) NewEvents() []*HistoryEvent { - return s.newEvents -} - -func (s *OrchestrationRuntimeState) FailureDetails() (*TaskFailureDetails, error) { - if s.completedEvent == nil { - return nil, api.ErrNotCompleted - } else if s.completedEvent.FailureDetails == nil { - return nil, api.ErrNoFailures - } - - return s.completedEvent.FailureDetails, nil -} - -func (s *OrchestrationRuntimeState) PendingTimers() []*HistoryEvent { - return s.pendingTimers -} - -func (s *OrchestrationRuntimeState) PendingTasks() []*HistoryEvent { - return s.pendingTasks -} - -func (s *OrchestrationRuntimeState) PendingMessages() []OrchestratorMessage { - return s.pendingMessages -} - -func (s *OrchestrationRuntimeState) ContinuedAsNew() bool { - return s.continuedAsNew -} - -func (s *OrchestrationRuntimeState) String() string { - return fmt.Sprintf("%v:%v", s.instanceID, helpers.ToRuntimeStatusString(s.RuntimeStatus())) -} - -func (s *OrchestrationRuntimeState) getStartedTime() time.Time { - var startTime time.Time - if len(s.oldEvents) > 0 { - startTime = s.oldEvents[0].Timestamp.AsTime() - } else if len(s.newEvents) > 0 { - startTime = s.newEvents[0].Timestamp.AsTime() - } - return startTime -} diff --git a/backend/sqlite/schema.sql b/backend/sqlite/schema.sql deleted file mode 100644 index 921194b9..00000000 --- a/backend/sqlite/schema.sql +++ /dev/null @@ -1,55 +0,0 @@ -CREATE TABLE IF NOT EXISTS Instances ( - [InstanceID] TEXT PRIMARY KEY NOT NULL, - [ExecutionID] TEXT NOT NULL, - [Name] TEXT NOT NULL, -- the type name of the orchestration or entity - [Version] TEXT NULL, -- the version of the orchestration (optional) - [RuntimeStatus] TEXT NOT NULL, - [CreatedTime] DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - [LastUpdatedTime] DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - [CompletedTime] DATETIME NULL, - [LockedBy] TEXT NULL, - [LockExpiration] DATETIME NULL, - [Input] TEXT NULL, - [Output] TEXT NULL, - [CustomStatus] TEXT NULL, - [FailureDetails] BLOB NULL, - [ParentInstanceID] TEXT NULL -); - --- This index is used by LockNext and Purge logic -CREATE INDEX IF NOT EXISTS IX_Instances_RuntimeStatus ON Instances(RuntimeStatus); - --- This index is intended to help the performance of multi-instance query -CREATE INDEX IF NOT EXISTS IX_Instances_CreatedTime ON Instances(CreatedTime); - -CREATE TABLE IF NOT EXISTS History ( - [InstanceID] TEXT NOT NULL, - [SequenceNumber] INTEGER NOT NULL, - [EventPayload] BLOB NOT NULL, - - CONSTRAINT PK_History PRIMARY KEY (InstanceID, SequenceNumber) -); - -CREATE TABLE IF NOT EXISTS NewEvents ( - [SequenceNumber] INTEGER PRIMARY KEY, -- order is important for FIFO - [InstanceID] TEXT NOT NULL, - [ExecutionID] TEXT NULL, - [Timestamp] DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - [VisibleTime] DATETIME NULL, -- for scheduled or abandoned messages - [DequeueCount] INTEGER NOT NULL DEFAULT 0, - [LockedBy] TEXT NULL, - [EventPayload] BLOB NOT NULL, - - CONSTRAINT UX_NewEvents UNIQUE (InstanceID, SequenceNumber) -); - -CREATE TABLE IF NOT EXISTS NewTasks ( - [SequenceNumber] INTEGER PRIMARY KEY, -- order is important for FIFO - [InstanceID] TEXT NOT NULL, - [ExecutionID] TEXT NULL, - [Timestamp] DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - [DequeueCount] INTEGER NOT NULL DEFAULT 0, - [LockedBy] TEXT NULL, - [LockExpiration] DATETIME NULL, - [EventPayload] BLOB NOT NULL -); \ No newline at end of file diff --git a/backend/sqlite/sqlite.go b/backend/sqlite/sqlite.go deleted file mode 100644 index 86b62e9c..00000000 --- a/backend/sqlite/sqlite.go +++ /dev/null @@ -1,1027 +0,0 @@ -package sqlite - -import ( - "context" - "database/sql" - _ "embed" - "errors" - "fmt" - "os" - "strings" - "time" - - "github.com/google/uuid" - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "google.golang.org/protobuf/proto" - - _ "modernc.org/sqlite" -) - -//go:embed schema.sql -var schema string - -var emptyString string = "" - -type SqliteOptions struct { - OrchestrationLockTimeout time.Duration - ActivityLockTimeout time.Duration - FilePath string -} - -type sqliteBackend struct { - dsn string - db *sql.DB - workerName string - logger backend.Logger - options *SqliteOptions -} - -// NewSqliteOptions creates a new options object for the sqlite backend provider. -// -// Specify "" for filePath to configure an in-memory database. -func NewSqliteOptions(filePath string) *SqliteOptions { - // Default values are provided for required options - return &SqliteOptions{ - FilePath: filePath, - OrchestrationLockTimeout: 2 * time.Minute, - ActivityLockTimeout: 2 * time.Minute, - } -} - -// NewSqliteBackend creates a new sqlite-based Backend object. -func NewSqliteBackend(opts *SqliteOptions, logger backend.Logger) backend.Backend { - hostname, err := os.Hostname() - if err != nil { - hostname = "unknown" - } - - pid := os.Getpid() - uuidStr := uuid.NewString() - - be := &sqliteBackend{ - db: nil, - workerName: fmt.Sprintf("%s,%d,%s", hostname, pid, uuidStr), - options: opts, - logger: logger, - } - - if opts == nil { - opts = NewSqliteOptions("") - } - switch { - case opts.FilePath == "": - be.dsn = "file::memory:" - case !strings.HasPrefix(opts.FilePath, "file:"): - be.dsn = "file:" + opts.FilePath - default: - be.dsn = opts.FilePath - } - - // used for local debug - // be.dsn = "file:file.sqlite" - - return be -} - -// CreateTaskHub creates the sqlite database and applies the schema -func (be *sqliteBackend) CreateTaskHub(ctx context.Context) error { - if err := be.Start(ctx); err != nil { - return fmt.Errorf("failed to start the backend: %w", err) - } - - // Initialize database - if _, err := be.db.Exec(schema); err != nil { - return fmt.Errorf("failed to initialize the database: %w", err) - } - - return nil -} - -func (be *sqliteBackend) DeleteTaskHub(ctx context.Context) error { - if be.db == nil { - return nil - } - - if err := be.Stop(ctx); err != nil { - return fmt.Errorf("failed to stop the backend: %w", err) - } - - if be.options.FilePath == "" { - // In-memory DB - return nil - } - - // File-system DB - err := os.Remove(be.options.FilePath) - switch { - case err == nil: - return nil - case os.IsNotExist(err): - return backend.ErrTaskHubNotFound - default: - return fmt.Errorf("failed to delete the database: %w", err) - } -} - -// AbandonOrchestrationWorkItem implements backend.Backend -func (be *sqliteBackend) AbandonOrchestrationWorkItem(ctx context.Context, wi *backend.OrchestrationWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - var visibleTime *time.Time = nil - if delay := wi.GetAbandonDelay(); delay > 0 { - t := time.Now().UTC().Add(delay) - visibleTime = &t - } - - dbResult, err := tx.ExecContext( - ctx, - "UPDATE NewEvents SET [LockedBy] = NULL, [VisibleTime] = ? WHERE [InstanceID] = ? AND [LockedBy] = ?", - visibleTime, - string(wi.InstanceID), - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to update NewEvents table: %w", err) - } - - rowsAffected, err := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by UPDATE NewEvents statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - dbResult, err = tx.ExecContext( - ctx, - "UPDATE Instances SET [LockedBy] = NULL, [LockExpiration] = NULL WHERE [InstanceID] = ? AND [LockedBy] = ?", - string(wi.InstanceID), - wi.LockedBy, - ) - - if err != nil { - return fmt.Errorf("failed to update Instances table: %w", err) - } - - rowsAffected, err = dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by UPDATE Instances statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -// CompleteOrchestrationWorkItem implements backend.Backend -func (be *sqliteBackend) CompleteOrchestrationWorkItem(ctx context.Context, wi *backend.OrchestrationWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - now := time.Now().UTC() - - // Dynamically generate the UPDATE statement for the Instances table - var sqlSB strings.Builder - sqlSB.WriteString("UPDATE Instances SET ") - - sqlUpdateArgs := make([]any, 0, 10) - isCreated := false - isCompleted := false - - for _, e := range wi.State.NewEvents() { - if es := e.GetExecutionStarted(); es != nil { - if isCreated { - // TODO: Log warning about duplicate start event - continue - } - isCreated = true - sqlSB.WriteString("[CreatedTime] = ?, [Input] = ?, ") - sqlUpdateArgs = append(sqlUpdateArgs, e.Timestamp.AsTime()) - sqlUpdateArgs = append(sqlUpdateArgs, es.Input.GetValue()) - } else if ec := e.GetExecutionCompleted(); ec != nil { - if isCompleted { - // TODO: Log warning about duplicate completion event - continue - } - isCompleted = true - sqlSB.WriteString("[CompletedTime] = ?, [Output] = ?, [FailureDetails] = ?, ") - sqlUpdateArgs = append(sqlUpdateArgs, now) - sqlUpdateArgs = append(sqlUpdateArgs, ec.Result.GetValue()) - if ec.FailureDetails != nil { - bytes, err := proto.Marshal(ec.FailureDetails) - if err != nil { - return fmt.Errorf("failed to marshal FailureDetails: %w", err) - } - sqlUpdateArgs = append(sqlUpdateArgs, &bytes) - } else { - sqlUpdateArgs = append(sqlUpdateArgs, nil) - } - } - // TODO: Execution suspended & resumed - } - - if wi.State.CustomStatus != nil { - sqlSB.WriteString("[CustomStatus] = ?, ") - sqlUpdateArgs = append(sqlUpdateArgs, wi.State.CustomStatus.Value) - } - - // TODO: Support for stickiness, which would extend the LockExpiration - sqlSB.WriteString("[RuntimeStatus] = ?, [LastUpdatedTime] = ?, [LockExpiration] = NULL WHERE [InstanceID] = ? AND [LockedBy] = ?") - sqlUpdateArgs = append(sqlUpdateArgs, helpers.ToRuntimeStatusString(wi.State.RuntimeStatus()), now, string(wi.InstanceID), wi.LockedBy) - - result, err := tx.ExecContext(ctx, sqlSB.String(), sqlUpdateArgs...) - if err != nil { - return fmt.Errorf("failed to update Instances table: %w", err) - } - - count, err := result.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get the number of rows affected by the Instance table update: %w", err) - } else if count == 0 { - return fmt.Errorf("instance '%s' no longer exists or was locked by a different worker", string(wi.InstanceID)) - } - - // If continue-as-new, delete all existing history - if wi.State.ContinuedAsNew() { - if _, err := tx.ExecContext(ctx, "DELETE FROM History WHERE InstanceID = ?", string(wi.InstanceID)); err != nil { - return fmt.Errorf("failed to delete from History table: %w", err) - } - } - - // Save new history events - newHistoryCount := len(wi.State.NewEvents()) - if newHistoryCount > 0 { - query := "INSERT INTO History ([InstanceID], [SequenceNumber], [EventPayload]) VALUES (?, ?, ?)" + - strings.Repeat(", (?, ?, ?)", newHistoryCount-1) - - args := make([]any, 0, newHistoryCount*3) - nextSequenceNumber := len(wi.State.OldEvents()) - for _, e := range wi.State.NewEvents() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - args = append(args, string(wi.InstanceID), nextSequenceNumber, eventPayload) - nextSequenceNumber++ - } - - _, err = tx.ExecContext(ctx, query, args...) - if err != nil { - return fmt.Errorf("failed to insert into the History table: %w", err) - } - } - - // Save outbound activity tasks - newActivityCount := len(wi.State.PendingTasks()) - if newActivityCount > 0 { - insertSql := "INSERT INTO NewTasks ([InstanceID], [EventPayload]) VALUES (?, ?)" + - strings.Repeat(", (?, ?)", newActivityCount-1) - - sqlInsertArgs := make([]any, 0, newActivityCount*2) - for _, e := range wi.State.PendingTasks() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - sqlInsertArgs = append(sqlInsertArgs, string(wi.InstanceID), eventPayload) - } - - _, err = tx.ExecContext(ctx, insertSql, sqlInsertArgs...) - if err != nil { - return fmt.Errorf("failed to insert into the NewTasks table: %w", err) - } - } - - // Save outbound orchestrator events - newEventCount := len(wi.State.PendingTimers()) + len(wi.State.PendingMessages()) - if newEventCount > 0 { - insertSql := "INSERT INTO NewEvents ([InstanceID], [EventPayload], [VisibleTime]) VALUES (?, ?, ?)" + - strings.Repeat(", (?, ?, ?)", newEventCount-1) - - sqlInsertArgs := make([]any, 0, newEventCount*3) - for _, e := range wi.State.PendingTimers() { - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - visibileTime := e.GetTimerFired().GetFireAt().AsTime() - sqlInsertArgs = append(sqlInsertArgs, string(wi.InstanceID), eventPayload, visibileTime) - } - - for _, msg := range wi.State.PendingMessages() { - if es := msg.HistoryEvent.GetExecutionStarted(); es != nil { - // Need to insert a new row into the DB - if _, err := be.createOrchestrationInstanceInternal(ctx, msg.HistoryEvent, tx, backend.WithOrchestrationIdReusePolicy(&protos.OrchestrationIdReusePolicy{ - OperationStatus: []protos.OrchestrationStatus{protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED}, - Action: api.REUSE_ID_ACTION_TERMINATE, - })); err != nil { - if errors.Is(err, backend.ErrDuplicateEvent) { - be.logger.Warnf( - "%v: dropping sub-orchestration creation event because an instance with the target ID (%v) already exists.", - wi.InstanceID, - es.OrchestrationInstance.InstanceId) - } else { - return err - } - } - } - - eventPayload, err := backend.MarshalHistoryEvent(msg.HistoryEvent) - if err != nil { - return err - } - - sqlInsertArgs = append(sqlInsertArgs, msg.TargetInstanceID, eventPayload, nil) - } - - _, err = tx.ExecContext(ctx, insertSql, sqlInsertArgs...) - if err != nil { - return fmt.Errorf("failed to insert into the NewEvents table: %w", err) - } - } - - // Delete inbound events - dbResult, err := tx.ExecContext( - ctx, - "DELETE FROM NewEvents WHERE [InstanceID] = ? AND [LockedBy] = ?", - string(wi.InstanceID), - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to delete from NewEvents table: %w", err) - } - - rowsAffected, err := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by delete statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err != nil { - return fmt.Errorf("failed to delete from the NewEvents table: %w", err) - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -// CreateOrchestrationInstance implements backend.Backend -func (be *sqliteBackend) CreateOrchestrationInstance(ctx context.Context, e *backend.HistoryEvent, opts ...backend.OrchestrationIdReusePolicyOptions) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return fmt.Errorf("failed to start transaction: %w", err) - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - var instanceID string - if instanceID, err = be.createOrchestrationInstanceInternal(ctx, e, tx, opts...); errors.Is(err, api.ErrIgnoreInstance) { - // choose to ignore, do nothing - return nil - } else if err != nil { - return err - } - - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - _, err = tx.ExecContext( - ctx, - `INSERT INTO NewEvents ([InstanceID], [EventPayload]) VALUES (?, ?)`, - instanceID, - eventPayload, - ) - - if err != nil { - return fmt.Errorf("failed to insert row into [NewEvents] table: %w", err) - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to create orchestration: %w", err) - } - - return nil -} - -func (be *sqliteBackend) createOrchestrationInstanceInternal(ctx context.Context, e *backend.HistoryEvent, tx *sql.Tx, opts ...backend.OrchestrationIdReusePolicyOptions) (string, error) { - if e == nil { - return "", backend.ErrNilHistoryEvent - } else if e.Timestamp == nil { - return "", backend.ErrNilEventTimestamp - } - - startEvent := e.GetExecutionStarted() - if startEvent == nil { - return "", backend.ErrNotExecutionStarted - } - instanceID := startEvent.OrchestrationInstance.InstanceId - - policy := &protos.OrchestrationIdReusePolicy{} - - for _, opt := range opts { - if err := opt(policy); err != nil { - return "", err - } - } - - rows, err := insertOrIgnoreInstanceTableInternal(ctx, tx, e, startEvent) - if err != nil { - return "", err - } - - // instance with same ID already exists - if rows <= 0 { - return instanceID, be.handleInstanceExists(ctx, tx, startEvent, policy, e) - } - return instanceID, nil -} - -func insertOrIgnoreInstanceTableInternal(ctx context.Context, tx *sql.Tx, e *backend.HistoryEvent, startEvent *protos.ExecutionStartedEvent) (int64, error) { - var parentInstanceID *string - if pi := startEvent.GetParentInstance(); pi != nil { - if instanceID := pi.GetOrchestrationInstance().GetInstanceId(); instanceID != "" { - parentInstanceID = &instanceID - } - } - res, err := tx.ExecContext( - ctx, - `INSERT OR IGNORE INTO [Instances] ( - [Name], - [Version], - [InstanceID], - [ExecutionID], - [Input], - [RuntimeStatus], - [CreatedTime], - [ParentInstanceID] - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, - startEvent.Name, - startEvent.Version.GetValue(), - startEvent.OrchestrationInstance.InstanceId, - startEvent.OrchestrationInstance.ExecutionId.GetValue(), - startEvent.Input.GetValue(), - "PENDING", - e.Timestamp.AsTime(), - parentInstanceID, - ) - if err != nil { - return -1, fmt.Errorf("failed to insert into [Instances] table: %w", err) - } - - rows, err := res.RowsAffected() - if err != nil { - return -1, fmt.Errorf("failed to count the rows affected: %w", err) - } - return rows, nil -} - -func (be *sqliteBackend) handleInstanceExists(ctx context.Context, tx *sql.Tx, startEvent *protos.ExecutionStartedEvent, policy *protos.OrchestrationIdReusePolicy, e *backend.HistoryEvent) error { - // query RuntimeStatus for the existing instance - queryRow := tx.QueryRowContext( - ctx, - `SELECT [RuntimeStatus] FROM Instances WHERE [InstanceID] = ?`, - startEvent.OrchestrationInstance.InstanceId, - ) - var runtimeStatus *string - err := queryRow.Scan(&runtimeStatus) - if errors.Is(err, sql.ErrNoRows) { - return api.ErrInstanceNotFound - } else if err != nil { - return fmt.Errorf("failed to scan the Instances table result: %w", err) - } - - // status not match, return instance duplicate error - if !isStatusMatch(policy.OperationStatus, helpers.FromRuntimeStatusString(*runtimeStatus)) { - return api.ErrDuplicateInstance - } - - // status match - switch policy.Action { - case protos.CreateOrchestrationAction_IGNORE: - // Log an warning message and ignore creating new instance - be.logger.Warnf("An instance with ID '%s' already exists; dropping duplicate create request", startEvent.OrchestrationInstance.InstanceId) - return api.ErrIgnoreInstance - case protos.CreateOrchestrationAction_TERMINATE: - // terminate existing instance - if err := be.cleanupOrchestrationStateInternal(ctx, tx, api.InstanceID(startEvent.OrchestrationInstance.InstanceId), false); err != nil { - return fmt.Errorf("failed to cleanup orchestration status: %w", err) - } - // create a new instance - var rows int64 - if rows, err = insertOrIgnoreInstanceTableInternal(ctx, tx, e, startEvent); err != nil { - return err - } - - // should never happen, because we clean up instance before create new one - if rows <= 0 { - return fmt.Errorf("failed to insert into [Instances] table because entry already exists") - } - return nil - } - // default behavior - return api.ErrDuplicateInstance -} - -func isStatusMatch(statuses []protos.OrchestrationStatus, runtimeStatus protos.OrchestrationStatus) bool { - for _, status := range statuses { - if status == runtimeStatus { - return true - } - } - return false -} - -func (be *sqliteBackend) cleanupOrchestrationStateInternal(ctx context.Context, tx *sql.Tx, id api.InstanceID, requireCompleted bool) error { - row := tx.QueryRowContext(ctx, "SELECT 1 FROM Instances WHERE [InstanceID] = ?", string(id)) - if err := row.Err(); err != nil { - return fmt.Errorf("failed to query for instance existence: %w", err) - } - - var unused int - if err := row.Scan(&unused); errors.Is(err, sql.ErrNoRows) { - return api.ErrInstanceNotFound - } else if err != nil { - return fmt.Errorf("failed to scan instance existence: %w", err) - } - - if requireCompleted { - // purge orchestration in ['COMPLETED', 'FAILED', 'TERMINATED'] - dbResult, err := tx.ExecContext(ctx, "DELETE FROM Instances WHERE [InstanceID] = ? AND [RuntimeStatus] IN ('COMPLETED', 'FAILED', 'TERMINATED')", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from the Instances table: %w", err) - } - - rowsAffected, err := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed to get rows affected in Instances delete operation: %w", err) - } - if rowsAffected == 0 { - return api.ErrNotCompleted - } - } else { - // clean up orchestration in all [RuntimeStatus] - _, err := tx.ExecContext(ctx, "DELETE FROM Instances WHERE [InstanceID] = ?", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from the Instances table: %w", err) - } - } - - _, err := tx.ExecContext(ctx, "DELETE FROM History WHERE [InstanceID] = ?", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from History table: %w", err) - } - - _, err = tx.ExecContext(ctx, "DELETE FROM NewEvents WHERE [InstanceID] = ?", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from NewEvents table: %w", err) - } - - _, err = tx.ExecContext(ctx, "DELETE FROM NewTasks WHERE [InstanceID] = ?", string(id)) - if err != nil { - return fmt.Errorf("failed to delete from NewTasks table: %w", err) - } - return nil -} - -func (be *sqliteBackend) AddNewOrchestrationEvent(ctx context.Context, iid api.InstanceID, e *backend.HistoryEvent) error { - if e == nil { - return backend.ErrNilHistoryEvent - } else if e.Timestamp == nil { - return backend.ErrNilEventTimestamp - } - - eventPayload, err := backend.MarshalHistoryEvent(e) - if err != nil { - return err - } - - _, err = be.db.ExecContext( - ctx, - `INSERT INTO NewEvents ([InstanceID], [EventPayload]) VALUES (?, ?)`, - string(iid), - eventPayload, - ) - - if err != nil { - return fmt.Errorf("failed to insert row into [NewEvents] table: %w", err) - } - - return nil -} - -// GetOrchestrationMetadata implements backend.Backend -func (be *sqliteBackend) GetOrchestrationMetadata(ctx context.Context, iid api.InstanceID) (*api.OrchestrationMetadata, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - row := be.db.QueryRowContext( - ctx, - `SELECT [InstanceID], [Name], [RuntimeStatus], [CreatedTime], [LastUpdatedTime], [Input], [Output], [CustomStatus], [FailureDetails] - FROM Instances WHERE [InstanceID] = ?`, - string(iid), - ) - - err := row.Err() - if errors.Is(err, sql.ErrNoRows) { - return nil, api.ErrInstanceNotFound - } else if err != nil { - return nil, fmt.Errorf("failed to query the Instances table: %w", row.Err()) - } - - var instanceID *string - var name *string - var runtimeStatus *string - var createdAt *time.Time - var lastUpdatedAt *time.Time - var input *string - var output *string - var customStatus *string - var failureDetails *protos.TaskFailureDetails - - var failureDetailsPayload []byte - err = row.Scan(&instanceID, &name, &runtimeStatus, &createdAt, &lastUpdatedAt, &input, &output, &customStatus, &failureDetailsPayload) - if errors.Is(err, sql.ErrNoRows) { - return nil, api.ErrInstanceNotFound - } else if err != nil { - return nil, fmt.Errorf("failed to scan the Instances table result: %w", err) - } - - if input == nil { - input = &emptyString - } - - if output == nil { - output = &emptyString - } - - if customStatus == nil { - customStatus = &emptyString - } - - if len(failureDetailsPayload) > 0 { - failureDetails = new(protos.TaskFailureDetails) - if err := proto.Unmarshal(failureDetailsPayload, failureDetails); err != nil { - return nil, fmt.Errorf("failed to unmarshal failure details: %w", err) - } - } - - metadata := api.NewOrchestrationMetadata( - iid, - *name, - helpers.FromRuntimeStatusString(*runtimeStatus), - *createdAt, - *lastUpdatedAt, - *input, - *output, - *customStatus, - failureDetails, - ) - return metadata, nil -} - -// GetOrchestrationRuntimeState implements backend.Backend -func (be *sqliteBackend) GetOrchestrationRuntimeState(ctx context.Context, wi *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - rows, err := be.db.QueryContext( - ctx, - "SELECT [EventPayload] FROM History WHERE [InstanceID] = ? ORDER BY [SequenceNumber] ASC", - string(wi.InstanceID), - ) - if err != nil { - return nil, err - } - - existingEvents := make([]*protos.HistoryEvent, 0, 50) - for rows.Next() { - var eventPayload []byte - if err := rows.Scan(&eventPayload); err != nil { - return nil, fmt.Errorf("failed to read history event: %w", err) - } - - e, err := backend.UnmarshalHistoryEvent(eventPayload) - if err != nil { - return nil, err - } - - existingEvents = append(existingEvents, e) - } - - state := backend.NewOrchestrationRuntimeState(wi.InstanceID, existingEvents) - return state, nil -} - -// GetOrchestrationWorkItem implements backend.Backend -func (be *sqliteBackend) GetOrchestrationWorkItem(ctx context.Context) (*backend.OrchestrationWorkItem, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return nil, err - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - now := time.Now().UTC() - newLockExpiration := now.Add(be.options.OrchestrationLockTimeout) - - // Place a lock on an orchestration instance that has new events that are ready to be executed. - row := tx.QueryRowContext( - ctx, - `UPDATE Instances SET [LockedBy] = ?, [LockExpiration] = ? - WHERE [rowid] = ( - SELECT [rowid] FROM Instances I - WHERE (I.[LockExpiration] IS NULL OR I.[LockExpiration] < ?) AND EXISTS ( - SELECT 1 FROM NewEvents E - WHERE E.[InstanceID] = I.[InstanceID] AND (E.[VisibleTime] IS NULL OR E.[VisibleTime] < ?) - ) - LIMIT 1 - ) RETURNING [InstanceID]`, - be.workerName, // LockedBy for Instances table - newLockExpiration, // Updated LockExpiration for Instances table - now, // LockExpiration for Instances table - now, // VisibleTime for NewEvents table - ) - - if err := row.Err(); err != nil { - return nil, fmt.Errorf("failed to query for orchestration work-items: %w", err) - } - - var instanceID string - if err := row.Scan(&instanceID); err != nil { - if err == sql.ErrNoRows { - // No new events to process - return nil, backend.ErrNoWorkItems - } - - return nil, fmt.Errorf("failed to scan the orchestration work-item: %w", err) - } - - // TODO: Get all the unprocessed events associated with the locked instance - events, err := tx.QueryContext( - ctx, - `UPDATE NewEvents SET [DequeueCount] = [DequeueCount] + 1, [LockedBy] = ? WHERE rowid IN ( - SELECT rowid FROM NewEvents - WHERE [InstanceID] = ? AND ([VisibleTime] IS NULL OR [VisibleTime] <= ?) - LIMIT 1000 - ) - RETURNING [EventPayload], [DequeueCount]`, - be.workerName, - instanceID, - now, - ) - if err != nil { - return nil, fmt.Errorf("failed to query for orchestration work-items: %w", err) - } - - maxDequeueCount := int32(0) - - newEvents := make([]*protos.HistoryEvent, 0, 10) - for events.Next() { - var eventPayload []byte - var dequeueCount int32 - if err := events.Scan(&eventPayload, &dequeueCount); err != nil { - return nil, fmt.Errorf("failed to read history event: %w", err) - } - - if dequeueCount > maxDequeueCount { - maxDequeueCount = dequeueCount - } - - e, err := backend.UnmarshalHistoryEvent(eventPayload) - if err != nil { - return nil, err - } - - newEvents = append(newEvents, e) - } - - if err = tx.Commit(); err != nil { - return nil, fmt.Errorf("failed to update orchestration work-item: %w", err) - } - - wi := &backend.OrchestrationWorkItem{ - InstanceID: api.InstanceID(instanceID), - NewEvents: newEvents, - LockedBy: be.workerName, - RetryCount: maxDequeueCount - 1, - } - - return wi, nil -} - -func (be *sqliteBackend) GetActivityWorkItem(ctx context.Context) (*backend.ActivityWorkItem, error) { - if err := be.ensureDB(); err != nil { - return nil, err - } - - now := time.Now().UTC() - newLockExpiration := now.Add(be.options.OrchestrationLockTimeout) - - row := be.db.QueryRowContext( - ctx, - `UPDATE NewTasks SET [LockedBy] = ?, [LockExpiration] = ?, [DequeueCount] = [DequeueCount] + 1 - WHERE [SequenceNumber] = ( - SELECT [SequenceNumber] FROM NewTasks T - WHERE T.[LockExpiration] IS NULL OR T.[LockExpiration] < ? - ORDER BY T.[SequenceNumber] ASC - LIMIT 1 - ) RETURNING [SequenceNumber], [InstanceID], [EventPayload]`, - be.workerName, - newLockExpiration, - now, - ) - - if err := row.Err(); err != nil { - return nil, fmt.Errorf("failed to query for activity work-items: %w", err) - } - - var sequenceNumber int64 - var instanceID string - var eventPayload []byte - - if err := row.Scan(&sequenceNumber, &instanceID, &eventPayload); err != nil { - if err == sql.ErrNoRows { - // No new activity tasks to process - return nil, backend.ErrNoWorkItems - } - - return nil, fmt.Errorf("failed to scan the activity work-item: %w", err) - } - - e, err := backend.UnmarshalHistoryEvent(eventPayload) - if err != nil { - return nil, err - } - - wi := &backend.ActivityWorkItem{ - SequenceNumber: sequenceNumber, - InstanceID: api.InstanceID(instanceID), - NewEvent: e, - LockedBy: be.workerName, - } - return wi, nil -} - -func (be *sqliteBackend) CompleteActivityWorkItem(ctx context.Context, wi *backend.ActivityWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - bytes, err := backend.MarshalHistoryEvent(wi.Result) - if err != nil { - return err - } - - _, err = tx.ExecContext(ctx, "INSERT INTO NewEvents ([InstanceID], [EventPayload]) VALUES (?, ?)", string(wi.InstanceID), bytes) - if err != nil { - return fmt.Errorf("failed to insert into NewEvents table: %w", err) - } - - dbResult, err := tx.ExecContext(ctx, "DELETE FROM NewTasks WHERE [SequenceNumber] = ? AND [LockedBy] = ?", wi.SequenceNumber, wi.LockedBy) - if err != nil { - return fmt.Errorf("failed to delete from NewTasks table: %w", err) - } - - rowsAffected, err := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by delete statement: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -func (be *sqliteBackend) AbandonActivityWorkItem(ctx context.Context, wi *backend.ActivityWorkItem) error { - if err := be.ensureDB(); err != nil { - return err - } - - dbResult, err := be.db.ExecContext( - ctx, - "UPDATE NewTasks SET [LockedBy] = NULL, [LockExpiration] = NULL WHERE [SequenceNumber] = ? AND [LockedBy] = ?", - wi.SequenceNumber, - wi.LockedBy, - ) - if err != nil { - return fmt.Errorf("failed to update the NewTasks table for abandon: %w", err) - } - - rowsAffected, err := dbResult.RowsAffected() - if err != nil { - return fmt.Errorf("failed get rows affected by update statement for abandon: %w", err) - } else if rowsAffected == 0 { - return backend.ErrWorkItemLockLost - } - - return nil -} - -func (be *sqliteBackend) PurgeOrchestrationState(ctx context.Context, id api.InstanceID) error { - if err := be.ensureDB(); err != nil { - return err - } - - tx, err := be.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer tx.Rollback() //nolint:errcheck // rollback after commit is a no-op - - if err := be.cleanupOrchestrationStateInternal(ctx, tx, id, true); err != nil { - return err - } - - if err = tx.Commit(); err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - return nil -} - -// Start implements backend.Backend -func (be *sqliteBackend) Start(context.Context) error { - if be.db == nil { - db, err := sql.Open("sqlite", be.dsn) - if err != nil { - return fmt.Errorf("failed to open the database: %w", err) - } - - // TODO: This is to avoid SQLITE_BUSY errors when there are concurrent - // operations on the database. However, it can hurt performance. - // We should consider removing this and looking for alternate - // solutions if sqlite performance becomes a problem for users. - db.SetMaxOpenConns(1) - - be.db = db - } - - return nil -} - -// Stop implements backend.Backend -func (be *sqliteBackend) Stop(context.Context) error { - if be.db != nil { - be.db = nil - } - - return nil -} - -func (be *sqliteBackend) ensureDB() error { - if be.db == nil { - return backend.ErrNotInitialized - } - return nil -} - -func (be *sqliteBackend) String() string { - return fmt.Sprintf("sqlite::%s", be.options.FilePath) -} diff --git a/backend/taskhub.go b/backend/taskhub.go deleted file mode 100644 index fa4603b9..00000000 --- a/backend/taskhub.go +++ /dev/null @@ -1,73 +0,0 @@ -package backend - -import ( - "context" - "errors" - "sync" -) - -type TaskHubWorker interface { - // Start starts the backend and the configured internal workers. - Start(context.Context) error - - // Shutdown stops the backend and all internal workers. - Shutdown(context.Context) error -} - -type taskHubWorker struct { - backend Backend - orchestrationWorker TaskWorker - activityWorker TaskWorker - logger Logger -} - -func NewTaskHubWorker(be Backend, orchestrationWorker TaskWorker, activityWorker TaskWorker, logger Logger) TaskHubWorker { - return &taskHubWorker{ - backend: be, - orchestrationWorker: orchestrationWorker, - activityWorker: activityWorker, - logger: logger, - } -} - -func (w *taskHubWorker) Start(ctx context.Context) error { - // TODO: Check for already started worker - if err := w.backend.CreateTaskHub(ctx); err != nil && !errors.Is(err, ErrTaskHubExists) { - return err - } - if err := w.backend.Start(ctx); err != nil { - return err - } - w.logger.Infof("worker started with backend %v", w.backend) - - w.orchestrationWorker.Start(ctx) - w.activityWorker.Start(ctx) - return nil -} - -func (w *taskHubWorker) Shutdown(ctx context.Context) error { - w.logger.Info("backend stopping...") - if err := w.backend.Stop(ctx); err != nil { - return err - } - - w.logger.Info("workers stopping and draining...") - defer w.logger.Info("finished stopping and draining workers!") - - wg := sync.WaitGroup{} - wg.Add(1) - go func() { - defer wg.Done() - w.orchestrationWorker.StopAndDrain() - }() - - wg.Add(1) - go func() { - defer wg.Done() - w.activityWorker.StopAndDrain() - }() - - wg.Wait() - - return nil -} diff --git a/backend/worker.go b/backend/worker.go deleted file mode 100644 index 640e7bc3..00000000 --- a/backend/worker.go +++ /dev/null @@ -1,255 +0,0 @@ -package backend - -import ( - "context" - "errors" - "sync" - "sync/atomic" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/marusama/semaphore/v2" -) - -type TaskWorker interface { - // Start starts background polling for the activity work items. - Start(context.Context) - - // ProcessNext attempts to fetch and process a work item. This method returns - // true if a work item was found and processing started; false otherwise. An - // error is returned if the context is cancelled. - ProcessNext(context.Context) (bool, error) - - // StopAndDrain stops the worker and waits for all outstanding work items to finish. - StopAndDrain() -} - -type TaskProcessor interface { - Name() string - FetchWorkItem(context.Context) (WorkItem, error) - ProcessWorkItem(context.Context, WorkItem) error - AbandonWorkItem(context.Context, WorkItem) error - CompleteWorkItem(context.Context, WorkItem) error -} - -type worker struct { - options *WorkerOptions - logger Logger - // dispatchSemaphore is for throttling orchestration concurrency. - dispatchSemaphore semaphore.Semaphore - - // pending is for keeping track of outstanding orchestration executions. - pending *sync.WaitGroup - - // cancel is used to cancel background polling. - // It will be nil if background polling isn't started. - cancel context.CancelFunc - processor TaskProcessor - waiting bool - stop atomic.Bool -} - -type NewTaskWorkerOptions func(*WorkerOptions) - -type WorkerOptions struct { - MaxParallelWorkItems int32 -} - -func NewWorkerOptions() *WorkerOptions { - return &WorkerOptions{ - MaxParallelWorkItems: 1, - } -} - -func WithMaxParallelism(n int32) NewTaskWorkerOptions { - return func(o *WorkerOptions) { - o.MaxParallelWorkItems = n - } -} - -func NewTaskWorker(p TaskProcessor, logger Logger, opts ...NewTaskWorkerOptions) TaskWorker { - options := &WorkerOptions{MaxParallelWorkItems: 1} - for _, configure := range opts { - configure(options) - } - return &worker{ - processor: p, - logger: logger, - dispatchSemaphore: semaphore.New(int(options.MaxParallelWorkItems)), - pending: &sync.WaitGroup{}, - cancel: nil, // assigned later - options: options, - } -} - -func (w *worker) Name() string { - return w.processor.Name() -} - -func (w *worker) Start(ctx context.Context) { - // TODO: Check for already started worker - ctx, cancel := context.WithCancel(ctx) - w.cancel = cancel - - w.stop.Store(false) - - go func() { - var b backoff.BackOff = &backoff.ExponentialBackOff{ - InitialInterval: 50 * time.Millisecond, - MaxInterval: 5 * time.Second, - Multiplier: 1.05, - RandomizationFactor: 0.05, - Stop: backoff.Stop, - Clock: backoff.SystemClock, - } - b = backoff.WithContext(b, ctx) - b.Reset() - - loop: - for { - // returns right away, with "ok" if a work item was found - ok, err := w.ProcessNext(ctx) - - switch { - case ok: - // found a work item - reset the backoff and check for the next item - b.Reset() - case err != nil && errors.Is(err, ctx.Err()): - // there's an error and it's due to the context being canceled - w.logger.Infof("%v: received cancellation signal", w.Name()) - break loop - case err != nil: - // another error was encountered - // log the error and inject some extra sleep to avoid tight failure loops - w.logger.Errorf("unexpected worker error: %v. Adding 5 extra seconds of backoff.", err) - t := time.NewTimer(5 * time.Second) - select { - case <-t.C: - // nop - all good - case <-ctx.Done(): - if !t.Stop() { - <-t.C - } - w.logger.Infof("%v: received cancellation signal", w.Name()) - break loop - } - default: - // no work item found, so sleep until the next backoff - t := time.NewTimer(b.NextBackOff()) - select { - case <-t.C: - // nop - all good - case <-ctx.Done(): - if !t.Stop() { - <-t.C - } - w.logger.Infof("%v: received cancellation signal", w.Name()) - break loop - } - } - } - - w.logger.Infof("%v: stopped listening for new work items", w.Name()) - }() -} - -func (w *worker) ProcessNext(ctx context.Context) (bool, error) { - if !w.dispatchSemaphore.TryAcquire(1) { - w.logger.Debugf("%v: waiting for one of %v in-flight execution(s) to complete", w.Name(), w.dispatchSemaphore.GetCount()) - if err := w.dispatchSemaphore.Acquire(ctx, 1); err != nil { - // cancelled - return false, err - } - } - w.pending.Add(1) - - processing := false - defer func() { - if !processing { - w.pending.Done() - w.dispatchSemaphore.Release(1) - } - }() - - wi, err := w.processor.FetchWorkItem(ctx) - switch { - case errors.Is(err, ErrNoWorkItems) || wi == nil: - if !w.waiting { - w.logger.Debugf("%v: waiting for new work items...", w.Name()) - w.waiting = true - } - return false, nil - case err != nil: - if !errors.Is(err, ctx.Err()) { - w.logger.Errorf("%v: failed to fetch work item: %v", w.Name(), err) - } - return false, err - default: - // process the work-item in the background - w.waiting = false - processing = true - go w.processWorkItem(ctx, wi) - return true, nil - } -} - -func (w *worker) StopAndDrain() { - w.logger.Debugf("%v: stop and drain...", w.Name()) - defer w.logger.Debugf("%v: finished stop and drain...", w.Name()) - - w.stop.Store(true) - - // Cancel the background poller and dispatcher(s) - if w.cancel != nil { - w.cancel() - } - - // Wait for outstanding work-items to finish processing. - // TODO: Need to find a way to cancel this if it takes too long for some reason. - w.pending.Wait() -} - -func (w *worker) processWorkItem(ctx context.Context, wi WorkItem) { - defer w.dispatchSemaphore.Release(1) - defer w.pending.Done() - - w.logger.Debugf("%v: processing work item: %s", w.Name(), wi) - - if w.stop.Load() { - if err := w.processor.AbandonWorkItem(context.Background(), wi); err != nil { - w.logger.Errorf("%v: failed to abandon work item: %v", w.Name(), err) - } - return - } - - if err := w.processor.ProcessWorkItem(ctx, wi); err != nil { - if errors.Is(err, ctx.Err()) { - w.logger.Warnf("%v: abandoning work item due to cancellation", w.Name()) - } else { - w.logger.Errorf("%v: failed to process work item: %v", w.Name(), err) - } - if w.stop.Load() { - ctx = context.Background() - } - if err := w.processor.AbandonWorkItem(ctx, wi); err != nil { - w.logger.Errorf("%v: failed to abandon work item: %v", w.Name(), err) - } - return - } - - if err := w.processor.CompleteWorkItem(ctx, wi); err != nil { - if errors.Is(err, ctx.Err()) { - w.logger.Warnf("%v: failed to complete work item due to cancellation", w.Name()) - } else { - w.logger.Errorf("%v: failed to complete work item: %v", w.Name(), err) - } - if w.stop.Load() { - ctx = context.Background() - } - if err := w.processor.AbandonWorkItem(ctx, wi); err != nil { - w.logger.Errorf("%v: failed to abandon work item: %v", w.Name(), err) - } - } - - w.logger.Debugf("%v: work item processed successfully", w.Name()) -} diff --git a/backend/workitem.go b/backend/workitem.go deleted file mode 100644 index 8b2e14ee..00000000 --- a/backend/workitem.go +++ /dev/null @@ -1,67 +0,0 @@ -package backend - -import ( - "errors" - "fmt" - "time" - - "github.com/microsoft/durabletask-go/api" -) - -var ErrNoWorkItems = errors.New("no work items were found") - -type WorkItem interface { - fmt.Stringer - IsWorkItem() bool -} - -type OrchestrationWorkItem struct { - InstanceID api.InstanceID - NewEvents []*HistoryEvent - LockedBy string - RetryCount int32 - State *OrchestrationRuntimeState - Properties map[string]any -} - -// String implements core.WorkItem and fmt.Stringer -func (wi OrchestrationWorkItem) String() string { - return fmt.Sprintf("%s (%d event(s))", wi.InstanceID, len(wi.NewEvents)) -} - -// IsWorkItem implements core.WorkItem -func (wi OrchestrationWorkItem) IsWorkItem() bool { - return true -} - -func (wi *OrchestrationWorkItem) GetAbandonDelay() time.Duration { - switch { - case wi.RetryCount == 0: - return time.Duration(0) // no delay - case wi.RetryCount > 100: - return 5 * time.Minute // max delay - default: - return time.Duration(wi.RetryCount) * time.Second // linear backoff - } -} - -type ActivityWorkItem struct { - SequenceNumber int64 - InstanceID api.InstanceID - NewEvent *HistoryEvent - Result *HistoryEvent - LockedBy string - Properties map[string]any -} - -// String implements core.WorkItem and fmt.Stringer -func (wi ActivityWorkItem) String() string { - name := wi.NewEvent.GetTaskScheduled().GetName() - taskID := wi.NewEvent.EventId - return fmt.Sprintf("%s/%s#%d", wi.InstanceID, name, taskID) -} - -// IsWorkItem implements core.WorkItem -func (wi ActivityWorkItem) IsWorkItem() bool { - return true -} diff --git a/client/client_grpc.go b/client/client_grpc.go index 80631530..1127ad68 100644 --- a/client/client_grpc.go +++ b/client/client_grpc.go @@ -2,42 +2,106 @@ package client import ( "context" + "errors" "fmt" + "strings" + "sync" + "time" "github.com/cenkalti/backoff/v4" "github.com/google/uuid" + "go.opentelemetry.io/otel/trace" "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/largepayload" "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" ) -// REVIEW: Can this be merged with backend/client.go somehow? - type TaskHubGrpcClient struct { - client protos.TaskHubSidecarServiceClient - logger backend.Logger + client protos.TaskHubSidecarServiceClient + connection grpc.ClientConnInterface + logger api.Logger + largePayloads *api.LargePayloadOptions + defaultVersion string + converter api.DataConverter + + listenerMu sync.Mutex + listener *TaskHubGrpcWorker +} + +func newInfiniteRetries() *backoff.ExponentialBackOff { + retries := backoff.NewExponentialBackOff() + retries.MaxInterval = 15 * time.Second + retries.MaxElapsedTime = 0 + retries.Reset() + return retries +} + +type TaskHubGrpcClientOption func(*TaskHubGrpcClient) + +// WithLargePayloads configures externalization and hydration for management payloads. +func WithLargePayloads(options *api.LargePayloadOptions) TaskHubGrpcClientOption { + return func(c *TaskHubGrpcClient) { + if options == nil { + c.largePayloads = nil + return + } + clone := *options + c.largePayloads = &clone + } +} + +// WithDefaultVersion configures the version used when a top-level orchestration +// is scheduled without an explicit [api.WithVersion] option. +func WithDefaultVersion(version string) TaskHubGrpcClientOption { + return func(c *TaskHubGrpcClient) { + c.defaultVersion = version + } +} + +// WithDataConverter configures application payload serialization. +func WithDataConverter(converter api.DataConverter) TaskHubGrpcClientOption { + return func(c *TaskHubGrpcClient) { + c.converter = api.NormalizeDataConverter(converter) + } } -// NewTaskHubGrpcClient creates a client that can be used to manage orchestrations over a gRPC connection. -// The gRPC connection must be to a task hub worker that understands the Durable Task gRPC protocol. -func NewTaskHubGrpcClient(cc grpc.ClientConnInterface, logger backend.Logger) *TaskHubGrpcClient { - return &TaskHubGrpcClient{ - client: protos.NewTaskHubSidecarServiceClient(cc), - logger: logger, +// NewTaskHubGrpcClient creates a client that can be used to manage orchestrations over a borrowed gRPC connection. +// The caller retains ownership of the connection; this client neither closes nor recreates it. DTS applications that +// want an owned, resilient management channel should use durabletaskscheduler.NewClient. +func NewTaskHubGrpcClient(cc grpc.ClientConnInterface, logger api.Logger, opts ...TaskHubGrpcClientOption) *TaskHubGrpcClient { + c := &TaskHubGrpcClient{ + client: protos.NewTaskHubSidecarServiceClient(cc), + connection: cc, + logger: logger, + converter: api.DefaultDataConverter(), + } + for _, configure := range opts { + configure(c) } + return c } // ScheduleNewOrchestration schedules a new orchestration instance with a specified set of options for execution. func (c *TaskHubGrpcClient) ScheduleNewOrchestration(ctx context.Context, orchestrator string, opts ...api.NewOrchestrationOptions) (api.InstanceID, error) { req := &protos.CreateInstanceRequest{Name: orchestrator} for _, configure := range opts { - if err := configure(req); err != nil { - return api.EmptyInstanceID, fmt.Errorf("failed to configure orchestration request: %w", err) + if err := configure(req, c.converter); err != nil { + return api.EmptyInstanceID, fmt.Errorf( + "failed to configure orchestration request: %w", + api.WrapInvalidArgument(err), + ) } } + if req.Version == nil && c.defaultVersion != "" { + req.Version = wrapperspb.String(c.defaultVersion) + } if req.InstanceId == "" { u, err := uuid.NewV7() if err == nil { @@ -46,13 +110,22 @@ func (c *TaskHubGrpcClient) ScheduleNewOrchestration(ctx context.Context, orches req.InstanceId = uuid.NewString() } } + var err error + req.Input, err = largepayload.Externalize(ctx, c.largePayloads, req.Input) + if err != nil { + return api.EmptyInstanceID, fmt.Errorf("failed to externalize orchestration input: %w", err) + } + + // Propagate the caller's distributed trace context so the service can parent the + // orchestration's trace to the code that scheduled it. Without this, every + // orchestration starts a brand new, disconnected trace. + if req.ParentTraceContext == nil { + req.ParentTraceContext = helpers.TraceContextFromSpan(trace.SpanFromContext(ctx)) + } resp, err := c.client.StartInstance(ctx, req) if err != nil { - if ctx.Err() != nil { - return api.EmptyInstanceID, ctx.Err() - } - return api.EmptyInstanceID, fmt.Errorf("failed to start orchestrator: %w", err) + return api.EmptyInstanceID, clientRPCError(ctx, "failed to start orchestration", err) } return api.InstanceID(resp.InstanceId), nil } @@ -64,12 +137,9 @@ func (c *TaskHubGrpcClient) FetchOrchestrationMetadata(ctx context.Context, id a req := makeGetInstanceRequest(id, opts) resp, err := c.client.GetInstance(ctx, req) if err != nil { - if ctx.Err() != nil { - return nil, ctx.Err() - } - return nil, fmt.Errorf("failed to fetch orchestration metadata: %w", err) + return nil, clientRPCError(ctx, "failed to fetch orchestration metadata", err) } - return makeOrchestrationMetadata(resp) + return c.makeOrchestrationMetadata(ctx, resp) } // WaitForOrchestrationStart waits for an orchestration to start running and returns an [api.OrchestrationMetadata] object that contains @@ -83,18 +153,21 @@ func (c *TaskHubGrpcClient) WaitForOrchestrationStart(ctx context.Context, id ap req := makeGetInstanceRequest(id, opts) resp, err = c.client.WaitForInstanceStart(ctx, req) if err != nil { - // if its context cancelled stop retrying if ctx.Err() != nil { return backoff.Permanent(ctx.Err()) } - return fmt.Errorf("failed to wait for orchestration start: %w", err) + mapped := clientRPCError(ctx, "failed to wait for orchestration start", err) + if !retryableWaitRPCError(err) { + return backoff.Permanent(mapped) + } + return mapped } return nil }, backoff.WithContext(newInfiniteRetries(), ctx)) if err != nil { return nil, err } - return makeOrchestrationMetadata(resp) + return c.makeOrchestrationMetadata(ctx, resp) } // WaitForOrchestrationCompletion waits for an orchestration to complete and returns an [api.OrchestrationMetadata] object that contains @@ -108,18 +181,21 @@ func (c *TaskHubGrpcClient) WaitForOrchestrationCompletion(ctx context.Context, req := makeGetInstanceRequest(id, opts) resp, err = c.client.WaitForInstanceCompletion(ctx, req) if err != nil { - // if its context cancelled stop retrying if ctx.Err() != nil { return backoff.Permanent(ctx.Err()) } - return fmt.Errorf("failed to wait for orchestration completion: %w", err) + mapped := clientRPCError(ctx, "failed to wait for orchestration completion", err) + if !retryableWaitRPCError(err) { + return backoff.Permanent(mapped) + } + return mapped } return nil }, backoff.WithContext(newInfiniteRetries(), ctx)) if err != nil { return nil, err } - return makeOrchestrationMetadata(resp) + return c.makeOrchestrationMetadata(ctx, resp) } // TerminateOrchestration terminates a running orchestration by causing it to stop receiving new events and @@ -127,17 +203,19 @@ func (c *TaskHubGrpcClient) WaitForOrchestrationCompletion(ctx context.Context, func (c *TaskHubGrpcClient) TerminateOrchestration(ctx context.Context, id api.InstanceID, opts ...api.TerminateOptions) error { req := &protos.TerminateRequest{InstanceId: string(id), Recursive: true} for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure termination request: %w", err) + if err := configure(req, c.converter); err != nil { + return fmt.Errorf("failed to configure termination request: %w", api.WrapInvalidArgument(err)) } } + var err error + req.Output, err = largepayload.Externalize(ctx, c.largePayloads, req.Output) + if err != nil { + return fmt.Errorf("failed to externalize termination output: %w", err) + } - _, err := c.client.TerminateInstance(ctx, req) + _, err = c.client.TerminateInstance(ctx, req) if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to terminate instance: %w", err) + return clientRPCError(ctx, "failed to terminate orchestration", err) } return nil } @@ -146,16 +224,18 @@ func (c *TaskHubGrpcClient) TerminateOrchestration(ctx context.Context, id api.I func (c *TaskHubGrpcClient) RaiseEvent(ctx context.Context, id api.InstanceID, eventName string, opts ...api.RaiseEventOptions) error { req := &protos.RaiseEventRequest{InstanceId: string(id), Name: eventName} for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure raise event request: %w", err) + if err := configure(req, c.converter); err != nil { + return fmt.Errorf("failed to configure raise event request: %w", api.WrapInvalidArgument(err)) } } + var err error + req.Input, err = largepayload.Externalize(ctx, c.largePayloads, req.Input) + if err != nil { + return fmt.Errorf("failed to externalize event payload: %w", err) + } if _, err := c.client.RaiseEvent(ctx, req); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to raise event: %w", err) + return clientRPCError(ctx, "failed to raise event", err) } return nil } @@ -169,10 +249,7 @@ func (c *TaskHubGrpcClient) SuspendOrchestration(ctx context.Context, id api.Ins Reason: wrapperspb.String(reason), } if _, err := c.client.SuspendInstance(ctx, req); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to suspend orchestration: %w", err) + return clientRPCError(ctx, "failed to suspend orchestration", err) } return nil } @@ -184,10 +261,7 @@ func (c *TaskHubGrpcClient) ResumeOrchestration(ctx context.Context, id api.Inst Reason: wrapperspb.String(reason), } if _, err := c.client.ResumeInstance(ctx, req); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to resume orchestration: %w", err) + return clientRPCError(ctx, "failed to resume orchestration", err) } return nil } @@ -201,22 +275,218 @@ func (c *TaskHubGrpcClient) PurgeOrchestrationState(ctx context.Context, id api. } for _, configure := range opts { if err := configure(req); err != nil { - return fmt.Errorf("failed to configure purge request: %w", err) + return fmt.Errorf("failed to configure purge request: %w", api.WrapInvalidArgument(err)) } } res, err := c.client.PurgeInstances(ctx, req) if err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - return fmt.Errorf("failed to purge orchestration state: %w", err) + return clientRPCError(ctx, "failed to purge orchestration state", err) } else if res.GetDeletedInstanceCount() == 0 { return api.ErrInstanceNotFound } return nil } +// SignalEntity sends a fire-and-forget operation to an entity. +func (c *TaskHubGrpcClient) SignalEntity(ctx context.Context, entityID api.EntityID, operationName string, opts ...api.SignalEntityOptions) error { + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return api.WrapInvalidArgument(err) + } + if operationName == "" { + return api.WrapInvalidArgument(errors.New("entity operation name must not be empty")) + } + req := &protos.SignalEntityRequest{ + InstanceId: entityID.String(), + Name: operationName, + RequestId: uuid.NewString(), + RequestTime: timestamppb.Now(), + ParentTraceContext: helpers.TraceContextFromSpan(trace.SpanFromContext(ctx)), + } + for _, configure := range opts { + if err := configure(req, c.converter); err != nil { + return fmt.Errorf("failed to configure signal entity request: %w", api.WrapInvalidArgument(err)) + } + } + var err error + req.Input, err = largepayload.Externalize(ctx, c.largePayloads, req.Input) + if err != nil { + return fmt.Errorf("failed to externalize entity signal input: %w", err) + } + if _, err := c.client.SignalEntity(ctx, req); err != nil { + return clientRPCError(ctx, "failed to signal entity", err) + } + return nil +} + +// GetEntity retrieves metadata for an entity instance or returns nil when it doesn't exist. +// State is included unless ExcludeState is set. +func (c *TaskHubGrpcClient) GetEntity( + ctx context.Context, + entityID api.EntityID, + options ...api.GetEntityOptions, +) (*api.EntityMetadata, error) { + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return nil, api.WrapInvalidArgument(err) + } + if len(options) > 1 { + return nil, api.WrapInvalidArgument(errors.New("at most one entity metadata options value may be supplied")) + } + includeState := len(options) == 0 || !options[0].ExcludeState + response, err := c.client.GetEntity(ctx, &protos.GetEntityRequest{ + InstanceId: entityID.String(), + IncludeState: includeState, + }) + if err != nil { + return nil, clientRPCError(ctx, "failed to get entity metadata", err) + } + if !response.Exists || response.Entity == nil { + return nil, nil + } + return entityMetadataFromProto(ctx, c.largePayloads, c.converter, response.Entity, includeState) +} + +// QueryEntities queries entities matching the supplied filters. +func (c *TaskHubGrpcClient) QueryEntities(ctx context.Context, query api.EntityQuery) (*api.EntityQueryResults, error) { + if query.PageSize < 0 { + return nil, api.WrapInvalidArgument(errors.New("entity query page size must not be negative")) + } + if err := api.ValidateTimeRange(query.LastModifiedFrom, query.LastModifiedTo); err != nil { + return nil, err + } + protoQuery := &protos.EntityQuery{ + IncludeState: !query.ExcludeState, + IncludeTransient: query.IncludeTransient, + } + if query.InstanceIDStartsWith != "" { + protoQuery.InstanceIdStartsWith = wrapperspb.String(normalizeEntityQueryPrefix(query.InstanceIDStartsWith)) + } + if !query.LastModifiedFrom.IsZero() { + protoQuery.LastModifiedFrom = timestamppb.New(query.LastModifiedFrom) + } + if !query.LastModifiedTo.IsZero() { + protoQuery.LastModifiedTo = timestamppb.New(query.LastModifiedTo) + } + if query.PageSize > 0 { + protoQuery.PageSize = wrapperspb.Int32(query.PageSize) + } + if query.ContinuationToken != "" { + protoQuery.ContinuationToken = wrapperspb.String(query.ContinuationToken) + } + response, err := c.client.QueryEntities(ctx, &protos.QueryEntitiesRequest{Query: protoQuery}) + if err != nil { + return nil, clientRPCError(ctx, "failed to query entities", err) + } + result := &api.EntityQueryResults{ + Entities: make([]*api.EntityMetadata, 0, len(response.Entities)), + ContinuationToken: response.ContinuationToken.GetValue(), + } + for _, entity := range response.Entities { + metadata, err := entityMetadataFromProto( + ctx, + c.largePayloads, + c.converter, + entity, + !query.ExcludeState, + ) + if err != nil { + return nil, err + } + result.Entities = append(result.Entities, metadata) + } + return result, nil +} + +// CleanEntityStorage removes empty entities and releases orphaned locks. +func (c *TaskHubGrpcClient) CleanEntityStorage( + ctx context.Context, + options ...api.CleanEntityStorageOptions, +) (*api.CleanEntityStorageResult, error) { + if len(options) > 1 { + return nil, api.WrapInvalidArgument(errors.New("at most one entity cleanup options value may be supplied")) + } + var option api.CleanEntityStorageOptions + if len(options) == 1 { + option = options[0] + } + result := new(api.CleanEntityStorageResult) + continuationToken := option.ContinuationToken + seenTokens := make(map[string]struct{}) + if continuationToken != "" { + seenTokens[continuationToken] = struct{}{} + } + for { + request := &protos.CleanEntityStorageRequest{ + RemoveEmptyEntities: !option.PreserveEmptyEntities, + ReleaseOrphanedLocks: !option.PreserveOrphanedLocks, + } + if continuationToken != "" { + request.ContinuationToken = wrapperspb.String(continuationToken) + } + response, err := c.client.CleanEntityStorage(ctx, request) + if err != nil { + return nil, clientRPCError(ctx, "failed to clean entity storage", err) + } + result.EmptyEntitiesRemoved += response.EmptyEntitiesRemoved + result.OrphanedLocksReleased += response.OrphanedLocksReleased + result.ContinuationToken = response.ContinuationToken.GetValue() + if option.SinglePage || result.ContinuationToken == "" { + return result, nil + } + if _, duplicate := seenTokens[result.ContinuationToken]; duplicate { + return nil, fmt.Errorf("entity cleanup returned repeated continuation token %q", result.ContinuationToken) + } + seenTokens[result.ContinuationToken] = struct{}{} + continuationToken = result.ContinuationToken + } +} + +func entityMetadataFromProto( + ctx context.Context, + options *api.LargePayloadOptions, + converter api.DataConverter, + entity *protos.EntityMetadata, + includeState bool, +) (*api.EntityMetadata, error) { + if entity == nil { + return nil, fmt.Errorf("entity metadata must not be nil") + } + entityID, err := api.EntityIDFromString(entity.InstanceId) + if err != nil { + return nil, fmt.Errorf("invalid entity metadata instance ID %q: %w", entity.InstanceId, err) + } + metadata := &api.EntityMetadata{ + InstanceID: entityID, + BacklogQueueSize: entity.BacklogQueueSize, + LockedBy: entity.LockedBy.GetValue(), + StateIncluded: includeState, + HasState: includeState && entity.SerializedState != nil, + Converter: converter, + } + if entity.LastModifiedTime != nil { + metadata.LastModifiedTime = entity.LastModifiedTime.AsTime() + } + if !metadata.HasState { + return metadata, nil + } + state, err := largepayload.Hydrate(ctx, options, entity.SerializedState) + if err != nil { + return nil, fmt.Errorf("failed to hydrate entity state: %w", err) + } + metadata.SerializedState = state.GetValue() + return metadata, nil +} + +func normalizeEntityQueryPrefix(value string) string { + value = strings.TrimPrefix(value, "@") + name, key, hasKey := strings.Cut(value, "@") + prefix := "@" + helpers.ToLowerInvariant(name) + if hasKey { + return prefix + "@" + key + } + return prefix +} + func makeGetInstanceRequest(id api.InstanceID, opts []api.FetchOrchestrationMetadataOptions) *protos.GetInstanceRequest { req := &protos.GetInstanceRequest{ InstanceId: string(id), @@ -230,26 +500,54 @@ func makeGetInstanceRequest(id api.InstanceID, opts []api.FetchOrchestrationMeta // makeOrchestrationMetadata validates and converts protos.GetInstanceResponse to api.OrchestrationMetadata // api.ErrInstanceNotFound is returned when the specified orchestration doesn't exist. -func makeOrchestrationMetadata(resp *protos.GetInstanceResponse) (*api.OrchestrationMetadata, error) { +func (c *TaskHubGrpcClient) makeOrchestrationMetadata( + ctx context.Context, + resp *protos.GetInstanceResponse, +) (*api.OrchestrationMetadata, error) { if !resp.Exists { return nil, api.ErrInstanceNotFound } if resp.OrchestrationState == nil { return nil, fmt.Errorf("orchestration state is nil") } + if err := largepayload.TransformOrchestrationState(ctx, c.largePayloads, resp.OrchestrationState); err != nil { + return nil, fmt.Errorf("failed to hydrate orchestration metadata: %w", err) + } + return orchestrationMetadataFromState(resp.OrchestrationState, c.converter) +} + +func orchestrationMetadataFromState( + state *protos.OrchestrationState, + converter api.DataConverter, +) (*api.OrchestrationMetadata, error) { + if state == nil { + return nil, errors.New("orchestration state is nil") + } metadata := &api.OrchestrationMetadata{ - InstanceID: api.InstanceID(resp.OrchestrationState.InstanceId), - Name: resp.OrchestrationState.Name, - RuntimeStatus: resp.OrchestrationState.OrchestrationStatus, - SerializedInput: resp.OrchestrationState.Input.GetValue(), - SerializedCustomStatus: resp.OrchestrationState.CustomStatus.GetValue(), - SerializedOutput: resp.OrchestrationState.Output.GetValue(), - } - if resp.OrchestrationState.CreatedTimestamp != nil { - metadata.CreatedAt = resp.OrchestrationState.CreatedTimestamp.AsTime() - } - if resp.OrchestrationState.LastUpdatedTimestamp != nil { - metadata.LastUpdatedAt = resp.OrchestrationState.LastUpdatedTimestamp.AsTime() + InstanceID: api.InstanceID(state.InstanceId), + Name: state.Name, + Version: state.Version.GetValue(), + ExecutionID: state.ExecutionId.GetValue(), + ParentInstanceID: api.InstanceID(state.ParentInstanceId.GetValue()), + RuntimeStatus: state.OrchestrationStatus, + SerializedInput: state.Input.GetValue(), + SerializedCustomStatus: state.CustomStatus.GetValue(), + SerializedOutput: state.Output.GetValue(), + FailureDetails: failure.FromProto(state.FailureDetails), + Tags: tagcodec.DecodeUserTagsOrPlain(state.Tags), + Converter: converter, + } + if state.ScheduledStartTimestamp != nil { + metadata.ScheduledStartAt = state.ScheduledStartTimestamp.AsTime() + } + if state.CreatedTimestamp != nil { + metadata.CreatedAt = state.CreatedTimestamp.AsTime() + } + if state.LastUpdatedTimestamp != nil { + metadata.LastUpdatedAt = state.LastUpdatedTimestamp.AsTime() + } + if state.CompletedTimestamp != nil { + metadata.CompletedAt = state.CompletedTimestamp.AsTime() } return metadata, nil } diff --git a/client/client_grpc_history_test.go b/client/client_grpc_history_test.go new file mode 100644 index 00000000..8ad1a6c2 --- /dev/null +++ b/client/client_grpc_history_test.go @@ -0,0 +1,289 @@ +package client + +import ( + "context" + "errors" + "io" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type historySchedulerClient struct { + protos.TaskHubSidecarServiceClient + request *protos.StreamInstanceHistoryRequest + stream protos.TaskHubSidecarService_StreamInstanceHistoryClient + err error +} + +func (c *historySchedulerClient) StreamInstanceHistory( + _ context.Context, + request *protos.StreamInstanceHistoryRequest, + _ ...grpc.CallOption, +) (protos.TaskHubSidecarService_StreamInstanceHistoryClient, error) { + c.request = request + return c.stream, c.err +} + +type historyClientStream struct { + protos.TaskHubSidecarService_StreamInstanceHistoryClient + chunks []*protos.HistoryChunk + err error + index int +} + +func (s *historyClientStream) Recv() (*protos.HistoryChunk, error) { + if s.index < len(s.chunks) { + chunk := s.chunks[s.index] + s.index++ + return chunk, nil + } + if s.err != nil { + return nil, s.err + } + return nil, io.EOF +} + +func TestTaskHubGrpcClientStreamsHistoryInOrder(t *testing.T) { + scheduler := &historySchedulerClient{stream: &historyClientStream{ + chunks: []*protos.HistoryChunk{ + {Events: []*protos.HistoryEvent{historyGenericEvent(1, `"one"`)}}, + {}, + {Events: []*protos.HistoryEvent{historyGenericEvent(2, `"two"`)}}, + }, + }} + client := &TaskHubGrpcClient{ + client: scheduler, + logger: api.DefaultLogger(), + converter: api.DefaultDataConverter(), + } + var values []string + err := client.StreamOrchestrationHistory( + context.Background(), + "instance", + api.HistoryQuery{ExecutionID: "execution", MaxEvents: 1}, + func(event *api.HistoryEvent) error { + var value string + require.NoError(t, event.ReadData(&value)) + values = append(values, value) + return nil + }, + ) + require.NoError(t, err) + require.Equal(t, []string{"one", "two"}, values) + require.Equal(t, "instance", scheduler.request.InstanceId) + require.Equal(t, "execution", scheduler.request.ExecutionId.GetValue()) + require.False(t, scheduler.request.ForWorkItemProcessing) +} + +func TestTaskHubGrpcClientHistoryLimitAndErrors(t *testing.T) { + tests := []struct { + name string + scheduler *historySchedulerClient + query api.HistoryQuery + expected error + }{ + { + name: "limit", + scheduler: &historySchedulerClient{stream: &historyClientStream{chunks: []*protos.HistoryChunk{ + {Events: []*protos.HistoryEvent{historyGenericEvent(1, "one"), historyGenericEvent(2, "two")}}, + }}}, + query: api.HistoryQuery{MaxEvents: 1}, + expected: api.ErrHistoryLimitExceeded, + }, + { + name: "byte limit", + scheduler: &historySchedulerClient{stream: &historyClientStream{chunks: []*protos.HistoryChunk{ + {Events: []*protos.HistoryEvent{historyGenericEvent(1, "payload")}}, + }}}, + query: api.HistoryQuery{MaxBytes: 1}, + expected: api.ErrHistoryLimitExceeded, + }, + { + name: "not found", + scheduler: &historySchedulerClient{err: status.Error(codes.NotFound, "missing")}, + expected: api.ErrInstanceNotFound, + }, + { + name: "unimplemented", + scheduler: &historySchedulerClient{err: status.Error(codes.Unimplemented, "unsupported")}, + expected: api.ErrFeatureNotSupported, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := &TaskHubGrpcClient{ + client: test.scheduler, + logger: api.DefaultLogger(), + converter: api.DefaultDataConverter(), + } + _, err := client.GetOrchestrationHistory(context.Background(), "instance", test.query) + require.ErrorIs(t, err, test.expected) + }) + } +} + +func TestTaskHubGrpcClientHistoryValidationAndCallbackError(t *testing.T) { + client := &TaskHubGrpcClient{ + client: &historySchedulerClient{}, + logger: api.DefaultLogger(), + converter: api.DefaultDataConverter(), + } + err := client.StreamOrchestrationHistory(context.Background(), "", api.HistoryQuery{}, func(*api.HistoryEvent) error { + return nil + }) + require.ErrorIs(t, err, api.ErrInvalidArgument) + + err = client.StreamOrchestrationHistory(context.Background(), "instance", api.HistoryQuery{}, nil) + require.ErrorIs(t, err, api.ErrInvalidArgument) + + callbackErr := errors.New("stop") + scheduler := &historySchedulerClient{stream: &historyClientStream{ + chunks: []*protos.HistoryChunk{{Events: []*protos.HistoryEvent{historyGenericEvent(1, "one")}}}, + }} + client.client = scheduler + err = client.StreamOrchestrationHistory( + context.Background(), + "instance", + api.HistoryQuery{}, + func(*api.HistoryEvent) error { return callbackErr }, + ) + require.ErrorIs(t, err, callbackErr) +} + +func TestTaskHubGrpcClientHistoryMapsCanceledReceive(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + scheduler := &historySchedulerClient{stream: &historyClientStream{ + err: status.Error(codes.Canceled, "canceled"), + }} + client := &TaskHubGrpcClient{ + client: scheduler, + logger: api.DefaultLogger(), + converter: api.DefaultDataConverter(), + } + err := client.StreamOrchestrationHistory(ctx, "instance", api.HistoryQuery{}, func(*api.HistoryEvent) error { + return nil + }) + require.ErrorIs(t, err, context.Canceled) +} + +func TestTaskHubGrpcClientHistoryExecutionIdentity(t *testing.T) { + for _, test := range []struct { + name string + requested string + events []*protos.HistoryEvent + observed string + wantError bool + }{ + {"pinned match", "A", []*protos.HistoryEvent{historyStartedEvent("A")}, "A", false}, + {"pinned mismatch", "A", []*protos.HistoryEvent{historyStartedEvent("B")}, "", true}, + {"pinned mixed chunks", "A", []*protos.HistoryEvent{historyStartedEvent("A"), historyStartedEvent("B")}, "", true}, + {"unpinned mixed chunks", "", []*protos.HistoryEvent{historyStartedEvent("A"), historyStartedEvent("B")}, "", true}, + {"pinned repeated match", "A", []*protos.HistoryEvent{historyStartedEvent("A"), historyStartedEvent("A")}, "A", false}, + {"pinned empty", "A", nil, "", true}, + {"pinned generic", "A", []*protos.HistoryEvent{historyGenericEvent(1, "value")}, "", true}, + {"pinned missing ID", "A", []*protos.HistoryEvent{historyStartedEvent("")}, "", true}, + {"pinned missing instance", "A", []*protos.HistoryEvent{{EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{}, + }}}, "", true}, + {"unpinned missing ID", "", []*protos.HistoryEvent{historyStartedEvent("")}, "", true}, + {"unpinned empty", "", nil, "", false}, + {"unpinned generic", "", []*protos.HistoryEvent{historyGenericEvent(1, "value")}, "", false}, + {"unpinned observed", "", []*protos.HistoryEvent{historyStartedEvent("A")}, "A", false}, + } { + t.Run(test.name, func(t *testing.T) { + stream := &historyClientStream{} + for _, event := range test.events { + stream.chunks = append(stream.chunks, &protos.HistoryChunk{Events: []*protos.HistoryEvent{event}}) + } + scheduler := &historySchedulerClient{stream: stream} + client := &TaskHubGrpcClient{client: scheduler, logger: api.DefaultLogger(), converter: api.DefaultDataConverter()} + result, err := client.GetOrchestrationHistory( + context.Background(), "instance", api.HistoryQuery{ExecutionID: test.requested}) + if test.wantError { + require.ErrorContains(t, err, "execution") + require.Nil(t, result) + } else { + require.NoError(t, err) + require.Equal(t, test.observed, result.ExecutionID) + require.Len(t, result.Events, len(test.events)) + } + require.Equal(t, test.requested, scheduler.request.GetExecutionId().GetValue()) + }) + } +} + +func TestTaskHubGrpcClientHistoryRetainedContentByteLimit(t *testing.T) { + large := strings.Repeat("x", 4096) + details := &protos.TaskFailureDetails{InnerFailure: &protos.TaskFailureDetails{ + Properties: map[string]*structpb.Value{"nested": structpb.NewListValue(&structpb.ListValue{ + Values: []*structpb.Value{structpb.NewStructValue(&structpb.Struct{ + Fields: map[string]*structpb.Value{"value": structpb.NewStringValue(large)}, + })}, + })}, + }} + for name, event := range map[string]*protos.HistoryEvent{ + "execution failure": {EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{FailureDetails: details}, + }}, + "task failure": {EventType: &protos.HistoryEvent_TaskFailed{ + TaskFailed: &protos.TaskFailedEvent{FailureDetails: details}, + }}, + "sub-orchestration failure": {EventType: &protos.HistoryEvent_SubOrchestrationInstanceFailed{ + SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{FailureDetails: details}, + }}, + "entity failure": {EventType: &protos.HistoryEvent_EntityOperationFailed{ + EntityOperationFailed: &protos.EntityOperationFailedEvent{FailureDetails: details}, + }}, + "history state failure": {EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{OrchestrationState: &protos.OrchestrationState{FailureDetails: details}}, + }}, + "history state input": {EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{OrchestrationState: &protos.OrchestrationState{Input: wrapperspb.String(large)}}, + }}, + "history state output": {EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{OrchestrationState: &protos.OrchestrationState{Output: wrapperspb.String(large)}}, + }}, + "history state status": {EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{OrchestrationState: &protos.OrchestrationState{CustomStatus: wrapperspb.String(large)}}, + }}, + } { + t.Run(name, func(t *testing.T) { + scheduler := &historySchedulerClient{stream: &historyClientStream{ + chunks: []*protos.HistoryChunk{{Events: []*protos.HistoryEvent{event}}}, + }} + client := &TaskHubGrpcClient{client: scheduler, logger: api.DefaultLogger(), converter: api.DefaultDataConverter()} + result, err := client.GetOrchestrationHistory(context.Background(), "instance", api.HistoryQuery{MaxBytes: 1024}) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + }) + } +} + +func historyStartedEvent(executionID string) *protos.HistoryEvent { + instance := &protos.OrchestrationInstance{InstanceId: "instance"} + if executionID != "" { + instance.ExecutionId = wrapperspb.String(executionID) + } + return &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{OrchestrationInstance: instance}, + }} +} + +func historyGenericEvent(id int32, value string) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: id, + EventType: &protos.HistoryEvent_GenericEvent{ + GenericEvent: &protos.GenericEvent{Data: wrapperspb.String(value)}, + }, + } +} diff --git a/client/client_grpc_large_payload_test.go b/client/client_grpc_large_payload_test.go new file mode 100644 index 00000000..052809c0 --- /dev/null +++ b/client/client_grpc_large_payload_test.go @@ -0,0 +1,209 @@ +package client + +import ( + "context" + "encoding/base64" + "fmt" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type largePayloadSchedulerClient struct { + protos.TaskHubSidecarServiceClient + start *protos.CreateInstanceRequest + event *protos.RaiseEventRequest + terminate *protos.TerminateRequest + signal *protos.SignalEntityRequest + state *protos.OrchestrationState +} + +func (c *largePayloadSchedulerClient) StartInstance( + _ context.Context, + req *protos.CreateInstanceRequest, + _ ...grpc.CallOption, +) (*protos.CreateInstanceResponse, error) { + c.start = req + return &protos.CreateInstanceResponse{InstanceId: req.InstanceId}, nil +} + +func (c *largePayloadSchedulerClient) RaiseEvent( + _ context.Context, + req *protos.RaiseEventRequest, + _ ...grpc.CallOption, +) (*protos.RaiseEventResponse, error) { + c.event = req + return &protos.RaiseEventResponse{}, nil +} + +func (c *largePayloadSchedulerClient) TerminateInstance( + _ context.Context, + req *protos.TerminateRequest, + _ ...grpc.CallOption, +) (*protos.TerminateResponse, error) { + c.terminate = req + return &protos.TerminateResponse{}, nil +} + +func (c *largePayloadSchedulerClient) SignalEntity( + _ context.Context, + req *protos.SignalEntityRequest, + _ ...grpc.CallOption, +) (*protos.SignalEntityResponse, error) { + c.signal = req + return &protos.SignalEntityResponse{}, nil +} + +func (c *largePayloadSchedulerClient) GetInstance( + context.Context, + *protos.GetInstanceRequest, + ...grpc.CallOption, +) (*protos.GetInstanceResponse, error) { + return &protos.GetInstanceResponse{Exists: true, OrchestrationState: c.state}, nil +} + +type textDataConverter struct{} + +func (textDataConverter) Serialize(value any) (string, error) { + return "text:" + base64.RawStdEncoding.EncodeToString([]byte(fmt.Sprint(value))), nil +} + +func (textDataConverter) Deserialize(payload string, target any) error { + if len(payload) < len("text:") || payload[:len("text:")] != "text:" { + return fmt.Errorf("unexpected payload %q", payload) + } + data, err := base64.RawStdEncoding.DecodeString(payload[len("text:"):]) + if err != nil { + return err + } + value, ok := target.(*string) + if !ok { + return fmt.Errorf("unsupported target %T", target) + } + *value = string(data) + return nil +} + +func TestTaskHubGrpcClientLargePayloadManagementFields(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + fake := &largePayloadSchedulerClient{} + client := &TaskHubGrpcClient{ + client: fake, + logger: api.DefaultLogger(), + largePayloads: options, + } + ctx := context.Background() + id, err := client.ScheduleNewOrchestration(ctx, "orchestrator", api.WithRawInput("create-payload")) + require.NoError(t, err) + require.NotEmpty(t, id) + requireLargePayloadValue(t, options, fake.start.Input, "create-payload") + + require.NoError(t, client.RaiseEvent(ctx, id, "event", api.WithRawEventData("event-payload"))) + requireLargePayloadValue(t, options, fake.event.Input, "event-payload") + + require.NoError(t, client.TerminateOrchestration(ctx, id, api.WithRawOutput("terminate-payload"))) + requireLargePayloadValue(t, options, fake.terminate.Output, "terminate-payload") + + input, err := largepayload.Externalize(ctx, options, wrapperspb.String("metadata-input")) + require.NoError(t, err) + output, err := largepayload.Externalize(ctx, options, wrapperspb.String("metadata-output")) + require.NoError(t, err) + status, err := largepayload.Externalize(ctx, options, wrapperspb.String("metadata-status")) + require.NoError(t, err) + fake.state = &protos.OrchestrationState{ + InstanceId: string(id), + Name: "orchestrator", + CreatedTimestamp: timestamppb.Now(), + LastUpdatedTimestamp: timestamppb.Now(), + Input: input, + Output: output, + CustomStatus: status, + } + metadata, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, "metadata-input", metadata.SerializedInput) + require.Equal(t, "metadata-output", metadata.SerializedOutput) + require.Equal(t, "metadata-status", metadata.SerializedCustomStatus) +} + +func TestTaskHubGrpcClientUsesConverterBeforeLargePayloadExternalization(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + fake := &largePayloadSchedulerClient{} + converter := textDataConverter{} + client := &TaskHubGrpcClient{ + client: fake, + logger: api.DefaultLogger(), + largePayloads: options, + converter: converter, + } + ctx := context.Background() + + id, err := client.ScheduleNewOrchestration(ctx, "orchestrator", api.WithInput("start")) + require.NoError(t, err) + requireLargePayloadValue(t, options, fake.start.Input, mustSerialize(t, converter, "start")) + + require.NoError(t, client.RaiseEvent(ctx, id, "event", api.WithEventPayload("event"))) + requireLargePayloadValue(t, options, fake.event.Input, mustSerialize(t, converter, "event")) + + require.NoError(t, client.TerminateOrchestration(ctx, id, api.WithOutput("output"))) + requireLargePayloadValue(t, options, fake.terminate.Output, mustSerialize(t, converter, "output")) + + entityID := api.NewEntityID("counter", "one") + require.NoError(t, client.SignalEntity(ctx, entityID, "add", api.WithSignalInput("signal"))) + requireLargePayloadValue(t, options, fake.signal.Input, mustSerialize(t, converter, "signal")) + + serializedInput := mustSerialize(t, converter, "metadata") + externalizedInput, err := largepayload.Externalize(ctx, options, wrapperspb.String(serializedInput)) + require.NoError(t, err) + fake.state = &protos.OrchestrationState{ + InstanceId: string(id), + Name: "orchestrator", + CreatedTimestamp: timestamppb.Now(), + LastUpdatedTimestamp: timestamppb.Now(), + Input: externalizedInput, + } + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + require.NoError(t, err) + var decoded string + require.NoError(t, metadata.ReadInput(&decoded)) + require.Equal(t, "metadata", decoded) +} + +func mustSerialize(t *testing.T, converter api.DataConverter, value any) string { + t.Helper() + payload, err := converter.Serialize(value) + require.NoError(t, err) + return payload +} + +func requireLargePayloadValue( + t *testing.T, + options *api.LargePayloadOptions, + value *wrapperspb.StringValue, + expected string, +) { + t.Helper() + require.NotEqual(t, expected, value.GetValue()) + hydrated, err := largepayload.Hydrate(context.Background(), options, value) + require.NoError(t, err) + require.Equal(t, expected, hydrated.GetValue()) +} diff --git a/client/client_grpc_management.go b/client/client_grpc_management.go new file mode 100644 index 00000000..a566fc66 --- /dev/null +++ b/client/client_grpc_management.go @@ -0,0 +1,384 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io" + "slices" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/historyconv" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func (c *TaskHubGrpcClient) GetOrchestrationHistory( + ctx context.Context, + id api.InstanceID, + query api.HistoryQuery, +) (*api.OrchestrationHistory, error) { + return historyconv.Collect(id, query, func(handler api.HistoryEventHandler) error { + return c.StreamOrchestrationHistory(ctx, id, query, handler) + }) +} + +func (c *TaskHubGrpcClient) StreamOrchestrationHistory( + ctx context.Context, + id api.InstanceID, + query api.HistoryQuery, + handler api.HistoryEventHandler, +) error { + normalized, err := historyconv.NormalizeStreamRequest(id, query, handler) + if err != nil { + return err + } + request := &protos.StreamInstanceHistoryRequest{ + InstanceId: string(id), + ForWorkItemProcessing: false, + } + if normalized.ExecutionID != "" { + request.ExecutionId = wrapperspb.String(normalized.ExecutionID) + } + + streamContext, cancel := context.WithCancel(ctx) + defer cancel() + stream, err := c.client.StreamInstanceHistory(streamContext, request) + if err != nil { + return clientRPCError(ctx, "failed to stream orchestration history", err) + } + + converter := historyconv.New(c.converter) + eventCount := 0 + for { + chunk, recvErr := stream.Recv() + if errors.Is(recvErr, io.EOF) { + return nil + } + if recvErr != nil { + return clientRPCError(ctx, "failed to stream orchestration history", recvErr) + } + if chunk == nil { + return errors.New("failed to stream orchestration history: received a nil history chunk") + } + for _, event := range chunk.GetEvents() { + if err := largepayload.TransformHistoryEvent(streamContext, c.largePayloads, event, false); err != nil { + return fmt.Errorf("failed to hydrate orchestration history event %d: %w", eventCount, err) + } + converted, err := converter.Convert(event) + if err != nil { + return fmt.Errorf("failed to convert orchestration history event %d: %w", eventCount, err) + } + if err := handler(converted); err != nil { + return err + } + eventCount++ + } + } +} + +func (c *TaskHubGrpcClient) QueryInstances(ctx context.Context, query api.OrchestrationQuery) (*api.OrchestrationQueryResult, error) { + pageSize, err := api.NormalizeInstanceQueryPageSize(query.PageSize) + if err != nil { + return nil, err + } + if err := api.ValidateTimeRange(query.CreatedTimeFrom, query.CreatedTimeTo); err != nil { + return nil, fmt.Errorf("invalid orchestration query: %w", err) + } + + result := &api.OrchestrationQueryResult{ + Orchestrations: make([]*api.OrchestrationMetadata, 0, pageSize), + } + continuationToken := query.ContinuationToken + scannedPages := 0 + for len(result.Orchestrations) < pageSize { + remaining := pageSize - len(result.Orchestrations) + wireQuery := &protos.InstanceQuery{ + RuntimeStatus: slices.Clone(query.RuntimeStatus), + MaxInstanceCount: int32(remaining), + InstanceIdPrefix: stringValue(query.InstanceIDPrefix), + ContinuationToken: stringValue(continuationToken), + FetchInputsAndOutputs: query.FetchInputsAndOutputs, + } + if !query.CreatedTimeFrom.IsZero() { + wireQuery.CreatedTimeFrom = timestamppb.New(query.CreatedTimeFrom) + } + if !query.CreatedTimeTo.IsZero() { + wireQuery.CreatedTimeTo = timestamppb.New(query.CreatedTimeTo) + } + if len(query.TaskHubNames) > 0 { + wireQuery.TaskHubNames = make([]*wrapperspb.StringValue, 0, len(query.TaskHubNames)) + for _, taskHubName := range query.TaskHubNames { + wireQuery.TaskHubNames = append(wireQuery.TaskHubNames, wrapperspb.String(taskHubName)) + } + } + + resp, err := c.client.QueryInstances(ctx, &protos.QueryInstancesRequest{Query: wireQuery}) + if err != nil { + return nil, clientRPCError(ctx, "failed to query orchestration instances", err) + } + if resp == nil { + return nil, errors.New("query service returned a nil response") + } + scannedPages++ + + for _, state := range resp.GetOrchestrationState() { + if state == nil { + return nil, errors.New("orchestration state is nil") + } + if !matchesTags(tagcodec.DecodeUserTagsOrPlain(state.Tags), query.Tags) { + continue + } + if err := largepayload.TransformOrchestrationState(ctx, c.largePayloads, state); err != nil { + return nil, fmt.Errorf("failed to hydrate orchestration query result: %w", err) + } + metadata, err := orchestrationMetadataFromState(state, c.converter) + if err != nil { + return nil, err + } + result.Orchestrations = append(result.Orchestrations, metadata) + } + + nextToken := resp.GetContinuationToken().GetValue() + if nextToken == "" { + return result, nil + } + if nextToken == continuationToken { + return nil, errors.New("query service returned a non-advancing continuation token") + } + continuationToken = nextToken + if len(query.Tags) > 0 && scannedPages >= api.MaxRemoteTagFilterScanPages { + result.ContinuationToken = continuationToken + return result, nil + } + } + result.ContinuationToken = continuationToken + return result, nil +} + +func (c *TaskHubGrpcClient) ListInstanceIDs(ctx context.Context, query api.InstanceIDQuery) (*api.InstanceIDQueryResult, error) { + pageSize, err := api.NormalizeInstanceQueryPageSize(query.PageSize) + if err != nil { + return nil, err + } + if err := api.ValidateTimeRange(query.CompletedTimeFrom, query.CompletedTimeTo); err != nil { + return nil, fmt.Errorf("invalid instance ID query: %w", err) + } + req := &protos.ListInstanceIdsRequest{ + RuntimeStatus: slices.Clone(query.RuntimeStatus), + PageSize: int32(pageSize), + LastInstanceKey: stringValue(query.ContinuationToken), + } + if !query.CompletedTimeFrom.IsZero() { + req.CompletedTimeFrom = timestamppb.New(query.CompletedTimeFrom) + } + if !query.CompletedTimeTo.IsZero() { + req.CompletedTimeTo = timestamppb.New(query.CompletedTimeTo) + } + + resp, err := c.client.ListInstanceIds(ctx, req) + if err != nil { + return nil, clientRPCError(ctx, "failed to list orchestration instance IDs", err) + } + result := &api.InstanceIDQueryResult{ + InstanceIDs: make([]api.InstanceID, 0, len(resp.GetInstanceIds())), + ContinuationToken: resp.GetLastInstanceKey().GetValue(), + } + for _, id := range resp.GetInstanceIds() { + result.InstanceIDs = append(result.InstanceIDs, api.InstanceID(id)) + } + return result, nil +} + +func (c *TaskHubGrpcClient) RestartInstance(ctx context.Context, id api.InstanceID, opts ...api.RestartOptions) (api.InstanceID, error) { + req := &protos.RestartInstanceRequest{InstanceId: string(id)} + for _, configure := range opts { + if err := configure(req); err != nil { + return api.EmptyInstanceID, fmt.Errorf("failed to configure restart request: %w", api.WrapInvalidArgument(err)) + } + } + resp, err := c.client.RestartInstance(ctx, req) + if err != nil { + return api.EmptyInstanceID, clientRPCError(ctx, "failed to restart orchestration instance", err) + } + return api.InstanceID(resp.GetInstanceId()), nil +} + +func (c *TaskHubGrpcClient) RewindInstance(ctx context.Context, id api.InstanceID, opts ...api.RewindOptions) error { + req := &protos.RewindInstanceRequest{InstanceId: string(id)} + for _, configure := range opts { + if err := configure(req); err != nil { + return fmt.Errorf("failed to configure rewind request: %w", api.WrapInvalidArgument(err)) + } + } + if _, err := c.client.RewindInstance(ctx, req); err != nil { + return clientRPCError(ctx, "failed to rewind orchestration instance", err) + } + return nil +} + +func (c *TaskHubGrpcClient) PurgeInstances(ctx context.Context, request api.PurgeInstancesRequest) (*api.PurgeInstancesResult, error) { + if err := request.Validate(); err != nil { + return nil, err + } + if request.Filter != nil && request.Filter.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, request.Filter.Timeout) + defer cancel() + } + pollInterval := request.PollInterval + if pollInterval <= 0 { + pollInterval = api.DefaultPurgePollInterval + } + result := &api.PurgeInstancesResult{IsComplete: true} + if request.Filter != nil { + req, err := makePurgeFilterRequest(request) + if err != nil { + return nil, err + } + return c.pollPurgeInstances(ctx, req, pollInterval) + } + + for start := 0; start < len(request.InstanceIDs); start += api.MaxInstanceBatchSize { + end := min(start+api.MaxInstanceBatchSize, len(request.InstanceIDs)) + ids := request.InstanceIDs[start:end] + instanceIDs := make([]string, len(ids)) + for i, id := range ids { + if id == api.EmptyInstanceID { + return nil, api.WrapInvalidArgument(errors.New("purge instance ID cannot be empty")) + } + instanceIDs[i] = string(id) + } + req := &protos.PurgeInstancesRequest{ + Request: &protos.PurgeInstancesRequest_InstanceBatch{ + InstanceBatch: &protos.InstanceBatch{InstanceIds: instanceIDs}, + }, + Recursive: request.Recursive, + IsOrchestration: true, + } + batchResult, err := c.pollPurgeInstances(ctx, req, pollInterval) + if batchResult != nil { + result.DeletedInstanceCount += batchResult.DeletedInstanceCount + result.IsComplete = result.IsComplete && batchResult.IsComplete + } + if err != nil { + return result, err + } + } + return result, nil +} + +func (c *TaskHubGrpcClient) pollPurgeInstances(ctx context.Context, req *protos.PurgeInstancesRequest, pollInterval time.Duration) (*api.PurgeInstancesResult, error) { + result := &api.PurgeInstancesResult{} + for { + resp, err := c.client.PurgeInstances(ctx, req) + if err != nil { + return result, clientRPCError(ctx, "failed to purge orchestration instances", err) + } + result.DeletedInstanceCount += int(resp.GetDeletedInstanceCount()) + if resp.GetIsComplete() == nil || resp.GetIsComplete().GetValue() { + result.IsComplete = true + return result, nil + } + timer := time.NewTimer(pollInterval) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + } + return result, ctx.Err() + case <-timer.C: + } + } +} + +func (c *TaskHubGrpcClient) SkipGracefulOrchestrationTerminations(ctx context.Context, ids []api.InstanceID, reason string) ([]api.InstanceID, error) { + if len(ids) == 0 { + return nil, api.WrapInvalidArgument(errors.New("at least one instance ID is required")) + } + if len(ids) > api.MaxInstanceBatchSize { + return nil, api.WrapInvalidArgument(fmt.Errorf("instance batch cannot exceed %d IDs", api.MaxInstanceBatchSize)) + } + instanceIDs := make([]string, len(ids)) + for i, id := range ids { + if id == api.EmptyInstanceID { + return nil, api.WrapInvalidArgument(errors.New("instance ID cannot be empty")) + } + instanceIDs[i] = string(id) + } + resp, err := c.client.SkipGracefulOrchestrationTerminations(ctx, &protos.SkipGracefulOrchestrationTerminationsRequest{ + InstanceBatch: &protos.InstanceBatch{InstanceIds: instanceIDs}, + Reason: stringValue(reason), + }) + if err != nil { + return nil, clientRPCError(ctx, "failed to skip graceful orchestration terminations", err) + } + unterminated := make([]api.InstanceID, 0, len(resp.GetUnterminatedInstanceIds())) + for _, id := range resp.GetUnterminatedInstanceIds() { + unterminated = append(unterminated, api.InstanceID(id)) + } + return unterminated, nil +} + +func (c *TaskHubGrpcClient) CreateTaskHub(ctx context.Context, opts ...api.CreateTaskHubOptions) error { + req := &protos.CreateTaskHubRequest{} + for _, configure := range opts { + if err := configure(req); err != nil { + return fmt.Errorf("failed to configure task hub creation request: %w", api.WrapInvalidArgument(err)) + } + } + if _, err := c.client.CreateTaskHub(ctx, req); err != nil { + return clientRPCError(ctx, "failed to create task hub", err) + } + return nil +} + +func (c *TaskHubGrpcClient) DeleteTaskHub(ctx context.Context) error { + if _, err := c.client.DeleteTaskHub(ctx, &protos.DeleteTaskHubRequest{}); err != nil { + return clientRPCError(ctx, "failed to delete task hub", err) + } + return nil +} + +func makePurgeFilterRequest(request api.PurgeInstancesRequest) (*protos.PurgeInstancesRequest, error) { + filter := request.Filter + wireFilter := &protos.PurgeInstanceFilter{ + RuntimeStatus: slices.Clone(filter.RuntimeStatus), + } + if !filter.CreatedTimeFrom.IsZero() { + wireFilter.CreatedTimeFrom = timestamppb.New(filter.CreatedTimeFrom) + } + if !filter.CreatedTimeTo.IsZero() { + wireFilter.CreatedTimeTo = timestamppb.New(filter.CreatedTimeTo) + } + if filter.Timeout > 0 { + wireFilter.Timeout = durationpb.New(filter.Timeout) + } + return &protos.PurgeInstancesRequest{ + Request: &protos.PurgeInstancesRequest_PurgeInstanceFilter{PurgeInstanceFilter: wireFilter}, + Recursive: request.Recursive, + IsOrchestration: true, + }, nil +} + +func matchesTags(actual, expected map[string]string) bool { + for key, value := range expected { + actualValue, exists := actual[key] + if !exists || actualValue != value { + return false + } + } + return true +} + +func stringValue(value string) *wrapperspb.StringValue { + if value == "" { + return nil + } + return wrapperspb.String(value) +} diff --git a/client/client_grpc_management_test.go b/client/client_grpc_management_test.go new file mode 100644 index 00000000..e933ce3d --- /dev/null +++ b/client/client_grpc_management_test.go @@ -0,0 +1,164 @@ +package client + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/grpcerrors" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// managementServer is a fake task hub service that answers only the management +// RPCs these tests drive over the wire. It stores nothing: the assertions are +// about the gRPC round trip performed by TaskHubGrpcClient. +type managementServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + mu sync.Mutex + created bool + deleted bool +} + +func (s *managementServer) CreateTaskHub( + context.Context, + *protos.CreateTaskHubRequest, +) (*protos.CreateTaskHubResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.created = true + return &protos.CreateTaskHubResponse{}, nil +} + +func (s *managementServer) DeleteTaskHub( + context.Context, + *protos.DeleteTaskHubRequest, +) (*protos.DeleteTaskHubResponse, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.deleted = true + return &protos.DeleteTaskHubResponse{}, nil +} + +func (*managementServer) QueryInstances( + _ context.Context, + req *protos.QueryInstancesRequest, +) (*protos.QueryInstancesResponse, error) { + if req.GetQuery().GetMaxInstanceCount() <= 0 { + return nil, status.Error(codes.InvalidArgument, "page size must be positive") + } + return &protos.QueryInstancesResponse{}, nil +} + +func (*managementServer) ListInstanceIds( + _ context.Context, + req *protos.ListInstanceIdsRequest, +) (*protos.ListInstanceIdsResponse, error) { + if req.GetPageSize() <= 0 { + return nil, status.Error(codes.InvalidArgument, "page size must be positive") + } + return &protos.ListInstanceIdsResponse{}, nil +} + +func (s *managementServer) lifecycleCalls() (bool, bool) { + s.mu.Lock() + defer s.mu.Unlock() + return s.created, s.deleted +} + +// lifecycleErrorServer fails both lifecycle RPCs with the durable error reasons +// a task hub service attaches, so the client-side sentinel mapping is observed. +type lifecycleErrorServer struct { + protos.UnimplementedTaskHubSidecarServiceServer +} + +func (*lifecycleErrorServer) CreateTaskHub( + context.Context, + *protos.CreateTaskHubRequest, +) (*protos.CreateTaskHubResponse, error) { + return nil, grpcerrors.New( + codes.AlreadyExists, + ErrTaskHubExists.Error(), + grpcerrors.ReasonTaskHubExists, + ) +} + +func (*lifecycleErrorServer) DeleteTaskHub( + context.Context, + *protos.DeleteTaskHubRequest, +) (*protos.DeleteTaskHubResponse, error) { + return nil, grpcerrors.New( + codes.NotFound, + ErrTaskHubNotFound.Error(), + grpcerrors.ReasonTaskHubNotFound, + ) +} + +// bareLifecycleErrorServer omits the durable error reason detail so the +// status-code-only fallback in clientRPCError is exercised too. +type bareLifecycleErrorServer struct { + protos.UnimplementedTaskHubSidecarServiceServer +} + +func (*bareLifecycleErrorServer) CreateTaskHub( + context.Context, + *protos.CreateTaskHubRequest, +) (*protos.CreateTaskHubResponse, error) { + return nil, status.Error(codes.AlreadyExists, "task hub already exists") +} + +func (*bareLifecycleErrorServer) DeleteTaskHub( + context.Context, + *protos.DeleteTaskHubRequest, +) (*protos.DeleteTaskHubResponse, error) { + return nil, status.Error(codes.NotFound, "task hub not found") +} + +func TestTaskHubGrpcManagementOverBufconn(t *testing.T) { + server := &managementServer{} + client := startQueryClient(t, server) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + require.NoError(t, client.CreateTaskHub(ctx)) + query, err := client.QueryInstances(ctx, api.OrchestrationQuery{PageSize: 10}) + require.NoError(t, err) + require.Empty(t, query.Orchestrations) + ids, err := client.ListInstanceIDs(ctx, api.InstanceIDQuery{PageSize: 10}) + require.NoError(t, err) + require.Empty(t, ids.InstanceIDs) + require.NoError(t, client.DeleteTaskHub(ctx)) + + created, deleted := server.lifecycleCalls() + require.True(t, created) + require.True(t, deleted) +} + +func TestTaskHubLifecycleErrorsRoundTripOverGRPC(t *testing.T) { + for _, test := range []struct { + name string + server protos.TaskHubSidecarServiceServer + }{ + {name: "with-durable-error-reason", server: &lifecycleErrorServer{}}, + {name: "status-code-only", server: &bareLifecycleErrorServer{}}, + } { + t.Run(test.name, func(t *testing.T) { + client := startQueryClient(t, test.server) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := client.CreateTaskHub(ctx) + require.True(t, errors.Is(err, ErrTaskHubExists), "CreateTaskHub() error = %v", err) + err = client.DeleteTaskHub(ctx) + require.True(t, errors.Is(err, ErrTaskHubNotFound), "DeleteTaskHub() error = %v", err) + }) + } +} diff --git a/client/client_grpc_tag_query_test.go b/client/client_grpc_tag_query_test.go new file mode 100644 index 00000000..a1361404 --- /dev/null +++ b/client/client_grpc_tag_query_test.go @@ -0,0 +1,379 @@ +package client + +import ( + "context" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" + "github.com/microsoft/durabletask-go/payload" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// pagedQueryServer serves a deterministic, unbounded stream of single-instance +// query pages so the client-side tag filter scan cap can be observed exactly. +type pagedQueryServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + // matchEvery makes every Nth served instance carry the searched tag. Zero + // means no instance ever matches. + matchEvery int + // totalPages bounds the stream. Zero means the stream never ends. + totalPages int + pageTags []map[string]string + pageStates [][]*protos.OrchestrationState + + mu sync.Mutex + requests []string +} + +func (s *pagedQueryServer) QueryInstances( + _ context.Context, + req *protos.QueryInstancesRequest, +) (*protos.QueryInstancesResponse, error) { + token := req.GetQuery().GetContinuationToken().GetValue() + s.mu.Lock() + s.requests = append(s.requests, token) + served := len(s.requests) + s.mu.Unlock() + + tags := map[string]string{"group": "other"} + if s.matchEvery > 0 && served%s.matchEvery == 0 { + tags = map[string]string{"group": "wanted"} + } + if served <= len(s.pageTags) { + tags = s.pageTags[served-1] + } + resp := &protos.QueryInstancesResponse{ + OrchestrationState: []*protos.OrchestrationState{{ + InstanceId: fmt.Sprintf("instance-%04d", served), + Name: "Paged", + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + Tags: tags, + }}, + } + if served <= len(s.pageStates) { + resp.OrchestrationState = s.pageStates[served-1] + } + if s.totalPages == 0 || served < s.totalPages { + resp.ContinuationToken = wrapperspb.String(fmt.Sprintf("token-%04d", served)) + } + return resp, nil +} + +func (s *pagedQueryServer) requestTokens() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.requests...) +} + +func startQueryClient(t *testing.T, server protos.TaskHubSidecarServiceServer) *TaskHubGrpcClient { + t.Helper() + listener := bufconn.Listen(1024 * 1024) + grpcServer := grpc.NewServer() + protos.RegisterTaskHubSidecarServiceServer(grpcServer, server) + go func() { + _ = grpcServer.Serve(listener) + }() + t.Cleanup(grpcServer.Stop) + + connection, err := grpc.NewClient( + "passthrough:///bufnet", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return listener.Dial() + }), + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, connection.Close()) }) + return NewTaskHubGrpcClient(connection, api.DefaultLogger()) +} + +// TestQueryInstancesTagFilterHonoursScanPageCap asserts the documented remote +// tag filter contract: because the wire query has no tag predicate, the client +// scans at most api.MaxRemoteTagFilterScanPages service pages per call and then +// returns a short page plus a continuation token the caller can resume from. +func TestQueryInstancesTagFilterHonoursScanPageCap(t *testing.T) { + for _, test := range []struct { + name string + matchEvery int + pageSize int + wantMatches int + wantServedPages int + wantTokenPresent bool + }{ + { + name: "no-matches-stops-at-cap", + pageSize: 5, + wantMatches: 0, + wantServedPages: api.MaxRemoteTagFilterScanPages, + wantTokenPresent: true, + }, + { + name: "sparse-matches-stop-at-cap", + matchEvery: 50, + pageSize: 5, + wantMatches: 2, + wantServedPages: api.MaxRemoteTagFilterScanPages, + wantTokenPresent: true, + }, + { + name: "dense-matches-fill-the-page-before-the-cap", + matchEvery: 1, + pageSize: 5, + wantMatches: 5, + wantServedPages: 5, + wantTokenPresent: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + server := &pagedQueryServer{matchEvery: test.matchEvery} + client := startQueryClient(t, server) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + PageSize: test.pageSize, + Tags: map[string]string{"group": "wanted"}, + }) + require.NoError(t, err) + require.Len(t, result.Orchestrations, test.wantMatches) + require.Len(t, server.requestTokens(), test.wantServedPages) + require.Equal(t, test.wantTokenPresent, result.ContinuationToken != "") + }) + } +} + +// TestQueryInstancesTagFilterResumesAfterScanPageCap asserts the continuation +// token returned by a capped scan resumes exactly where the previous scan +// stopped, so callers can keep paging without losing or repeating instances. +func TestQueryInstancesTagFilterResumesAfterScanPageCap(t *testing.T) { + server := &pagedQueryServer{} + client := startQueryClient(t, server) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + query := api.OrchestrationQuery{PageSize: 5, Tags: map[string]string{"group": "wanted"}} + first, err := client.QueryInstances(ctx, query) + require.NoError(t, err) + require.Empty(t, first.Orchestrations) + require.Equal(t, fmt.Sprintf("token-%04d", api.MaxRemoteTagFilterScanPages), first.ContinuationToken) + + query.ContinuationToken = first.ContinuationToken + second, err := client.QueryInstances(ctx, query) + require.NoError(t, err) + require.Empty(t, second.Orchestrations) + require.Equal(t, fmt.Sprintf("token-%04d", 2*api.MaxRemoteTagFilterScanPages), second.ContinuationToken) + + tokens := server.requestTokens() + require.Len(t, tokens, 2*api.MaxRemoteTagFilterScanPages) + // The resumed scan starts from the token the capped scan handed back. + require.Equal(t, "", tokens[0]) + require.Equal(t, first.ContinuationToken, tokens[api.MaxRemoteTagFilterScanPages]) +} + +// TestQueryInstancesWithoutTagsIgnoresScanPageCap asserts the scan cap only +// exists to bound client-side tag filtering: an untagged query pages until the +// requested page size is filled. +func TestQueryInstancesWithoutTagsIgnoresScanPageCap(t *testing.T) { + server := &pagedQueryServer{} + client := startQueryClient(t, server) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + pageSize := api.MaxRemoteTagFilterScanPages + 7 + result, err := client.QueryInstances(ctx, api.OrchestrationQuery{PageSize: pageSize}) + require.NoError(t, err) + require.Len(t, result.Orchestrations, pageSize) + require.Len(t, server.requestTokens(), pageSize) + require.NotEmpty(t, result.ContinuationToken) +} + +// TestQueryInstancesTagFilterStopsWhenServiceExhausted asserts a capped tag scan +// that reaches the end of the service results returns no continuation token. +func TestQueryInstancesTagFilterStopsWhenServiceExhausted(t *testing.T) { + server := &pagedQueryServer{matchEvery: 3, totalPages: 10} + client := startQueryClient(t, server) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + result, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + PageSize: 50, + Tags: map[string]string{"group": "wanted"}, + }) + require.NoError(t, err) + require.Len(t, result.Orchestrations, 3) + require.Empty(t, result.ContinuationToken) + require.Len(t, server.requestTokens(), 10) +} + +func TestQueryInstancesEmptyTagRequiresPresenceAcrossPages(t *testing.T) { + server := &pagedQueryServer{ + totalPages: 3, + pageTags: []map[string]string{ + nil, + {"flag": ""}, + {"flag": "set"}, + }, + } + client := startQueryClient(t, server) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + query := api.OrchestrationQuery{PageSize: 1, Tags: map[string]string{"flag": ""}} + first, err := client.QueryInstances(ctx, query) + require.NoError(t, err) + require.Len(t, first.Orchestrations, 1) + require.Equal(t, api.InstanceID("instance-0002"), first.Orchestrations[0].InstanceID) + require.NotEmpty(t, first.ContinuationToken) + + query.ContinuationToken = first.ContinuationToken + second, err := client.QueryInstances(ctx, query) + require.NoError(t, err) + require.Empty(t, second.Orchestrations) + require.Empty(t, second.ContinuationToken) + require.Len(t, server.requestTokens(), 3) +} + +// nonAdvancingQueryServer always echoes the same continuation token, which the +// client must reject instead of looping forever. +type nonAdvancingQueryServer struct { + protos.UnimplementedTaskHubSidecarServiceServer +} + +func (*nonAdvancingQueryServer) QueryInstances( + _ context.Context, + req *protos.QueryInstancesRequest, +) (*protos.QueryInstancesResponse, error) { + token := req.GetQuery().GetContinuationToken().GetValue() + if token == "" { + token = "stuck" + } + return &protos.QueryInstancesResponse{ContinuationToken: wrapperspb.String(token)}, nil +} + +func TestQueryInstancesRejectsNonAdvancingContinuationToken(t *testing.T) { + client := startQueryClient(t, &nonAdvancingQueryServer{}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + PageSize: 5, + ContinuationToken: "stuck", + }) + require.ErrorContains(t, err, "non-advancing continuation token") +} + +type queryPayloadResolver struct { + calls atomic.Int32 + err error +} + +func (r *queryPayloadResolver) Resolve(context.Context, string) ([]byte, error) { + r.calls.Add(1) + return nil, r.err +} + +func TestQueryInstancesFiltersTagsBeforePayloadHydration(t *testing.T) { + for _, tags := range []struct { + name string + rejected, matching map[string]string + query map[string]string + }{ + {"plain", map[string]string{"group": "other"}, map[string]string{"group": "wanted"}, map[string]string{"group": "wanted"}}, + {"encoded", map[string]string{tagcodec.UserTagPrefix + "group": "other"}, map[string]string{tagcodec.UserTagPrefix + "group": "wanted"}, map[string]string{"group": "wanted"}}, + {"context only", map[string]string{tagcodec.ContextFieldPrefix + "group": "wanted"}, map[string]string{"group": "wanted"}, map[string]string{"group": "wanted"}}, + {"empty value", nil, map[string]string{"flag": ""}, map[string]string{"flag": ""}}, + } { + for _, unreadable := range []string{"rejected row", "matching input", "matching output", "matching status"} { + t.Run(tags.name+"/"+unreadable, func(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{Store: store, Resolver: store, ThresholdBytes: 1, MaxPayloadBytes: 1024} + reference, err := largepayload.Externalize(t.Context(), options, wrapperspb.String(`"external"`)) + require.NoError(t, err) + rejected := &protos.OrchestrationState{ + InstanceId: "rejected", Tags: tags.rejected, + Input: reference, Output: reference, CustomStatus: reference, + } + matching := &protos.OrchestrationState{ + InstanceId: "matching", Tags: tags.matching, + Input: wrapperspb.String(`"input"`), Output: wrapperspb.String(`"output"`), + CustomStatus: wrapperspb.String(`"status"`), + } + switch unreadable { + case "matching input": + matching.Input = reference + case "matching output": + matching.Output = reference + case "matching status": + matching.CustomStatus = reference + } + resolver := &queryPayloadResolver{err: errors.New("unreadable query payload")} + options.Resolver = resolver + server := &pagedQueryServer{ + totalPages: 1, + pageStates: [][]*protos.OrchestrationState{{rejected, matching}}, + } + client := startQueryClient(t, server) + client.largePayloads = options + result, err := client.QueryInstances(t.Context(), api.OrchestrationQuery{ + PageSize: 2, Tags: tags.query, FetchInputsAndOutputs: true, + }) + if unreadable != "rejected row" { + require.ErrorIs(t, err, resolver.err) + require.Nil(t, result) + require.EqualValues(t, 1, resolver.calls.Load()) + return + } + require.NoError(t, err) + require.Zero(t, resolver.calls.Load()) + require.Len(t, result.Orchestrations, 1) + require.Equal(t, api.InstanceID("matching"), result.Orchestrations[0].InstanceID) + require.Equal(t, tags.query, result.Orchestrations[0].Tags) + require.Equal(t, `"input"`, result.Orchestrations[0].SerializedInput) + require.Equal(t, `"output"`, result.Orchestrations[0].SerializedOutput) + require.Equal(t, `"status"`, result.Orchestrations[0].SerializedCustomStatus) + require.Empty(t, result.ContinuationToken) + require.Len(t, server.requestTokens(), 1) + }) + } + } +} + +type queryResponseClient struct { + protos.TaskHubSidecarServiceClient + response *protos.QueryInstancesResponse +} + +func (c *queryResponseClient) QueryInstances(context.Context, *protos.QueryInstancesRequest, ...grpc.CallOption) (*protos.QueryInstancesResponse, error) { + return c.response, nil +} + +func TestQueryInstancesRejectsNilResponsesAndRowsBeforeFiltering(t *testing.T) { + for _, test := range []struct { + name string + response *protos.QueryInstancesResponse + message string + }{ + {"response", nil, "nil response"}, + {"row", &protos.QueryInstancesResponse{OrchestrationState: []*protos.OrchestrationState{nil}}, "orchestration state is nil"}, + } { + t.Run(test.name, func(t *testing.T) { + client := &TaskHubGrpcClient{client: &queryResponseClient{response: test.response}} + result, err := client.QueryInstances(t.Context(), api.OrchestrationQuery{Tags: map[string]string{"group": "wanted"}}) + require.ErrorContains(t, err, test.message) + require.Nil(t, result) + }) + } +} diff --git a/client/client_grpc_test.go b/client/client_grpc_test.go new file mode 100644 index 00000000..150f1d5e --- /dev/null +++ b/client/client_grpc_test.go @@ -0,0 +1,84 @@ +package client + +import ( + "context" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +func TestGrpcClientDefaultVersionAndExplicitUnversionedOverride(t *testing.T) { + scheduler := new(largePayloadSchedulerClient) + client := &TaskHubGrpcClient{ + client: scheduler, + defaultVersion: "v2", + } + + _, err := client.ScheduleNewOrchestration(context.Background(), "orchestration") + require.NoError(t, err) + require.Equal(t, "v2", scheduler.start.GetVersion().GetValue()) + + _, err = client.ScheduleNewOrchestration(context.Background(), "orchestration", api.WithVersion("")) + require.NoError(t, err) + require.NotNil(t, scheduler.start.Version) + require.Empty(t, scheduler.start.GetVersion().GetValue()) +} + +func TestOrchestrationIDReusePolicyUsesCurrentDedupeSemantics(t *testing.T) { + tests := []struct { + name string + statuses []api.OrchestrationStatus + wantPolicy bool + replaceable []protos.OrchestrationStatus + }{ + { + name: "nil uses service default", + statuses: nil, + wantPolicy: false, + }, + { + name: "empty allows every reusable status", + statuses: []api.OrchestrationStatus{}, + wantPolicy: true, + replaceable: []protos.OrchestrationStatus{ + protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CANCELED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_PENDING, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, + }, + }, + { + name: "dedupe statuses are removed from replaceable statuses", + statuses: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_PENDING}, + wantPolicy: true, + replaceable: []protos.OrchestrationStatus{ + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CANCELED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := &protos.CreateInstanceRequest{} + configure := api.WithOrchestrationIDReusePolicy(&api.OrchestrationIDReusePolicy{ + DedupeStatuses: tt.statuses, + }) + require.NoError(t, configure(req, api.DefaultDataConverter())) + if tt.wantPolicy { + require.NotNil(t, req.OrchestrationIdReusePolicy) + require.Equal(t, tt.replaceable, req.OrchestrationIdReusePolicy.ReplaceableStatus) + } else { + require.Nil(t, req.OrchestrationIdReusePolicy) + } + }) + } +} diff --git a/client/client_grpc_wait_test.go b/client/client_grpc_wait_test.go new file mode 100644 index 00000000..8f37ea43 --- /dev/null +++ b/client/client_grpc_wait_test.go @@ -0,0 +1,58 @@ +package client + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type missingWaitSchedulerClient struct { + protos.TaskHubSidecarServiceClient + startCalls atomic.Int32 + completionCalls atomic.Int32 +} + +func (c *missingWaitSchedulerClient) WaitForInstanceStart( + context.Context, + *protos.GetInstanceRequest, + ...grpc.CallOption, +) (*protos.GetInstanceResponse, error) { + c.startCalls.Add(1) + return nil, status.Error(codes.NotFound, "missing instance") +} + +func (c *missingWaitSchedulerClient) WaitForInstanceCompletion( + context.Context, + *protos.GetInstanceRequest, + ...grpc.CallOption, +) (*protos.GetInstanceResponse, error) { + c.completionCalls.Add(1) + return nil, status.Error(codes.NotFound, "missing instance") +} + +func TestWaitForMissingInstanceReturnsImmediately(t *testing.T) { + scheduler := new(missingWaitSchedulerClient) + client := &TaskHubGrpcClient{client: scheduler} + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := client.WaitForOrchestrationStart(ctx, "missing") + require.ErrorIs(t, err, api.ErrInstanceNotFound) + require.False(t, errors.Is(err, context.DeadlineExceeded)) + require.EqualValues(t, 1, scheduler.startCalls.Load()) + + _, err = client.WaitForOrchestrationCompletion(ctx, "missing") + require.ErrorIs(t, err, api.ErrInstanceNotFound) + require.False(t, errors.Is(err, context.DeadlineExceeded)) + require.EqualValues(t, 1, scheduler.completionCalls.Load()) +} diff --git a/client/doc.go b/client/doc.go new file mode 100644 index 00000000..5bee72f0 --- /dev/null +++ b/client/doc.go @@ -0,0 +1,3 @@ +// Package client implements the transport-neutral Durable Task gRPC client and +// worker used by the Durable Task Scheduler integration. +package client diff --git a/client/entity_batch.go b/client/entity_batch.go new file mode 100644 index 00000000..5f7a13f0 --- /dev/null +++ b/client/entity_batch.go @@ -0,0 +1,75 @@ +package client + +import ( + "fmt" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/structpb" +) + +// entityBatchFromRequestV2 converts a scheduler-dispatched V2 entity request +// into the executor-facing batch model and response routing metadata. +func entityBatchFromRequestV2(request *protos.EntityRequest) (*protos.EntityBatchRequest, []*protos.OperationInfo, error) { + if request == nil { + return nil, nil, fmt.Errorf("entity request must not be nil") + } + if _, err := api.EntityIDFromString(request.InstanceId); err != nil { + return nil, nil, fmt.Errorf("invalid entity instance ID: %w", err) + } + batch := &protos.EntityBatchRequest{ + InstanceId: request.InstanceId, + EntityState: request.EntityState, + Operations: make([]*protos.OperationRequest, 0, len(request.OperationRequests)), + Properties: make(map[string]*structpb.Value), + } + operationInfos := make([]*protos.OperationInfo, 0, len(request.OperationRequests)) + for _, historyEvent := range request.OperationRequests { + if historyEvent == nil { + return nil, nil, fmt.Errorf("entity operation history event must not be nil") + } + switch { + case historyEvent.GetEntityOperationSignaled() != nil: + event := historyEvent.GetEntityOperationSignaled() + if _, err := uuid.Parse(event.RequestId); err != nil { + return nil, nil, fmt.Errorf("invalid entity signal request ID %q: %w", event.RequestId, err) + } + batch.Operations = append(batch.Operations, &protos.OperationRequest{ + Operation: event.Operation, + RequestId: event.RequestId, + Input: event.Input, + }) + batch.Properties[helpers.EntitySignalProperty(event.RequestId)] = structpb.NewBoolValue(true) + operationInfos = append(operationInfos, &protos.OperationInfo{RequestId: event.RequestId}) + case historyEvent.GetEntityOperationCalled() != nil: + event := historyEvent.GetEntityOperationCalled() + if _, err := uuid.Parse(event.RequestId); err != nil { + return nil, nil, fmt.Errorf("invalid entity call request ID %q: %w", event.RequestId, err) + } + if event.ParentInstanceId.GetValue() == "" { + return nil, nil, fmt.Errorf("entity call %q is missing its response destination", event.RequestId) + } + batch.Operations = append(batch.Operations, &protos.OperationRequest{ + Operation: event.Operation, + RequestId: event.RequestId, + Input: event.Input, + }) + info := &protos.OperationInfo{ + RequestId: event.RequestId, + ResponseDestination: &protos.OrchestrationInstance{ + InstanceId: event.ParentInstanceId.GetValue(), + ExecutionId: event.ParentExecutionId, + }, + } + operationInfos = append(operationInfos, info) + default: + return nil, nil, fmt.Errorf("unsupported entity operation history event") + } + } + if len(batch.Properties) == 0 { + batch.Properties = nil + } + return batch, operationInfos, nil +} diff --git a/client/entity_batch_test.go b/client/entity_batch_test.go new file mode 100644 index 00000000..b2aecdf8 --- /dev/null +++ b/client/entity_batch_test.go @@ -0,0 +1,130 @@ +package client + +import ( + "testing" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestEntityBatchFromRequestV2RejectsMissingParentDestination(t *testing.T) { + requestID := uuid.NewString() + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: requestID, + Operation: "get", + }, + }, + }}, + }) + require.ErrorContains(t, err, "missing its response destination") +} + +func TestEntityBatchFromRequestV2Validation(t *testing.T) { + t.Run("nil-request", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(nil) + require.ErrorContains(t, err, "entity request must not be nil") + }) + + t.Run("invalid-instance-id", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{InstanceId: "not-an-entity"}) + require.ErrorContains(t, err, "invalid entity instance ID") + }) + + t.Run("nil-operation-event", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{nil}, + }) + require.ErrorContains(t, err, "entity operation history event must not be nil") + }) + + t.Run("invalid-signal-request-id", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{ + RequestId: "not-a-guid", + Operation: "add", + }, + }, + }}, + }) + require.ErrorContains(t, err, "invalid entity signal request ID") + }) + + t.Run("invalid-call-request-id", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: "not-a-guid", + Operation: "get", + }, + }, + }}, + }) + require.ErrorContains(t, err, "invalid entity call request ID") + }) + + t.Run("unsupported-history-event", func(t *testing.T) { + _, _, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_TimerCreated{ + TimerCreated: &protos.TimerCreatedEvent{}, + }, + }}, + }) + require.ErrorContains(t, err, "unsupported entity operation history event") + }) +} + +// Signals are marked in batch properties so the executor can skip building a +// response for them, and calls carry a routable response destination. +func TestEntityBatchFromRequestV2RoutesSignalsAndCalls(t *testing.T) { + signalID := uuid.NewString() + callID := uuid.NewString() + batch, infos, err := entityBatchFromRequestV2(&protos.EntityRequest{ + InstanceId: "@counter@key", + EntityState: wrapperspb.String("7"), + OperationRequests: []*protos.HistoryEvent{ + { + EventType: &protos.HistoryEvent_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{ + RequestId: signalID, + Operation: "add", + }, + }, + }, + { + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: callID, + Operation: "get", + ParentInstanceId: wrapperspb.String("parent-instance"), + ParentExecutionId: wrapperspb.String("parent-execution"), + }, + }, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, "7", batch.EntityState.GetValue()) + require.Len(t, batch.Operations, 2) + require.True(t, batch.Properties[helpers.EntitySignalProperty(signalID)].GetBoolValue()) + require.NotContains(t, batch.Properties, helpers.EntitySignalProperty(callID)) + + require.Len(t, infos, 2) + require.Nil(t, infos[0].ResponseDestination) + require.Equal(t, "parent-instance", infos[1].ResponseDestination.GetInstanceId()) + require.Equal(t, "parent-execution", infos[1].ResponseDestination.GetExecutionId().GetValue()) +} diff --git a/client/entity_client_test.go b/client/entity_client_test.go new file mode 100644 index 00000000..12b4af5f --- /dev/null +++ b/client/entity_client_test.go @@ -0,0 +1,181 @@ +package client + +import ( + "context" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type entityClientTestService struct { + protos.TaskHubSidecarServiceClient + getRequest *protos.GetEntityRequest + getResponse *protos.GetEntityResponse + queryRequest *protos.QueryEntitiesRequest + cleanupRequests []*protos.CleanEntityStorageRequest + cleanupResponses []*protos.CleanEntityStorageResponse +} + +func (service *entityClientTestService) GetEntity( + _ context.Context, + request *protos.GetEntityRequest, + _ ...grpc.CallOption, +) (*protos.GetEntityResponse, error) { + service.getRequest = request + if service.getResponse == nil { + return &protos.GetEntityResponse{}, nil + } + return service.getResponse, nil +} + +func (service *entityClientTestService) QueryEntities( + _ context.Context, + request *protos.QueryEntitiesRequest, + _ ...grpc.CallOption, +) (*protos.QueryEntitiesResponse, error) { + service.queryRequest = request + return &protos.QueryEntitiesResponse{}, nil +} + +func (service *entityClientTestService) CleanEntityStorage( + _ context.Context, + request *protos.CleanEntityStorageRequest, + _ ...grpc.CallOption, +) (*protos.CleanEntityStorageResponse, error) { + service.cleanupRequests = append(service.cleanupRequests, request) + response := service.cleanupResponses[0] + service.cleanupResponses = service.cleanupResponses[1:] + return response, nil +} + +func Test_EntityClient_GetEntityDefaultsAndMissing(t *testing.T) { + service := new(entityClientTestService) + client := &TaskHubGrpcClient{client: service, converter: api.DefaultDataConverter()} + + metadata, err := client.GetEntity(context.Background(), api.NewEntityID("counter", "key")) + require.NoError(t, err) + require.Nil(t, metadata) + require.True(t, service.getRequest.IncludeState) + + service.getResponse = &protos.GetEntityResponse{ + Exists: true, + Entity: &protos.EntityMetadata{ + InstanceId: "@counter@key", + SerializedState: wrapperspb.String("42"), + }, + } + metadata, err = client.GetEntity(context.Background(), api.NewEntityID("counter", "key")) + require.NoError(t, err) + require.True(t, metadata.StateIncluded) + require.True(t, metadata.HasState) + var state int + require.NoError(t, metadata.ReadState(&state)) + require.Equal(t, 42, state) + + metadata, err = client.GetEntity( + context.Background(), + api.NewEntityID("counter", "key"), + api.GetEntityOptions{ExcludeState: true}, + ) + require.NoError(t, err) + require.False(t, service.getRequest.IncludeState) + require.False(t, metadata.StateIncluded) + require.ErrorIs(t, metadata.ReadState(&state), api.ErrEntityStateNotIncluded) +} + +func Test_EntityClient_QueryPrefixNormalizationAndStateDefault(t *testing.T) { + service := new(entityClientTestService) + client := &TaskHubGrpcClient{client: service, converter: api.DefaultDataConverter()} + tests := map[string]string{ + "Counter": "@counter", + "@Counter": "@counter", + "Counter@Key": "@counter@Key", + "@Counter@Key": "@counter@Key", + } + for input, expected := range tests { + t.Run(input, func(t *testing.T) { + _, err := client.QueryEntities(context.Background(), api.EntityQuery{ + InstanceIDStartsWith: input, + }) + require.NoError(t, err) + require.Equal(t, expected, service.queryRequest.Query.InstanceIdStartsWith.GetValue()) + require.True(t, service.queryRequest.Query.IncludeState) + }) + } + + _, err := client.QueryEntities(context.Background(), api.EntityQuery{ExcludeState: true}) + require.NoError(t, err) + require.False(t, service.queryRequest.Query.IncludeState) +} + +func Test_EntityClient_QueryValidation(t *testing.T) { + client := &TaskHubGrpcClient{client: new(entityClientTestService)} + _, err := client.QueryEntities(context.Background(), api.EntityQuery{PageSize: -1}) + require.ErrorContains(t, err, "page size") + + now := time.Now().UTC() + _, err = client.QueryEntities(context.Background(), api.EntityQuery{ + LastModifiedFrom: now, + LastModifiedTo: now.Add(-time.Second), + }) + require.ErrorContains(t, err, "start time must not be after end time") +} + +func Test_EntityClient_CleanupDefaultsAndPagination(t *testing.T) { + service := &entityClientTestService{ + cleanupResponses: []*protos.CleanEntityStorageResponse{ + { + EmptyEntitiesRemoved: 2, + OrphanedLocksReleased: 3, + ContinuationToken: wrapperspb.String("next"), + }, + { + EmptyEntitiesRemoved: 5, + OrphanedLocksReleased: 7, + }, + }, + } + client := &TaskHubGrpcClient{client: service} + result, err := client.CleanEntityStorage(context.Background()) + require.NoError(t, err) + require.EqualValues(t, 7, result.EmptyEntitiesRemoved) + require.EqualValues(t, 10, result.OrphanedLocksReleased) + require.Empty(t, result.ContinuationToken) + require.Len(t, service.cleanupRequests, 2) + require.True(t, service.cleanupRequests[0].RemoveEmptyEntities) + require.True(t, service.cleanupRequests[0].ReleaseOrphanedLocks) + require.Equal(t, "next", service.cleanupRequests[1].ContinuationToken.GetValue()) +} + +func Test_EntityClient_CleanupSinglePageAndRepeatedToken(t *testing.T) { + singlePageService := &entityClientTestService{ + cleanupResponses: []*protos.CleanEntityStorageResponse{{ + ContinuationToken: wrapperspb.String("next"), + }}, + } + client := &TaskHubGrpcClient{client: singlePageService} + result, err := client.CleanEntityStorage( + context.Background(), + api.CleanEntityStorageOptions{SinglePage: true}, + ) + require.NoError(t, err) + require.Equal(t, "next", result.ContinuationToken) + require.Len(t, singlePageService.cleanupRequests, 1) + + repeatedService := &entityClientTestService{ + cleanupResponses: []*protos.CleanEntityStorageResponse{{ + ContinuationToken: wrapperspb.String("same"), + }}, + } + client = &TaskHubGrpcClient{client: repeatedService} + _, err = client.CleanEntityStorage( + context.Background(), + api.CleanEntityStorageOptions{ContinuationToken: "same"}, + ) + require.ErrorContains(t, err, "repeated continuation token") +} diff --git a/client/errors.go b/client/errors.go new file mode 100644 index 00000000..04defa8f --- /dev/null +++ b/client/errors.go @@ -0,0 +1,39 @@ +package client + +import ( + "errors" + "fmt" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var ( + // ErrTaskHubExists indicates that task-hub creation targeted an existing hub. + ErrTaskHubExists = errors.New("task hub already exists") + // ErrTaskHubNotFound indicates that a task-hub operation targeted a missing hub. + ErrTaskHubNotFound = errors.New("task hub not found") + // ErrStreamedHistoryLimitExceeded indicates that a worker refused to retain + // an orchestration history beyond its configured safety limit. + ErrStreamedHistoryLimitExceeded = errors.New("streamed orchestration history limit exceeded") +) + +type streamedHistoryLimitError struct { + message string +} + +func newStreamedHistoryLimitError(format string, args ...any) error { + return &streamedHistoryLimitError{message: fmt.Sprintf(format, args...)} +} + +func (e *streamedHistoryLimitError) Error() string { + return e.message +} + +func (*streamedHistoryLimitError) Unwrap() error { + return ErrStreamedHistoryLimitExceeded +} + +func (e *streamedHistoryLimitError) GRPCStatus() *status.Status { + return status.New(codes.ResourceExhausted, e.message) +} diff --git a/client/grpc_errors.go b/client/grpc_errors.go new file mode 100644 index 00000000..f457db96 --- /dev/null +++ b/client/grpc_errors.go @@ -0,0 +1,129 @@ +package client + +import ( + "context" + "fmt" + "strings" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/grpcerrors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type grpcRequestError struct { + operation string + category error + status *status.Status +} + +func (e *grpcRequestError) Error() string { + message := e.status.Message() + if e.category == nil { + return fmt.Sprintf("%s: %s", e.operation, message) + } + categoryMessage := e.category.Error() + message = strings.TrimPrefix(message, categoryMessage+": ") + if message == "" || message == categoryMessage { + return fmt.Sprintf("%s: %s", e.operation, categoryMessage) + } + return fmt.Sprintf("%s: %s: %s", e.operation, categoryMessage, message) +} + +func (e *grpcRequestError) Unwrap() []error { + if e.category == nil { + return []error{e.status.Err()} + } + return []error{e.category, e.status.Err()} +} + +func (e *grpcRequestError) GRPCStatus() *status.Status { + return e.status +} + +func clientRPCError(ctx context.Context, operation string, err error) error { + if err == nil { + return nil + } + var category error + grpcStatus := status.Convert(err) + code := grpcStatus.Code() + if ctxErr := ctx.Err(); ctxErr != nil && + (code == codes.Canceled || code == codes.DeadlineExceeded || code == codes.Unknown) { + category = ctxErr + } else if reasonCategory := clientErrorReasonCategory(grpcerrors.Reason(grpcStatus)); reasonCategory != nil { + category = reasonCategory + } else { + switch code { + case codes.AlreadyExists: + if operation == "failed to create task hub" { + category = ErrTaskHubExists + } else { + category = api.ErrDuplicateInstance + } + case codes.InvalidArgument: + category = api.ErrInvalidArgument + case codes.NotFound: + if operation == "failed to delete task hub" { + category = ErrTaskHubNotFound + } else { + category = api.ErrInstanceNotFound + } + case codes.FailedPrecondition: + if strings.HasPrefix(grpcStatus.Message(), api.ErrNotCompleted.Error()) { + category = api.ErrNotCompleted + } else { + category = api.ErrInvalidState + } + case codes.Unimplemented: + category = api.ErrFeatureNotSupported + case codes.Canceled: + category = context.Canceled + case codes.DeadlineExceeded: + category = context.DeadlineExceeded + } + } + return &grpcRequestError{ + operation: operation, + category: category, + status: grpcStatus, + } +} + +func clientErrorReasonCategory(reason string) error { + switch reason { + case grpcerrors.ReasonDuplicateInstance: + return api.ErrDuplicateInstance + case grpcerrors.ReasonFeatureUnsupported: + return api.ErrFeatureNotSupported + case grpcerrors.ReasonInstanceNotFound: + return api.ErrInstanceNotFound + case grpcerrors.ReasonInvalidArgument: + return api.ErrInvalidArgument + case grpcerrors.ReasonInvalidState: + return api.ErrInvalidState + case grpcerrors.ReasonNotCompleted: + return api.ErrNotCompleted + case grpcerrors.ReasonTaskHubExists: + return ErrTaskHubExists + case grpcerrors.ReasonTaskHubNotFound: + return ErrTaskHubNotFound + default: + return nil + } +} + +func retryableWaitRPCError(err error) bool { + switch status.Code(err) { + case codes.Canceled, + codes.DeadlineExceeded, + codes.ResourceExhausted, + codes.Aborted, + codes.Internal, + codes.Unavailable, + codes.Unknown: + return true + default: + return false + } +} diff --git a/client/grpc_errors_test.go b/client/grpc_errors_test.go new file mode 100644 index 00000000..a456d351 --- /dev/null +++ b/client/grpc_errors_test.go @@ -0,0 +1,105 @@ +package client + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/grpcerrors" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func TestClientRPCErrorMappingsPreserveStatus(t *testing.T) { + tests := []struct { + code codes.Code + category error + }{ + {codes.AlreadyExists, api.ErrDuplicateInstance}, + {codes.InvalidArgument, api.ErrInvalidArgument}, + {codes.NotFound, api.ErrInstanceNotFound}, + {codes.FailedPrecondition, api.ErrInvalidState}, + {codes.Unimplemented, api.ErrFeatureNotSupported}, + {codes.Canceled, context.Canceled}, + {codes.DeadlineExceeded, context.DeadlineExceeded}, + } + + for _, test := range tests { + t.Run(test.code.String(), func(t *testing.T) { + err := clientRPCError(context.Background(), "operation", status.Error(test.code, test.category.Error())) + if !errors.Is(err, test.category) { + t.Fatalf("error %v does not match %v", err, test.category) + } + if got := status.Code(err); got != test.code { + t.Fatalf("status.Code() = %v, want %v", got, test.code) + } + if count := strings.Count(err.Error(), test.category.Error()); count != 1 { + t.Fatalf("category appears %d times in %q", count, err) + } + }) + } +} + +func TestClientRPCErrorPrefersCallerContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := clientRPCError(ctx, "operation", status.Error(codes.Canceled, "canceled")) + if !errors.Is(err, context.Canceled) { + t.Fatalf("error = %v", err) + } + if status.Code(err) != codes.Canceled { + t.Fatalf("status.Code() = %v", status.Code(err)) + } +} + +func TestClientRPCErrorPreservesLifecycleAndStateCategories(t *testing.T) { + tests := []struct { + code codes.Code + message string + category error + }{ + {codes.AlreadyExists, ErrTaskHubExists.Error(), ErrTaskHubExists}, + {codes.NotFound, ErrTaskHubNotFound.Error(), ErrTaskHubNotFound}, + {codes.FailedPrecondition, api.ErrNotCompleted.Error(), api.ErrNotCompleted}, + } + for _, test := range tests { + var reason string + switch { + case errors.Is(test.category, ErrTaskHubExists): + reason = grpcerrors.ReasonTaskHubExists + case errors.Is(test.category, ErrTaskHubNotFound): + reason = grpcerrors.ReasonTaskHubNotFound + case errors.Is(test.category, api.ErrNotCompleted): + reason = grpcerrors.ReasonNotCompleted + } + err := clientRPCError(context.Background(), "operation", grpcerrors.New(test.code, test.message, reason)) + if !errors.Is(err, test.category) { + t.Fatalf("error %v does not match %v", err, test.category) + } + } +} + +func TestRetryableWaitRPCErrorIncludesServerLongPollAndTransientFailures(t *testing.T) { + for _, code := range []codes.Code{ + codes.Canceled, + codes.DeadlineExceeded, + codes.ResourceExhausted, + codes.Aborted, + codes.Internal, + codes.Unavailable, + codes.Unknown, + } { + if !retryableWaitRPCError(status.Error(code, "transient")) { + t.Fatalf("%v should be retryable", code) + } + } + if retryableWaitRPCError(status.Error(codes.InvalidArgument, "invalid")) { + t.Fatal("InvalidArgument should not be retryable") + } + if retryableWaitRPCError(status.Error(codes.NotFound, "missing instance")) { + t.Fatal("NotFound should return ErrInstanceNotFound without retrying") + } +} diff --git a/client/grpc_worker.go b/client/grpc_worker.go new file mode 100644 index 00000000..2ca3cad7 --- /dev/null +++ b/client/grpc_worker.go @@ -0,0 +1,1370 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "math" + "math/rand/v2" + "runtime" + "slices" + "strings" + "sync" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +var ( + ErrTaskHubGrpcWorkerAlreadyRunning = errors.New("gRPC worker is already running") + errSilentDisconnect = errors.New("work item stream was silent beyond the configured timeout") +) + +// maxWorkerConcurrency is the largest concurrency limit representable on the +// GetWorkItems wire contract, whose fields are 32-bit. +const maxWorkerConcurrency = math.MaxInt32 + +type workItemsStream interface { + Recv() (*protos.WorkItem, error) +} + +type grpcWorkerClientFactory func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) + +// TaskHubGrpcWorkerConnectionFactory creates a gRPC connection for a worker +// stream generation. A non-nil closer transfers ownership to the worker. +type TaskHubGrpcWorkerConnectionFactory func(context.Context) (grpc.ClientConnInterface, io.Closer, error) + +type TaskHubGrpcWorkerOption func(*taskHubGrpcWorkerOptions) error + +type WorkerCapability = protos.WorkerCapability + +const ( + WorkerCapabilityHistoryStreaming WorkerCapability = protos.WorkerCapability_WORKER_CAPABILITY_HISTORY_STREAMING + WorkerCapabilityScheduledTasks WorkerCapability = protos.WorkerCapability_WORKER_CAPABILITY_SCHEDULED_TASKS + WorkerCapabilityLargePayloads WorkerCapability = protos.WorkerCapability_WORKER_CAPABILITY_LARGE_PAYLOADS + + // DefaultMaxStreamedHistoryEvents and DefaultMaxStreamedHistoryBytes are + // the worker's default per-work-item replay-history safety bounds. + DefaultMaxStreamedHistoryEvents = api.DefaultHistoryMaxEvents + DefaultMaxStreamedHistoryBytes int64 = api.DefaultHistoryMaxBytes + // MaxStreamedHistoryEvents and MaxStreamedHistoryBytes bound configurable + // worker replay-history limits. + MaxStreamedHistoryEvents = api.MaxHistoryMaxEvents + MaxStreamedHistoryBytes int64 = api.MaxHistoryMaxBytes + + // DefaultMaxOrchestratorCompletionBytes leaves headroom below the modern DTS + // orchestration-completion message limit. + DefaultMaxOrchestratorCompletionBytes = 4_089_446 + minOrchestratorCompletionBytes = 64 * 1024 +) + +type WorkItemFilter struct { + Name string + Versions []string +} + +type WorkItemFilters struct { + Orchestrations []WorkItemFilter + Activities []WorkItemFilter + Entities []string + RejectAllOrchestrations bool + RejectAllActivities bool + RejectAllEntities bool +} + +type taskHubGrpcWorkerOptions struct { + maxConcurrentOrchestrations int + maxConcurrentActivities int + maxConcurrentEntities int + helloTimeout time.Duration + silentDisconnectTimeout time.Duration + rpcTimeout time.Duration + reconnectBaseDelay time.Duration + reconnectMaxDelay time.Duration + transientRetryMaxAttempts int + transientRetryBaseDelay time.Duration + transientRetryMaxDelay time.Duration + maxStreamedHistoryEvents int + maxStreamedHistoryBytes int64 + maxOrchestratorCompletionBytes int + maximumTimerInterval *time.Duration + taskExecutorOptions []task.TaskExecutorOption + versioning *task.VersioningOptions + capabilities []WorkerCapability + workItemFilters *WorkItemFilters + workItemFiltersConfigured bool + autoWorkItemFilters bool + largePayloads *api.LargePayloadOptions + converter api.DataConverter + unversionedOrchestrators map[string]struct{} + unversionedActivities map[string]struct{} + reconnectRandom randomInt64N + // waitFn overrides every delay the worker imposes on itself: reconnect + // backoff, transient RPC retry backoff, and work-item abandon delays. It is + // only set by tests so the deterministic delay schedule can be observed + // without sleeping. + waitFn func(context.Context, time.Duration) error +} + +func defaultTaskHubGrpcWorkerOptions() taskHubGrpcWorkerOptions { + defaultConcurrency := 100 * runtime.GOMAXPROCS(0) + return taskHubGrpcWorkerOptions{ + maxConcurrentOrchestrations: defaultConcurrency, + maxConcurrentActivities: defaultConcurrency, + maxConcurrentEntities: defaultConcurrency, + helloTimeout: 30 * time.Second, + silentDisconnectTimeout: 2 * time.Minute, + rpcTimeout: 30 * time.Second, + reconnectBaseDelay: 200 * time.Millisecond, + reconnectMaxDelay: 15 * time.Second, + transientRetryMaxAttempts: 10, + transientRetryBaseDelay: 200 * time.Millisecond, + transientRetryMaxDelay: 15 * time.Second, + maxStreamedHistoryEvents: DefaultMaxStreamedHistoryEvents, + maxStreamedHistoryBytes: DefaultMaxStreamedHistoryBytes, + maxOrchestratorCompletionBytes: DefaultMaxOrchestratorCompletionBytes, + reconnectRandom: rand.Int64N, + capabilities: []WorkerCapability{WorkerCapabilityHistoryStreaming}, + } +} + +func WithMaxConcurrentOrchestrationWorkItems(n int) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if err := validateWorkerConcurrency("orchestration", n); err != nil { + return err + } + options.maxConcurrentOrchestrations = n + return nil + } +} + +func WithMaxConcurrentActivityWorkItems(n int) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if err := validateWorkerConcurrency("activity", n); err != nil { + return err + } + options.maxConcurrentActivities = n + return nil + } +} + +// WithMaxConcurrentEntityWorkItems limits concurrent entity batch execution. +func WithMaxConcurrentEntityWorkItems(n int) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if err := validateWorkerConcurrency("entity", n); err != nil { + return err + } + options.maxConcurrentEntities = n + return nil + } +} + +// WithMaxStreamedHistoryEvents limits the number of history events retained +// while receiving a streamed orchestration history. +func WithMaxStreamedHistoryEvents(n int) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if n <= 0 || n > MaxStreamedHistoryEvents { + return fmt.Errorf( + "maximum streamed history events must be between 1 and %d", + MaxStreamedHistoryEvents, + ) + } + options.maxStreamedHistoryEvents = n + return nil + } +} + +// WithMaxStreamedHistoryBytes limits the protobuf bytes retained while +// receiving a streamed orchestration history. +func WithMaxStreamedHistoryBytes(n int64) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if n <= 0 || n > MaxStreamedHistoryBytes { + return fmt.Errorf( + "maximum streamed history bytes must be between 1 and %d", + MaxStreamedHistoryBytes, + ) + } + options.maxStreamedHistoryBytes = n + return nil + } +} + +// WithMaxOrchestratorCompletionBytes lowers the post-externalization response +// bound used before completing an orchestration work item. Multiple options are +// monotonic: a later option cannot raise an earlier transport-derived cap. +func WithMaxOrchestratorCompletionBytes(n int) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if n < minOrchestratorCompletionBytes || n > DefaultMaxOrchestratorCompletionBytes { + return fmt.Errorf( + "maximum orchestrator completion bytes must be between %d and %d", + minOrchestratorCompletionBytes, + DefaultMaxOrchestratorCompletionBytes, + ) + } + options.maxOrchestratorCompletionBytes = min(options.maxOrchestratorCompletionBytes, n) + return nil + } +} + +// validateWorkerConcurrency rejects limits that cannot be advertised on the +// 32-bit GetWorkItems fields, which would otherwise silently wrap to a negative +// value on 64-bit platforms. +func validateWorkerConcurrency(kind string, n int) error { + if n <= 0 { + return fmt.Errorf("maximum concurrent %s work items must be greater than zero", kind) + } + if n > maxWorkerConcurrency { + return fmt.Errorf( + "maximum concurrent %s work items must be at most %d", + kind, + maxWorkerConcurrency, + ) + } + return nil +} + +func WithWorkerHelloTimeout(timeout time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if timeout <= 0 { + return fmt.Errorf("worker Hello timeout must be greater than zero") + } + options.helloTimeout = timeout + return nil + } +} + +func WithWorkerSilentDisconnectTimeout(timeout time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if timeout <= 0 { + return fmt.Errorf("worker silent disconnect timeout must be greater than zero") + } + options.silentDisconnectTimeout = timeout + return nil + } +} + +func WithWorkerRPCTimeout(timeout time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if timeout <= 0 { + return fmt.Errorf("worker RPC timeout must be greater than zero") + } + options.rpcTimeout = timeout + return nil + } +} + +// WithWorkerReconnectBackoff configures the reconnect schedule. Nominal delays +// double from baseDelay, receive ±25% jitter, and remain within +// [baseDelay, maxDelay]. +func WithWorkerReconnectBackoff(baseDelay, maxDelay time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if baseDelay <= 0 || maxDelay < baseDelay { + return fmt.Errorf("worker reconnect delays must be positive and max delay must be at least the base delay") + } + options.reconnectBaseDelay = baseDelay + options.reconnectMaxDelay = maxDelay + return nil + } +} + +// WithWorkerTransientRetryPolicy configures the deterministic retry schedule for +// completion and abandon RPCs. Delays double from baseDelay and are always +// within [baseDelay, maxDelay]. +func WithWorkerTransientRetryPolicy(maxAttempts int, baseDelay, maxDelay time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if maxAttempts <= 0 { + return fmt.Errorf("worker transient retry attempts must be greater than zero") + } + if baseDelay <= 0 || maxDelay < baseDelay { + return fmt.Errorf("worker transient retry delays must be positive and max delay must be at least the base delay") + } + options.transientRetryMaxAttempts = maxAttempts + options.transientRetryBaseDelay = baseDelay + options.transientRetryMaxDelay = maxDelay + return nil + } +} + +// WithMaximumTimerInterval configures deterministic splitting of long durable +// timers. Changing this value is replay-breaking for affected in-flight +// orchestrations. +func WithMaximumTimerInterval(interval time.Duration) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if interval < 0 { + return fmt.Errorf("maximum timer interval cannot be negative") + } + if interval == 0 { + interval = task.DefaultMaximumTimerInterval + } + options.maximumTimerInterval = &interval + return nil + } +} + +// WithTaskExecutorOptions configures the task executor used by the gRPC worker. +// This allows task options such as task.WithVersioning to participate in DTS dispatch. +func WithTaskExecutorOptions(options ...task.TaskExecutorOption) TaskHubGrpcWorkerOption { + return func(workerOptions *taskHubGrpcWorkerOptions) error { + for _, option := range options { + if option == nil { + return errors.New("task executor option cannot be nil") + } + } + workerOptions.taskExecutorOptions = append(workerOptions.taskExecutorOptions, options...) + return nil + } +} + +// WithTaskVersioning configures worker version acceptance and the default +// version used for sub-orchestrations. +func WithTaskVersioning(versioning task.VersioningOptions) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + if err := versioning.Validate(); err != nil { + return err + } + options.versioning = &versioning + return nil + } +} + +// WithWorkerCapabilities explicitly configures the capabilities advertised to DTS. +func WithWorkerCapabilities(capabilities ...WorkerCapability) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + seen := make(map[WorkerCapability]struct{}, len(capabilities)) + configured := make([]WorkerCapability, 0, len(capabilities)) + for _, capability := range capabilities { + switch capability { + case WorkerCapabilityHistoryStreaming, WorkerCapabilityScheduledTasks, WorkerCapabilityLargePayloads: + default: + return fmt.Errorf("unsupported worker capability: %d", capability) + } + if _, ok := seen[capability]; ok { + continue + } + seen[capability] = struct{}{} + configured = append(configured, capability) + } + options.capabilities = configured + return nil + } +} + +// WithScheduledTaskCapability controls scheduled-task capability advertisement. +func WithScheduledTaskCapability(enabled bool) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + options.capabilities = setWorkerCapability(options.capabilities, WorkerCapabilityScheduledTasks, enabled) + return nil + } +} + +// CombineTaskHubGrpcWorkerOptions combines worker options into one option. +func CombineTaskHubGrpcWorkerOptions(options ...TaskHubGrpcWorkerOption) TaskHubGrpcWorkerOption { + return func(target *taskHubGrpcWorkerOptions) error { + for _, configure := range options { + if configure == nil { + continue + } + if err := configure(target); err != nil { + return err + } + } + return nil + } +} + +// WithUnversionedOrchestratorNames allows explicitly unversioned system +// orchestrators to remain routable when strict worker versioning is enabled. +func WithUnversionedOrchestratorNames(names ...string) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + return addUnversionedTaskNames(&options.unversionedOrchestrators, "orchestrator", names) + } +} + +// WithUnversionedActivityNames allows explicitly unversioned system activities +// to remain routable when strict worker versioning is enabled. System +// orchestrations run unversioned and an activity inherits its caller's version, +// so a system component's activities must be advertised unversioned too. +func WithUnversionedActivityNames(names ...string) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + return addUnversionedTaskNames(&options.unversionedActivities, "activity", names) + } +} + +func addUnversionedTaskNames( + allowed *map[string]struct{}, + kind string, + names []string, +) error { + if *allowed == nil { + *allowed = make(map[string]struct{}, len(names)) + } + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + return fmt.Errorf("unversioned %s name cannot be empty", kind) + } + (*allowed)[strings.ToLower(name)] = struct{}{} + } + return nil +} + +// WithWorkItemFilters restricts orchestration/activity names and versions and entity names accepted by the worker. +// A nil per-kind list means no restriction for that kind. RejectAll* explicitly rejects a kind. +func WithWorkItemFilters(filters *WorkItemFilters) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + options.workItemFiltersConfigured = true + if filters == nil { + options.workItemFilters = nil + return nil + } + clone, err := cloneWorkItemFilters(filters) + if err != nil { + return err + } + options.workItemFilters = clone + return nil + } +} + +// WithAutoWorkItemFilters derives task-name and task-version filters from the registry. +// Explicit [WithWorkItemFilters] configuration takes precedence. +func WithAutoWorkItemFilters() TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + options.autoWorkItemFilters = true + return nil + } +} + +// WithWorkerLargePayloads configures worker payload hydration/externalization and advertises support. +func WithWorkerLargePayloads(options *api.LargePayloadOptions) TaskHubGrpcWorkerOption { + return func(workerOptions *taskHubGrpcWorkerOptions) error { + normalized, err := api.NormalizeLargePayloadOptions(options) + if err != nil { + return err + } + workerOptions.largePayloads = normalized + workerOptions.capabilities = setWorkerCapability( + workerOptions.capabilities, + WorkerCapabilityLargePayloads, + normalized != nil, + ) + return nil + } +} + +// WithWorkerDataConverter configures application payload serialization. +func WithWorkerDataConverter(converter api.DataConverter) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + options.converter = api.NormalizeDataConverter(converter) + return nil + } +} + +// TaskHubGrpcWorker executes orchestration, activity, and entity work received from a +// DTS gRPC work-item stream (named TaskHubSidecarService in the wire contract). +type TaskHubGrpcWorker struct { + clientFactory grpcWorkerClientFactory + executor task.Executor + logger api.Logger + options taskHubGrpcWorkerOptions + + mu sync.Mutex + run *grpcWorkerRun + lastErr error +} + +type grpcWorkerRun struct { + intakeCtx context.Context + cancelIntake context.CancelFunc + processingCtx context.Context + cancelProcessing context.CancelFunc + done chan struct{} + + orchestrationSlots chan struct{} + activitySlots chan struct{} + entitySlots chan struct{} + pending sync.WaitGroup + retired sync.WaitGroup + err error +} + +type grpcWorkerConnection struct { + client protos.TaskHubSidecarServiceClient + stream workItemsStream + cancelStream context.CancelFunc + closer io.Closer + + pending sync.WaitGroup +} + +// NewTaskHubGrpcWorker creates a worker that borrows a caller-owned connection. +// +// The worker never closes or replaces the supplied connection: a reconnect only +// recreates the work-item stream on that same connection. Recovery is therefore +// limited to what the caller's connection can do on its own. A connection that +// is permanently wedged (for example one whose credentials expired, or one whose +// endpoint moved) keeps producing poisoned streams and the worker keeps retrying +// on the escalating reconnect schedule without ever obtaining a fresh channel. +// Use [NewTaskHubGrpcWorkerWithConnectionFactory], or the Durable Task Scheduler +// NewWorker, when the worker should own and recreate its channels. +func NewTaskHubGrpcWorker( + cc grpc.ClientConnInterface, + registry *task.TaskRegistry, + logger api.Logger, + opts ...TaskHubGrpcWorkerOption, +) (*TaskHubGrpcWorker, error) { + if cc == nil { + return nil, fmt.Errorf("gRPC connection is required") + } + return newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return protos.NewTaskHubSidecarServiceClient(cc), nil, nil + }, + registry, + logger, + opts..., + ) +} + +// NewTaskHubGrpcWorkerWithConnectionFactory creates a worker whose connection +// can be recreated after disconnects. Each returned closer is called only after +// all work dispatched through that connection has completed or been abandoned. +func NewTaskHubGrpcWorkerWithConnectionFactory( + factory TaskHubGrpcWorkerConnectionFactory, + registry *task.TaskRegistry, + logger api.Logger, + opts ...TaskHubGrpcWorkerOption, +) (*TaskHubGrpcWorker, error) { + if factory == nil { + return nil, fmt.Errorf("gRPC worker connection factory is required") + } + return newTaskHubGrpcWorker( + func(ctx context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + cc, closer, err := factory(ctx) + if err != nil { + return nil, nil, err + } + if cc == nil { + if closer != nil { + _ = closer.Close() + } + return nil, nil, fmt.Errorf("gRPC worker connection factory returned a nil connection") + } + return protos.NewTaskHubSidecarServiceClient(cc), closer, nil + }, + registry, + logger, + opts..., + ) +} + +func newTaskHubGrpcWorker( + factory grpcWorkerClientFactory, + registry *task.TaskRegistry, + logger api.Logger, + opts ...TaskHubGrpcWorkerOption, +) (*TaskHubGrpcWorker, error) { + if factory == nil { + return nil, fmt.Errorf("gRPC worker client factory is required") + } + if registry == nil { + return nil, fmt.Errorf("task registry is required") + } + if logger == nil { + logger = api.DefaultLogger() + } + + options := defaultTaskHubGrpcWorkerOptions() + for _, configure := range opts { + if configure == nil { + continue + } + if err := configure(&options); err != nil { + return nil, err + } + } + hasLargePayloadCapability := slices.Contains(options.capabilities, WorkerCapabilityLargePayloads) + if hasLargePayloadCapability && options.largePayloads == nil { + return nil, fmt.Errorf("large-payload capability requires worker large-payload options") + } + if options.largePayloads != nil { + options.capabilities = setWorkerCapability(options.capabilities, WorkerCapabilityLargePayloads, true) + } + snapshot := registry.Snapshot() + if options.workItemFiltersConfigured && options.workItemFilters != nil { + if err := validateWorkItemFilters(options.workItemFilters, snapshot); err != nil { + return nil, err + } + } + if options.autoWorkItemFilters && !options.workItemFiltersConfigured { + if err := validateStrictAutoFilters(snapshot, options.versioning); err != nil { + return nil, err + } + options.workItemFilters = workItemFiltersFromRegistry( + snapshot, + options.versioning, + options.unversionedOrchestrators, + options.unversionedActivities, + ) + } + return &TaskHubGrpcWorker{ + clientFactory: factory, + executor: task.NewTaskExecutor(registry, options.executorOptions()...), + logger: logger, + options: options, + }, nil +} + +// executorOptions returns the canonical executor configuration. Dedicated +// worker options are appended last so they consistently override generic ones. +func (options taskHubGrpcWorkerOptions) executorOptions() []task.TaskExecutorOption { + executorOptions := slices.Clone(options.taskExecutorOptions) + if options.maximumTimerInterval != nil { + executorOptions = append(executorOptions, task.WithMaximumTimerInterval(*options.maximumTimerInterval)) + } + if options.versioning != nil { + executorOptions = append(executorOptions, task.WithVersioning(*options.versioning)) + } + if options.converter != nil { + executorOptions = append(executorOptions, task.WithDataConverter(options.converter)) + } + if len(options.unversionedOrchestrators) > 0 { + executorOptions = append( + executorOptions, + task.WithUnversionedOrchestratorNames(slices.Collect(maps.Keys(options.unversionedOrchestrators))...), + ) + } + if len(options.unversionedActivities) > 0 { + executorOptions = append( + executorOptions, + task.WithUnversionedActivityNames(slices.Collect(maps.Keys(options.unversionedActivities))...), + ) + } + return executorOptions +} + +func workItemFiltersFromRegistry( + snapshot task.TaskRegistrySnapshot, + versioning *task.VersioningOptions, + allowedUnversionedOrchestrators map[string]struct{}, + allowedUnversionedActivities map[string]struct{}, +) *WorkItemFilters { + entities := slices.Clone(snapshot.Entities) + if slices.Contains(entities, "*") { + entities = nil + } + orchestrations := taskRegistrationsToFilters( + snapshot.Orchestrators, versioning, allowedUnversionedOrchestrators) + activities := taskRegistrationsToFilters( + snapshot.Activities, versioning, allowedUnversionedActivities) + return &WorkItemFilters{ + Orchestrations: orchestrations, + Activities: activities, + Entities: entities, + RejectAllOrchestrations: len(snapshot.Orchestrators) == 0, + RejectAllActivities: len(snapshot.Activities) == 0, + RejectAllEntities: len(snapshot.Entities) == 0, + } +} + +func validateStrictAutoFilters( + snapshot task.TaskRegistrySnapshot, + versioning *task.VersioningOptions, +) error { + if versioning == nil || versioning.MatchStrategy != task.VersionMatchStrict { + return nil + } + if err := validateStrictRegistrations("orchestrator", snapshot.Orchestrators, versioning.Version); err != nil { + return err + } + return validateStrictRegistrations("activity", snapshot.Activities, versioning.Version) +} + +func validateStrictRegistrations( + kind string, + registrations []task.TaskRegistration, + version string, +) error { + type versions struct { + hasUnversioned bool + versioned map[string]struct{} + } + byName := make(map[string]*versions) + for _, registration := range registrations { + name := strings.ToLower(registration.Name) + group := byName[name] + if group == nil { + group = &versions{versioned: make(map[string]struct{})} + byName[name] = group + } + if registration.Version == "" { + group.hasUnversioned = true + } else { + group.versioned[strings.ToLower(registration.Version)] = struct{}{} + } + } + for name, group := range byName { + if len(group.versioned) == 0 && group.hasUnversioned { + continue + } + if _, ok := group.versioned[strings.ToLower(version)]; !ok { + return fmt.Errorf("%s %q has no registration for strict worker version %q", kind, name, version) + } + } + return nil +} + +func validateWorkItemFilters(filters *WorkItemFilters, snapshot task.TaskRegistrySnapshot) error { + if err := validateTaskFilterNames("orchestration", filters.Orchestrations, snapshot.Orchestrators); err != nil { + return err + } + if err := validateTaskFilterNames("activity", filters.Activities, snapshot.Activities); err != nil { + return err + } + if len(snapshot.Entities) == 0 || slices.Contains(snapshot.Entities, "*") { + return nil + } + entityNames := make(map[string]struct{}, len(snapshot.Entities)) + for _, name := range snapshot.Entities { + entityNames[helpers.ToLowerInvariant(name)] = struct{}{} + } + for _, name := range filters.Entities { + if _, ok := entityNames[helpers.ToLowerInvariant(name)]; !ok { + return fmt.Errorf("entity work-item filter %q is not registered", name) + } + } + return nil +} + +func validateTaskFilterNames( + kind string, + filters []WorkItemFilter, + registrations []task.TaskRegistration, +) error { + if len(registrations) == 0 { + return nil + } + names := make(map[string]struct{}, len(registrations)) + for _, registration := range registrations { + if registration.Name == "*" { + return nil + } + names[strings.ToLower(registration.Name)] = struct{}{} + } + for _, filter := range filters { + if _, ok := names[strings.ToLower(filter.Name)]; !ok { + return fmt.Errorf("%s work-item filter %q is not registered", kind, filter.Name) + } + } + return nil +} + +func taskRegistrationsToFilters( + registrations []task.TaskRegistration, + versioning *task.VersioningOptions, + allowedUnversioned map[string]struct{}, +) []WorkItemFilter { + type filterGroup struct { + name string + versions map[string]string + } + groups := make(map[string]*filterGroup) + for _, registration := range registrations { + if registration.Name == "*" { + return nil + } + normalizedName := strings.ToLower(registration.Name) + group := groups[normalizedName] + if group == nil { + group = &filterGroup{name: registration.Name, versions: make(map[string]string)} + groups[normalizedName] = group + } + group.versions[strings.ToLower(registration.Version)] = registration.Version + } + filters := make([]WorkItemFilter, 0, len(groups)) + for _, group := range groups { + if versioning != nil && versioning.MatchStrategy == task.VersionMatchStrict { + if _, allowed := allowedUnversioned[strings.ToLower(group.name)]; allowed { + if unversioned, ok := group.versions[""]; ok { + filters = append(filters, WorkItemFilter{ + Name: group.name, + Versions: []string{unversioned}, + }) + continue + } + } + filters = append(filters, WorkItemFilter{ + Name: group.name, + Versions: []string{versioning.Version}, + }) + continue + } + versions := make([]string, 0, len(group.versions)) + _, hasUnversioned := group.versions[""] + for normalized, version := range group.versions { + if normalized != "" { + versions = append(versions, version) + } + } + slices.SortFunc(versions, func(left, right string) int { + return strings.Compare(strings.ToLower(left), strings.ToLower(right)) + }) + if hasUnversioned && len(versions) > 0 { + versions = append([]string{""}, versions...) + } + filters = append(filters, WorkItemFilter{Name: group.name, Versions: versions}) + } + slices.SortFunc(filters, func(left, right WorkItemFilter) int { + return strings.Compare(strings.ToLower(left.Name), strings.ToLower(right.Name)) + }) + return filters +} + +// Start connects to the service, performs the Hello handshake, and starts the +// worker in the background. +func (w *TaskHubGrpcWorker) Start(ctx context.Context) error { + run, connection, err := w.begin(ctx) + if err != nil { + return err + } + go w.execute(run, connection) + return nil +} + +// Run connects to the service, performs the Hello handshake, and blocks until +// the worker stops. +func (w *TaskHubGrpcWorker) Run(ctx context.Context) error { + run, connection, err := w.begin(ctx) + if err != nil { + return err + } + w.execute(run, connection) + return run.err +} + +// Shutdown stops intake and waits for in-flight work to finish. If ctx expires, +// in-flight execution and completion RPCs are canceled. +func (w *TaskHubGrpcWorker) Shutdown(ctx context.Context) error { + w.mu.Lock() + run := w.run + if run == nil { + w.mu.Unlock() + return nil + } + run.cancelIntake() + w.mu.Unlock() + + select { + case <-run.done: + return run.err + case <-ctx.Done(): + run.cancelProcessing() + return ctx.Err() + } +} + +// Wait waits for a worker started with Start to stop. +func (w *TaskHubGrpcWorker) Wait(ctx context.Context) error { + w.mu.Lock() + run := w.run + lastErr := w.lastErr + w.mu.Unlock() + if run == nil { + return lastErr + } + + select { + case <-run.done: + return run.err + case <-ctx.Done(): + return ctx.Err() + } +} + +func (w *TaskHubGrpcWorker) Running() bool { + w.mu.Lock() + defer w.mu.Unlock() + return w.run != nil +} + +func (w *TaskHubGrpcWorker) begin(ctx context.Context) (*grpcWorkerRun, *grpcWorkerConnection, error) { + if ctx == nil { + ctx = context.Background() + } + + w.mu.Lock() + if w.run != nil { + w.mu.Unlock() + return nil, nil, ErrTaskHubGrpcWorkerAlreadyRunning + } + intakeCtx, cancelIntake := context.WithCancel(ctx) + processingCtx, cancelProcessing := context.WithCancel(context.WithoutCancel(ctx)) + run := &grpcWorkerRun{ + intakeCtx: intakeCtx, + cancelIntake: cancelIntake, + processingCtx: processingCtx, + cancelProcessing: cancelProcessing, + done: make(chan struct{}), + orchestrationSlots: make( + chan struct{}, + w.options.maxConcurrentOrchestrations, + ), + activitySlots: make(chan struct{}, w.options.maxConcurrentActivities), + entitySlots: make(chan struct{}, w.options.maxConcurrentEntities), + } + w.run = run + w.lastErr = nil + w.mu.Unlock() + + connection, err := w.connect(run.intakeCtx) + if err != nil { + run.err = err + run.cancelIntake() + run.cancelProcessing() + close(run.done) + w.mu.Lock() + if w.run == run { + w.run = nil + w.lastErr = err + } + w.mu.Unlock() + return nil, nil, err + } + return run, connection, nil +} + +func (w *TaskHubGrpcWorker) execute(run *grpcWorkerRun, connection *grpcWorkerConnection) { + defer func() { + run.cancelIntake() + run.pending.Wait() + run.cancelProcessing() + run.retired.Wait() + if err := w.executor.Shutdown(context.Background()); err != nil && run.err == nil { + run.err = fmt.Errorf("failed to shut down worker executor: %w", err) + } + if run.err != nil { + w.logger.Errorf("gRPC worker stopped: %v", run.err) + } + + w.mu.Lock() + if w.run == run { + w.run = nil + w.lastErr = run.err + } + close(run.done) + w.mu.Unlock() + }() + + run.err = w.runLoop(run, connection) +} + +func (w *TaskHubGrpcWorker) runLoop(run *grpcWorkerRun, connection *grpcWorkerConnection) error { + reconnectBackoff := newWorkerBackoff( + w.options.reconnectBaseDelay, + w.options.reconnectMaxDelay, + w.options.reconnectRandom, + ) + for { + observedMessage, err := w.consumeConnection(run, connection) + w.retireConnection(run, connection) + + if run.intakeCtx.Err() != nil { + return nil + } + if !isTransientWorkerError(err) { + return fmt.Errorf("work item stream stopped with a non-retryable error: %w", err) + } + if code := status.Code(err); code == codes.Unauthenticated || code == codes.PermissionDenied { + w.logger.Warnf("gRPC worker authentication or authorization failed; reconnecting so refreshed credentials or RBAC can recover: %v", err) + } + if observedMessage { + // The stream proved the endpoint healthy before it ended, so this is + // a drain rather than a poisoned connection and the escalating + // reconnect schedule restarts from the base delay. + reconnectBackoff.Reset() + } else if errors.Is(err, errSilentDisconnect) { + // Silence before the first message means the stream was accepted but + // never produced anything, so keep escalating instead of hammering a + // wedged endpoint. + w.logger.Warnf( + "gRPC work item stream stayed silent for %v before delivering any message; treating it as poisoned", + w.options.silentDisconnectTimeout, + ) + } + + for { + delay := reconnectBackoff.Next() + if err := w.wait(run.intakeCtx, delay); err != nil { + return nil + } + + next, connectErr := w.connect(run.intakeCtx) + if connectErr == nil { + w.logger.Info("reconnected gRPC work item stream") + connection = next + break + } + if run.intakeCtx.Err() != nil { + return nil + } + if !isTransientWorkerError(connectErr) { + return fmt.Errorf("failed to reconnect gRPC work item stream: %w", connectErr) + } + w.logger.Warnf("transient gRPC worker reconnect failure: %v", connectErr) + } + } +} + +func (w *TaskHubGrpcWorker) connect(ctx context.Context) (*grpcWorkerConnection, error) { + client, closer, err := w.clientFactory(ctx) + if err != nil { + return nil, fmt.Errorf("failed to create gRPC worker connection: %w", err) + } + closeOnError := func() { + if closer != nil { + _ = closer.Close() + } + } + + helloCtx, cancelHello := context.WithTimeout(ctx, w.options.helloTimeout) + _, err = client.Hello(helloCtx, &emptypb.Empty{}) + cancelHello() + if err != nil { + closeOnError() + return nil, fmt.Errorf("gRPC worker Hello failed: %w", err) + } + + streamCtx, cancelStream := context.WithCancel(ctx) + stream, err := client.GetWorkItems(streamCtx, &protos.GetWorkItemsRequest{ + MaxConcurrentOrchestrationWorkItems: int32(w.options.maxConcurrentOrchestrations), + MaxConcurrentActivityWorkItems: int32(w.options.maxConcurrentActivities), + MaxConcurrentEntityWorkItems: int32(w.options.maxConcurrentEntities), + Capabilities: slices.Clone(w.options.capabilities), + WorkItemFilters: workItemFiltersToProto(w.options.workItemFilters), + }) + if err != nil { + cancelStream() + closeOnError() + return nil, fmt.Errorf("failed to open gRPC work item stream: %w", err) + } + + return &grpcWorkerConnection{ + client: client, + stream: stream, + cancelStream: cancelStream, + closer: closer, + }, nil +} + +func setWorkerCapability(capabilities []WorkerCapability, capability WorkerCapability, enabled bool) []WorkerCapability { + index := slices.Index(capabilities, capability) + switch { + case enabled && index < 0: + return append(capabilities, capability) + case !enabled && index >= 0: + return slices.Delete(capabilities, index, index+1) + default: + return capabilities + } +} + +func cloneWorkItemFilters(filters *WorkItemFilters) (*WorkItemFilters, error) { + result := &WorkItemFilters{ + RejectAllOrchestrations: filters.RejectAllOrchestrations, + RejectAllActivities: filters.RejectAllActivities, + RejectAllEntities: filters.RejectAllEntities, + } + var err error + if result.Orchestrations, err = cloneNamedFilters( + "orchestration", "RejectAllOrchestrations", filters.RejectAllOrchestrations, filters.Orchestrations, + ); err != nil { + return nil, err + } + if result.Activities, err = cloneNamedFilters( + "activity", "RejectAllActivities", filters.RejectAllActivities, filters.Activities, + ); err != nil { + return nil, err + } + if result.Entities, err = cloneEntityFilters(filters.RejectAllEntities, filters.Entities); err != nil { + return nil, err + } + return result, nil +} + +// cloneNamedFilters validates one kind of named filter and returns a sorted deep +// copy with sorted version sets, so the advertised filter set is deterministic. +func cloneNamedFilters( + kind string, + rejectAllField string, + rejectAll bool, + filters []WorkItemFilter, +) ([]WorkItemFilter, error) { + if rejectAll && len(filters) > 0 { + return nil, fmt.Errorf("%s filters cannot be combined with %s", kind, rejectAllField) + } + if filters == nil { + return nil, nil + } + cloned := make([]WorkItemFilter, len(filters)) + for i, filter := range filters { + if filter.Name == "" { + return nil, fmt.Errorf("%s filter name cannot be empty", kind) + } + versions := slices.Clone(filter.Versions) + slices.Sort(versions) + cloned[i] = WorkItemFilter{Name: filter.Name, Versions: versions} + } + slices.SortFunc(cloned, func(left, right WorkItemFilter) int { + return strings.Compare(left.Name, right.Name) + }) + if err := rejectDuplicateFilterNames(kind, cloned); err != nil { + return nil, err + } + return cloned, nil +} + +// cloneEntityFilters validates entity filter names and returns them lowercased +// and sorted, which is the case-insensitive form the service is given. +func cloneEntityFilters(rejectAll bool, names []string) ([]string, error) { + if rejectAll && len(names) > 0 { + return nil, errors.New("entity filters cannot be combined with RejectAllEntities") + } + if names == nil { + return nil, nil + } + cloned := make([]string, len(names)) + for i, name := range names { + normalized := helpers.ToLowerInvariant(strings.TrimSpace(name)) + if normalized == "" { + return nil, errors.New("entity filter name cannot be empty") + } + cloned[i] = normalized + } + slices.Sort(cloned) + for i := 1; i < len(cloned); i++ { + if cloned[i] == cloned[i-1] { + return nil, fmt.Errorf("entity work-item filter %q is declared more than once", cloned[i]) + } + } + return cloned, nil +} + +// rejectDuplicateFilterNames reports names declared more than once, which would +// otherwise make the effective version set silently depend on iteration order. +func rejectDuplicateFilterNames(kind string, filters []WorkItemFilter) error { + seen := make(map[string]struct{}, len(filters)) + for _, filter := range filters { + normalized := strings.ToLower(filter.Name) + if _, ok := seen[normalized]; ok { + return fmt.Errorf("%s work-item filter %q is declared more than once", kind, filter.Name) + } + seen[normalized] = struct{}{} + } + return nil +} + +func workItemFiltersToProto(filters *WorkItemFilters) *protos.WorkItemFilters { + if filters == nil { + return nil + } + result := &protos.WorkItemFilters{ + Orchestrations: make([]*protos.OrchestrationFilter, 0, len(filters.Orchestrations)), + Activities: make([]*protos.ActivityFilter, 0, len(filters.Activities)), + Entities: make([]*protos.EntityFilter, 0, len(filters.Entities)), + } + if filters.RejectAllOrchestrations { + result.Orchestrations = append(result.Orchestrations, &protos.OrchestrationFilter{ + Name: helpers.RejectAllWorkItemFilterName, + }) + } else { + for _, filter := range filters.Orchestrations { + result.Orchestrations = append(result.Orchestrations, &protos.OrchestrationFilter{ + Name: filter.Name, + Versions: slices.Clone(filter.Versions), + }) + } + } + if filters.RejectAllActivities { + result.Activities = append(result.Activities, &protos.ActivityFilter{ + Name: helpers.RejectAllWorkItemFilterName, + }) + } else { + for _, filter := range filters.Activities { + result.Activities = append(result.Activities, &protos.ActivityFilter{ + Name: filter.Name, + Versions: slices.Clone(filter.Versions), + }) + } + } + if filters.RejectAllEntities { + result.Entities = append(result.Entities, &protos.EntityFilter{ + Name: helpers.RejectAllWorkItemFilterName, + }) + } else { + for _, name := range filters.Entities { + result.Entities = append(result.Entities, &protos.EntityFilter{Name: name}) + } + } + return result +} + +func (w *TaskHubGrpcWorker) retireConnection(run *grpcWorkerRun, connection *grpcWorkerConnection) { + connection.cancelStream() + run.retired.Add(1) + go func() { + defer run.retired.Done() + connection.pending.Wait() + if connection.closer != nil { + if err := connection.closer.Close(); err != nil { + w.logger.Warnf("failed to close retired gRPC worker connection: %v", err) + } + } + }() +} + +func isTransientWorkerError(err error) bool { + if err == nil { + return false + } + if errors.Is(err, ErrStreamedHistoryLimitExceeded) { + return false + } + if errors.Is(err, io.EOF) || errors.Is(err, errSilentDisconnect) { + return true + } + grpcStatus, ok := status.FromError(err) + if !ok { + return false + } + if grpcStatus.Code() == codes.Canceled && + strings.Contains(grpcStatus.Message(), "client connection is closing") { + return false + } + return isTransientWorkerGRPCStatus(grpcStatus) +} + +func isTransientWorkerGRPCStatus(grpcStatus *status.Status) bool { + if grpcStatus.Code() == codes.ResourceExhausted { + message := grpcStatus.Message() + if strings.Contains(message, "received message larger than max") || + strings.Contains(message, "trying to send message larger than max") { + return false + } + } + return isTransientWorkerGRPCCode(grpcStatus.Code()) +} + +func isTransientWorkerGRPCCode(code codes.Code) bool { + switch code { + case codes.Canceled, + codes.DeadlineExceeded, + codes.NotFound, + codes.Unauthenticated, + codes.PermissionDenied, + codes.ResourceExhausted, + codes.Aborted, + codes.Internal, + codes.Unavailable, + codes.Unknown: + return true + default: + return false + } +} + +// wait blocks for the given delay, honoring the test wait seam when one is +// configured. Every worker-imposed delay must go through it so no code path +// sleeps for real under test. +func (w *TaskHubGrpcWorker) wait(ctx context.Context, delay time.Duration) error { + if w.options.waitFn != nil { + return w.options.waitFn(ctx, delay) + } + return waitForRetry(ctx, delay) +} + +func waitForRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { + // A zero timer fires immediately and would race with cancellation, so + // observe the context first and keep shutdown deterministic. + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +type randomInt64N func(int64) int64 + +// workerBackoff is a bounded, jittered exponential backoff. Every delay it +// produces stays within [base, max], and doubling saturates instead of +// overflowing for very large maximums. +type workerBackoff struct { + base time.Duration + max time.Duration + current time.Duration + random randomInt64N +} + +func newWorkerBackoff(base, max time.Duration, random randomInt64N) *workerBackoff { + if base <= 0 { + base = time.Millisecond + } + if max < base { + max = base + } + return &workerBackoff{base: base, max: max, current: base, random: random} +} + +// Reset restarts the schedule at the base delay. +func (b *workerBackoff) Reset() { + b.current = b.base +} + +// Next returns the current delay and advances the schedule. +func (b *workerBackoff) Next() time.Duration { + delay := b.current + b.current = doubleDurationBounded(b.current, b.max) + if b.random != nil { + jitter := delay / 4 + lower := max(b.base, delay-jitter) + upper := b.max + if delay <= time.Duration(math.MaxInt64)-jitter { + upper = min(b.max, delay+jitter) + } + if width := upper - lower; width > 0 { + delay = lower + time.Duration(b.random(int64(width)+1)) + } + } + return delay +} + +// doubleDurationBounded doubles delay without overflowing int64 and clamps the +// result to max. +func doubleDurationBounded(delay, max time.Duration) time.Duration { + if delay >= max || delay > math.MaxInt64/2 { + return max + } + if doubled := delay * 2; doubled < max { + return doubled + } + return max +} diff --git a/client/grpc_worker_bench_test.go b/client/grpc_worker_bench_test.go new file mode 100644 index 00000000..cd0e0fc4 --- /dev/null +++ b/client/grpc_worker_bench_test.go @@ -0,0 +1,128 @@ +package client + +import ( + "fmt" + "testing" + + "github.com/microsoft/durabletask-go/task" +) + +var workItemFilterBenchmarkSizes = [...]int{4, 64, 512} + +// benchmarkRegistrySnapshot builds a snapshot with size orchestrator and +// activity names plus entities. Every other orchestrator carries two explicit +// versions so filter construction exercises the version-grouping path. +func benchmarkRegistrySnapshot(b *testing.B, size int) task.TaskRegistrySnapshot { + b.Helper() + registry := task.NewTaskRegistry() + orchestrator := func(*task.OrchestrationContext) (any, error) { return nil, nil } + activity := func(task.ActivityContext) (any, error) { return nil, nil } + entity := func(ctx *task.EntityContext) (any, error) { return nil, nil } + for i := 0; i < size; i++ { + name := fmt.Sprintf("Orchestrator%d", i) + if err := registry.AddOrchestratorN(name, orchestrator); err != nil { + b.Fatalf("add orchestrator: %v", err) + } + if err := registry.AddActivityN(fmt.Sprintf("Activity%d", i), activity); err != nil { + b.Fatalf("add activity: %v", err) + } + if err := registry.AddEntityN(fmt.Sprintf("Entity%d", i), entity); err != nil { + b.Fatalf("add entity: %v", err) + } + if i%2 != 0 { + continue + } + for _, version := range []string{"1.0", "2.0"} { + if err := registry.AddOrchestratorNVersion(name, version, orchestrator); err != nil { + b.Fatalf("add versioned orchestrator: %v", err) + } + } + } + return registry.Snapshot() +} + +// BenchmarkWorkItemFiltersFromRegistry measures the automatic filter derivation +// a worker performs once per start, for both version match strategies. +func BenchmarkWorkItemFiltersFromRegistry(b *testing.B) { + strict := &task.VersioningOptions{Version: "1.0", MatchStrategy: task.VersionMatchStrict} + for _, size := range workItemFilterBenchmarkSizes { + snapshot := benchmarkRegistrySnapshot(b, size) + b.Run(fmt.Sprintf("unversioned/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + filters := workItemFiltersFromRegistry(snapshot, nil, nil, nil) + if len(filters.Orchestrations) == 0 { + b.Fatal("no orchestration filters were derived") + } + } + }) + b.Run(fmt.Sprintf("strict/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + filters := workItemFiltersFromRegistry(snapshot, strict, nil, nil) + if len(filters.Orchestrations) == 0 { + b.Fatal("no orchestration filters were derived") + } + } + }) + } +} + +// BenchmarkWorkItemFiltersToProto measures the wire conversion of the derived +// filters, which is sent on every worker stream generation. +func BenchmarkWorkItemFiltersToProto(b *testing.B) { + for _, size := range workItemFilterBenchmarkSizes { + filters := workItemFiltersFromRegistry(benchmarkRegistrySnapshot(b, size), nil, nil, nil) + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + wire := workItemFiltersToProto(filters) + if wire == nil { + b.Fatal("filters did not convert to the wire form") + } + } + }) + } +} + +// BenchmarkValidateWorkItemFilters measures the validation applied to explicit +// filter configuration at worker construction. +func BenchmarkValidateWorkItemFilters(b *testing.B) { + for _, size := range workItemFilterBenchmarkSizes { + snapshot := benchmarkRegistrySnapshot(b, size) + filters := workItemFiltersFromRegistry(snapshot, nil, nil, nil) + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := validateWorkItemFilters(filters, snapshot); err != nil { + b.Fatalf("validate filters: %v", err) + } + } + }) + } +} + +// BenchmarkMatchesWorkItemFilters measures the per-work-item filter check that +// runs on the worker's hot path. +func BenchmarkMatchesWorkItemFilters(b *testing.B) { + for _, size := range workItemFilterBenchmarkSizes { + filters := workItemFiltersFromRegistry(benchmarkRegistrySnapshot(b, size), nil, nil, nil) + name := fmt.Sprintf("Orchestrator%d", size/2) + b.Run(fmt.Sprintf("accept/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if !matchesWorkItemFilters(filters, true, name, "") { + b.Fatalf("filters rejected registered orchestration %q", name) + } + } + }) + b.Run(fmt.Sprintf("reject/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if matchesWorkItemFilters(filters, true, "Absent", "") { + b.Fatal("filters accepted an unregistered orchestration") + } + } + }) + } +} diff --git a/client/grpc_worker_compat.go b/client/grpc_worker_compat.go new file mode 100644 index 00000000..600f51df --- /dev/null +++ b/client/grpc_worker_compat.go @@ -0,0 +1,60 @@ +package client + +import ( + "context" + "fmt" + + "github.com/microsoft/durabletask-go/task" +) + +// StartWorkItemListener preserves the original client API. New applications +// should create a dedicated TaskHubGrpcWorker, ideally with its own connection. +// +// The listener borrows the client's connection, so it inherits the connection +// ownership limitations documented on [NewTaskHubGrpcWorker]. +func (c *TaskHubGrpcClient) StartWorkItemListener( + ctx context.Context, + registry *task.TaskRegistry, + workerOptions ...TaskHubGrpcWorkerOption, +) error { + c.listenerMu.Lock() + defer c.listenerMu.Unlock() + if c.listener != nil && c.listener.Running() { + return ErrTaskHubGrpcWorkerAlreadyRunning + } + + listenerOptions := []TaskHubGrpcWorkerOption{WithWorkerDataConverter(c.converter)} + if c.largePayloads != nil { + listenerOptions = append(listenerOptions, WithWorkerLargePayloads(c.largePayloads)) + } + listenerOptions = append(listenerOptions, workerOptions...) + worker, err := NewTaskHubGrpcWorker(c.connection, registry, c.logger, listenerOptions...) + if err != nil { + return fmt.Errorf("failed to create gRPC worker: %w", err) + } + if err := worker.Start(ctx); err != nil { + return err + } + c.listener = worker + return nil +} + +// StopWorkItemListener gracefully stops the compatibility listener. +func (c *TaskHubGrpcClient) StopWorkItemListener(ctx context.Context) error { + c.listenerMu.Lock() + worker := c.listener + c.listenerMu.Unlock() + if worker == nil { + return nil + } + if err := worker.Shutdown(ctx); err != nil { + return err + } + + c.listenerMu.Lock() + if c.listener == worker { + c.listener = nil + } + c.listenerMu.Unlock() + return nil +} diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go new file mode 100644 index 00000000..0fd2e53d --- /dev/null +++ b/client/grpc_worker_processor.go @@ -0,0 +1,929 @@ +package client + +import ( + "context" + "errors" + "fmt" + "io" + "runtime/debug" + "slices" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/contextprop" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func (w *TaskHubGrpcWorker) consumeConnection(run *grpcWorkerRun, connection *grpcWorkerConnection) (bool, error) { + observedMessage := false + for { + workItem, err := w.receiveWorkItem(connection) + if err != nil { + return observedMessage, err + } + observedMessage = true + + switch request := workItem.Request.(type) { + case *protos.WorkItem_HealthPing: + continue + case *protos.WorkItem_OrchestratorRequest: + if err := w.dispatchOrchestration(run, connection, workItem.GetCompletionToken(), request.OrchestratorRequest); err != nil { + return observedMessage, err + } + case *protos.WorkItem_ActivityRequest: + if err := w.dispatchActivity(run, connection, workItem.GetCompletionToken(), request.ActivityRequest); err != nil { + return observedMessage, err + } + case *protos.WorkItem_EntityRequest: + if err := w.dispatchEntity(run, connection, workItem.GetCompletionToken(), func(ctx context.Context) { + w.processEntityBatch(ctx, connection.client, workItem.GetCompletionToken(), request.EntityRequest, nil) + }); err != nil { + return observedMessage, err + } + case *protos.WorkItem_EntityRequestV2: + if err := w.dispatchEntity(run, connection, workItem.GetCompletionToken(), func(ctx context.Context) { + w.processEntityV2(ctx, connection.client, workItem.GetCompletionToken(), request.EntityRequestV2) + }); err != nil { + return observedMessage, err + } + default: + w.logger.Warnf("received unknown work item type with completion token present=%t", workItem.GetCompletionToken() != "") + } + } +} + +func (w *TaskHubGrpcWorker) receiveWorkItem(connection *grpcWorkerConnection) (*protos.WorkItem, error) { + workItem, err := recvBeforeSilenceTimeout(connection.stream.Recv, connection.cancelStream, w.options.silentDisconnectTimeout) + if err != nil { + return nil, err + } + if workItem == nil { + return nil, status.Error(codes.Internal, "received a nil work item") + } + return workItem, nil +} + +// recvBeforeSilenceTimeout receives one message from a server stream, canceling +// the stream and reporting errSilentDisconnect if nothing arrives before the +// configured silent disconnect timeout. A message that is delivered +// concurrently with the timeout is still returned: the stream is already +// canceled and the following receive reports the disconnect, so the caller +// never silently drops a delivered work item that the service considers +// dispatched. +// +// A terminal error that races the timeout is only rewritten to +// errSilentDisconnect when it is the cancellation the timer itself induced. +// Any other status, such as Unauthenticated or PermissionDenied, is propagated +// rather than masked as a silent disconnect. +func recvBeforeSilenceTimeout[T any](recv func() (T, error), cancelStream context.CancelFunc, timeout time.Duration) (T, error) { + timedOut := make(chan struct{}) + timer := time.AfterFunc(timeout, func() { + cancelStream() + close(timedOut) + }) + message, err := recv() + if timer.Stop() { + return message, err + } + <-timedOut + if err == nil { + return message, nil + } + if !isStreamCancellation(err) { + return message, err + } + var zero T + return zero, errSilentDisconnect +} + +// isStreamCancellation reports whether err is the context cancellation that +// canceling the stream produces, as opposed to a status the service returned on +// its own. +func isStreamCancellation(err error) bool { + return errors.Is(err, context.Canceled) || status.Code(err) == codes.Canceled +} + +func (w *TaskHubGrpcWorker) dispatchOrchestration( + run *grpcWorkerRun, + connection *grpcWorkerConnection, + completionToken string, + request *protos.OrchestratorRequest, +) error { + return w.dispatch(run, connection, run.orchestrationSlots, + func(ctx context.Context) { w.abandonOrchestration(ctx, connection.client, completionToken) }, + func(ctx context.Context) { w.processOrchestration(ctx, connection.client, completionToken, request) }, + ) +} + +func (w *TaskHubGrpcWorker) dispatchActivity( + run *grpcWorkerRun, + connection *grpcWorkerConnection, + completionToken string, + request *protos.ActivityRequest, +) error { + return w.dispatch(run, connection, run.activitySlots, + func(ctx context.Context) { w.abandonActivity(ctx, connection.client, completionToken) }, + func(ctx context.Context) { w.processActivity(ctx, connection.client, completionToken, request) }, + ) +} + +func (w *TaskHubGrpcWorker) dispatchEntity( + run *grpcWorkerRun, + connection *grpcWorkerConnection, + completionToken string, + process func(context.Context), +) error { + return w.dispatch( + run, + connection, + run.entitySlots, + func(ctx context.Context) { w.abandonEntity(ctx, connection.client, completionToken) }, + process, + ) +} + +// dispatch reserves a concurrency slot and runs process in the background under +// the processing context, so a graceful drain can still complete in-flight work. +// If intake is canceled before a slot is free, the work item is abandoned instead. +func (w *TaskHubGrpcWorker) dispatch( + run *grpcWorkerRun, + connection *grpcWorkerConnection, + slots chan struct{}, + abandon func(context.Context), + process func(context.Context), +) error { + select { + case slots <- struct{}{}: + case <-run.intakeCtx.Done(): + abandon(run.processingCtx) + return run.intakeCtx.Err() + } + + run.pending.Add(1) + connection.pending.Add(1) + go func() { + defer func() { + <-slots + connection.pending.Done() + run.pending.Done() + }() + process(run.processingCtx) + }() + return nil +} + +func (w *TaskHubGrpcWorker) processOrchestration( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, + request *protos.OrchestratorRequest, +) { + pastEvents := request.PastEvents + if request.RequiresHistoryStreaming { + history, err := w.streamHistory(ctx, client, request) + if err != nil { + w.logger.Errorf("%s: failed to stream required orchestration history: %v", request.InstanceId, err) + if errors.Is(err, ErrStreamedHistoryLimitExceeded) { + _ = w.wait(ctx, w.options.transientRetryMaxDelay) + } + w.abandonOrchestration(ctx, client, completionToken) + return + } + pastEvents = history + } + payloadRequest := &protos.OrchestratorRequest{ + PastEvents: pastEvents, + NewEvents: request.NewEvents, + } + if err := largepayload.TransformOrchestratorRequest(ctx, w.options.largePayloads, payloadRequest); err != nil { + w.logger.Errorf("%s: failed to hydrate orchestration work item payloads: %v", request.InstanceId, err) + w.abandonOrchestration(ctx, client, completionToken) + return + } + if w.options.workItemFilters != nil { + name, version, ok := orchestrationWorkItemIdentity(pastEvents, request.NewEvents) + if ok && !matchesWorkItemFilters(w.options.workItemFilters, true, name, version) { + w.logger.Warnf("%s: orchestration work item does not match configured filters; abandoning it", request.InstanceId) + _ = w.wait(ctx, w.options.reconnectBaseDelay) + w.abandonOrchestration(ctx, client, completionToken) + return + } + } + + results, err := w.executor.ExecuteOrchestrator( + ctx, + api.InstanceID(request.InstanceId), + pastEvents, + request.NewEvents, + request.EntityParameters, + ) + var delayed workItemAbandonDelayError + if errors.As(err, &delayed) { + w.logger.Warnf("%s: orchestration work item rejected; abandoning it: %v", request.InstanceId, err) + _ = w.wait(ctx, delayed.WorkItemAbandonDelay()) + w.abandonOrchestration(ctx, client, completionToken) + return + } + if err != nil && ctx.Err() != nil { + w.logger.Warnf("%s: orchestration execution canceled; abandoning work item", request.InstanceId) + w.abandonOrchestration(ctx, client, completionToken) + return + } + + response := &protos.OrchestratorResponse{ + InstanceId: request.InstanceId, + CompletionToken: completionToken, + OrchestrationTraceContext: request.OrchestrationTraceContext, + } + switch { + case err != nil: + response.Actions = []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( + -1, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + wrapperspb.String("An internal error occurred while executing the orchestration."), + nil, + failure.FromError(err), + )} + case results == nil || results.Response == nil: + response.Actions = []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( + -1, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + wrapperspb.String("The orchestration executor returned no response."), + nil, + &protos.TaskFailureDetails{ + ErrorType: "MissingOrchestratorResponse", + ErrorMessage: "the orchestration executor returned no response", + }, + )} + default: + response = results.Response + response.InstanceId = request.InstanceId + response.CompletionToken = completionToken + if response.OrchestrationTraceContext == nil { + response.OrchestrationTraceContext = request.OrchestrationTraceContext + } + if err := populateOrchestratorActionTraceContexts( + response.Actions, + pastEvents, + request.NewEvents, + ); err != nil { + w.logger.Warnf("%s: failed to populate action trace contexts: %v", request.InstanceId, err) + } + } + if err := largepayload.TransformOrchestratorResponse(ctx, w.options.largePayloads, response); err != nil { + w.logger.Errorf("%s: failed to externalize orchestration response payloads: %v", request.InstanceId, err) + w.abandonOrchestration(ctx, client, completionToken) + return + } + + response = failOversizedOrchestratorResponse(response, w.options.maxOrchestratorCompletionBytes) + err = w.executeRPCWithRetry(ctx, "complete orchestration task", func(callCtx context.Context) error { + _, callErr := client.CompleteOrchestratorTask(callCtx, response) + return callErr + }) + if err != nil { + if isWorkItemGone(err) { + w.logger.Warnf("%s: orchestration work item was no longer available before completion", request.InstanceId) + return + } + w.logger.Errorf("%s: failed to complete orchestration work item: %v", request.InstanceId, err) + w.abandonOrchestration(ctx, client, completionToken) + } +} + +// failOversizedOrchestratorResponse converts a response that modern DTS cannot +// accept into a small, non-retriable orchestration failure. The Go worker does +// not use the deprecated legacy response-chunking fields; validation against the +// current DTS emulator confirmed that splitting is not a portable recovery path. +func failOversizedOrchestratorResponse( + response *protos.OrchestratorResponse, + maxBytes int, +) *protos.OrchestratorResponse { + if response == nil || maxBytes <= 0 { + return response + } + responseSize := proto.Size(response) + if responseSize <= maxBytes { + return response + } + boundMiB := float64(maxBytes) / (1024 * 1024) + message := fmt.Sprintf( + "orchestrator response with %d actions exceeds the %.2f MiB worker completion bound: %.2f MiB; reduce per-turn fan-out or configure large-payload externalization for action payloads", + len(response.Actions), boundMiB, float64(responseSize)/(1024*1024), + ) + for _, action := range response.Actions { + if actionSize := proto.Size(action); actionSize > maxBytes { + message = fmt.Sprintf( + "orchestrator action %d exceeds the %.2f MiB worker completion bound: %.2f MiB; configure large-payload externalization for oversized action payloads", + action.GetId(), boundMiB, float64(actionSize)/(1024*1024), + ) + break + } + } + return &protos.OrchestratorResponse{ + InstanceId: response.InstanceId, + CompletionToken: response.CompletionToken, + OrchestrationTraceContext: response.OrchestrationTraceContext, + Actions: []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( + -1, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + nil, + nil, + &protos.TaskFailureDetails{ + ErrorType: string(api.ErrorTypeOrchestratorResponseTooLarge), + ErrorMessage: message, + IsNonRetriable: true, + }, + )}, + } +} + +func populateOrchestratorActionTraceContexts( + actions []*protos.OrchestratorAction, + pastEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, +) error { + parent := executionStartedTraceContext(pastEvents, newEvents) + if parent == nil { + return nil + } + for _, action := range actions { + if action == nil { + continue + } + switch { + case action.GetScheduleTask() != nil: + scheduled := action.GetScheduleTask() + if scheduled.ParentTraceContext == nil { + traceContext, err := helpers.OrchestratorActionTraceContext(parent) + if err != nil { + return fmt.Errorf("schedule task action %d: %w", action.Id, err) + } + scheduled.ParentTraceContext = traceContext + } + case action.GetCreateSubOrchestration() != nil: + created := action.GetCreateSubOrchestration() + if created.ParentTraceContext == nil { + traceContext, err := helpers.OrchestratorActionTraceContext(parent) + if err != nil { + return fmt.Errorf("create sub-orchestration action %d: %w", action.Id, err) + } + created.ParentTraceContext = traceContext + } + } + } + return nil +} + +func executionStartedTraceContext(eventLists ...[]*protos.HistoryEvent) *protos.TraceContext { + for _, events := range eventLists { + for _, event := range events { + if started := event.GetExecutionStarted(); started != nil { + return started.GetParentTraceContext() + } + } + } + return nil +} + +func (w *TaskHubGrpcWorker) streamHistory( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + request *protos.OrchestratorRequest, +) ([]*protos.HistoryEvent, error) { + historyCtx, cancelHistory := context.WithCancel(ctx) + defer cancelHistory() + stream, err := client.StreamInstanceHistory(historyCtx, &protos.StreamInstanceHistoryRequest{ + InstanceId: request.InstanceId, + ExecutionId: request.ExecutionId, + ForWorkItemProcessing: true, + }) + if err != nil { + return nil, err + } + + var ( + history []*protos.HistoryEvent + historyBytes int64 + ) + for { + chunk, recvErr := recvBeforeSilenceTimeout(stream.Recv, cancelHistory, w.options.silentDisconnectTimeout) + if errors.Is(recvErr, io.EOF) { + return history, nil + } + if recvErr != nil { + return nil, recvErr + } + if chunk == nil { + return nil, status.Error(codes.Internal, "received a nil history chunk") + } + if limit := w.options.maxStreamedHistoryEvents; limit > 0 && + len(chunk.Events) > limit-len(history) { + return nil, newStreamedHistoryLimitError( + "streamed history exceeds %d events", + limit, + ) + } + chunkBytes := int64(proto.Size(chunk)) + if limit := w.options.maxStreamedHistoryBytes; limit > 0 && + chunkBytes > limit-historyBytes { + return nil, newStreamedHistoryLimitError( + "streamed history exceeds %d bytes", + limit, + ) + } + history = append(history, chunk.Events...) + historyBytes += chunkBytes + } +} + +func (w *TaskHubGrpcWorker) processActivity( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, + request *protos.ActivityRequest, +) { + if request.OrchestrationInstance == nil { + w.logger.Error("received activity work item without an orchestration instance; abandoning it") + w.abandonActivity(ctx, client, completionToken) + return + } + if err := largepayload.TransformActivityRequest(ctx, w.options.largePayloads, request); err != nil { + w.logger.Errorf( + "%s/%s#%d: failed to hydrate activity payload: %v", + request.OrchestrationInstance.InstanceId, + request.Name, + request.TaskId, + err, + ) + w.abandonActivity(ctx, client, completionToken) + return + } + if !matchesWorkItemFilters(w.options.workItemFilters, false, request.Name, request.Version.GetValue()) { + w.logger.Warnf( + "%s/%s#%d: activity work item does not match configured filters; abandoning it", + request.OrchestrationInstance.InstanceId, + request.Name, + request.TaskId, + ) + _ = w.wait(ctx, w.options.reconnectBaseDelay) + w.abandonActivity(ctx, client, completionToken) + return + } + + event := helpers.NewTaskScheduledEvent( + request.TaskId, + request.Name, + request.Version, + request.Input, + request.ParentTraceContext, + ) + event.GetTaskScheduled().Tags = contextprop.Clone(request.Tags) + result, err := w.executor.ExecuteActivity(ctx, api.InstanceID(request.OrchestrationInstance.InstanceId), event) + var delayed workItemAbandonDelayError + if errors.As(err, &delayed) { + w.logger.Warnf( + "%s/%s#%d: activity work item rejected; abandoning it: %v", + request.OrchestrationInstance.InstanceId, + request.Name, + request.TaskId, + err, + ) + _ = w.wait(ctx, delayed.WorkItemAbandonDelay()) + w.abandonActivity(ctx, client, completionToken) + return + } + + if err != nil && ctx.Err() != nil { + w.logger.Warnf("%s/%s#%d: activity execution canceled; abandoning work item", request.OrchestrationInstance.InstanceId, request.Name, request.TaskId) + w.abandonActivity(ctx, client, completionToken) + return + } + + response := &protos.ActivityResponse{ + InstanceId: request.OrchestrationInstance.InstanceId, + TaskId: request.TaskId, + CompletionToken: completionToken, + } + if err != nil { + response.FailureDetails = failure.FromError(err) + } else if completed := result.GetTaskCompleted(); completed != nil { + response.Result = completed.Result + } else if failed := result.GetTaskFailed(); failed != nil { + response.FailureDetails = failed.FailureDetails + } else { + response.FailureDetails = &protos.TaskFailureDetails{ + ErrorType: "UnknownTaskResult", + ErrorMessage: "activity executor returned an unknown task result", + } + } + if err := largepayload.TransformActivityResponse(ctx, w.options.largePayloads, response); err != nil { + w.logger.Errorf( + "%s/%s#%d: failed to externalize activity response payload: %v", + request.OrchestrationInstance.InstanceId, + request.Name, + request.TaskId, + err, + ) + w.abandonActivity(ctx, client, completionToken) + return + } + + err = w.executeRPCWithRetry(ctx, "complete activity task", func(callCtx context.Context) error { + _, callErr := client.CompleteActivityTask(callCtx, response) + return callErr + }) + if err != nil { + if isWorkItemGone(err) { + w.logger.Warnf( + "%s/%s#%d: activity work item was no longer available before completion", + request.OrchestrationInstance.InstanceId, + request.Name, + request.TaskId, + ) + return + } + w.logger.Errorf("%s/%s#%d: failed to complete activity work item: %v", request.OrchestrationInstance.InstanceId, request.Name, request.TaskId, err) + w.abandonActivity(ctx, client, completionToken) + } +} + +func (w *TaskHubGrpcWorker) processEntityV2( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, + request *protos.EntityRequest, +) { + batch, operationInfos, err := entityBatchFromRequestV2(request) + if err != nil { + w.logger.Errorf("invalid V2 entity work item: %v", err) + w.abandonEntity(ctx, client, completionToken) + return + } + w.processEntityBatch(ctx, client, completionToken, batch, operationInfos) +} + +func (w *TaskHubGrpcWorker) processEntityBatch( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, + request *protos.EntityBatchRequest, + operationInfos []*protos.OperationInfo, +) { + entityID, parseErr := api.EntityIDFromString(request.GetInstanceId()) + if parseErr != nil { + w.logger.Errorf("%s: invalid entity instance ID: %v", request.GetInstanceId(), parseErr) + w.abandonEntity(ctx, client, completionToken) + return + } + if !matchesEntityWorkItemFilters(w.options.workItemFilters, entityID.Name) { + w.logger.Warnf("%s: entity work item does not match configured filters; abandoning it", request.GetInstanceId()) + w.abandonEntity(ctx, client, completionToken) + return + } + if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, request); err != nil { + w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) + if entityProcessingCanceled(ctx, err) { + w.abandonEntity(ctx, client, completionToken) + return + } + w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) + return + } + executor, ok := w.executor.(task.EntityExecutor) + if !ok { + w.logger.Error("task executor does not support entity work items") + w.abandonEntity(ctx, client, completionToken) + return + } + result, err := executeEntitySafely(ctx, executor, request) + if err != nil { + if entityProcessingCanceled(ctx, err) { + w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) + w.abandonEntity(ctx, client, completionToken) + return + } + w.logger.Errorf("%s: entity execution failed: %v", request.GetInstanceId(), err) + result = newEntityBatchFailure(completionToken, err) + } else if result == nil { + missingResultErr := errors.New("entity executor returned no result") + w.logger.Errorf("%s: %v", request.GetInstanceId(), missingResultErr) + result = newEntityBatchFailure(completionToken, missingResultErr) + } + result.CompletionToken = completionToken + if err := validateEntityBatchResult(result, len(request.Operations), operationInfos); err != nil { + w.logger.Errorf("%s: invalid entity executor result: %v", request.GetInstanceId(), err) + result = newEntityBatchFailure(completionToken, err) + } + if err := largepayload.TransformEntityBatchResult(ctx, w.options.largePayloads, result); err != nil { + w.logger.Errorf("%s: failed to externalize entity batch payloads: %v", request.GetInstanceId(), err) + if entityProcessingCanceled(ctx, err) { + w.abandonEntity(ctx, client, completionToken) + return + } + w.completeEntityBatchFailure(ctx, client, request.GetInstanceId(), completionToken, err) + return + } + + w.completeEntityBatchResult(ctx, client, request.GetInstanceId(), result) +} + +func executeEntitySafely( + ctx context.Context, + executor task.EntityExecutor, + request *protos.EntityBatchRequest, +) (result *protos.EntityBatchResult, err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = &entityExecutorPanicError{ + message: fmt.Sprintf("entity executor panic: %v", recovered), + stack: string(debug.Stack()), + } + } + }() + return executor.ExecuteEntity(ctx, request) +} + +type entityExecutorPanicError struct { + message string + stack string +} + +func (e *entityExecutorPanicError) Error() string { + return e.message +} + +func (e *entityExecutorPanicError) DurableTaskStackTrace() string { + return e.stack +} + +func newEntityBatchFailure(completionToken string, err error) *protos.EntityBatchResult { + return &protos.EntityBatchResult{ + CompletionToken: completionToken, + FailureDetails: failure.FromError(err), + } +} + +func (w *TaskHubGrpcWorker) completeEntityBatchFailure( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + instanceID string, + completionToken string, + err error, +) { + w.completeEntityBatchResult(ctx, client, instanceID, newEntityBatchFailure(completionToken, err)) +} + +func validateEntityBatchResult( + result *protos.EntityBatchResult, + operationCount int, + operationInfos []*protos.OperationInfo, +) error { + if result.FailureDetails != nil || result.RequiresState { + if result.FailureDetails != nil && result.RequiresState { + return fmt.Errorf("entity result cannot contain both failure details and a state request") + } + if len(result.Results) != 0 || len(result.Actions) != 0 || result.EntityState != nil || + len(result.OperationInfos) != 0 { + return fmt.Errorf("entity batch failures and state requests must not contain partial effects") + } + result.OperationInfos = nil + return nil + } + if len(result.Results) != operationCount { + return fmt.Errorf( + "entity result count %d does not match operation count %d", + len(result.Results), + operationCount, + ) + } + if len(result.OperationInfos) != 0 { + return fmt.Errorf("entity executor result must not set operation routing metadata") + } + if operationInfos == nil { + return nil + } + if len(operationInfos) != len(result.Results) { + return fmt.Errorf( + "V2 entity result count %d does not match operation count %d", + len(result.Results), + len(operationInfos), + ) + } + result.OperationInfos = append([]*protos.OperationInfo(nil), operationInfos...) + return nil +} + +func entityProcessingCanceled(ctx context.Context, err error) bool { + return ctx.Err() != nil || + errors.Is(err, context.Canceled) || + errors.Is(err, context.DeadlineExceeded) +} + +func (w *TaskHubGrpcWorker) completeEntityBatchResult( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + instanceID string, + result *protos.EntityBatchResult, +) { + err := w.executeRPCWithRetry(ctx, "complete entity task", func(callCtx context.Context) error { + _, callErr := client.CompleteEntityTask(callCtx, result) + return callErr + }) + if err != nil { + if isWorkItemGone(err) { + w.logger.Warnf("%s: entity work item was no longer available before completion", instanceID) + return + } + w.logger.Errorf("%s: failed to complete entity work item: %v", instanceID, err) + w.abandonEntity(ctx, client, result.CompletionToken) + } +} + +func orchestrationWorkItemIdentity(pastEvents, newEvents []*protos.HistoryEvent) (string, string, bool) { + for _, events := range [][]*protos.HistoryEvent{newEvents, pastEvents} { + for i := len(events) - 1; i >= 0; i-- { + event := events[i] + if event == nil { + continue + } + if rewound := event.GetExecutionRewound(); rewound != nil && rewound.GetName().GetValue() != "" { + return rewound.GetName().GetValue(), rewound.GetVersion().GetValue(), true + } + if started := event.GetExecutionStarted(); started != nil { + return started.GetName(), started.GetVersion().GetValue(), true + } + } + } + return "", "", false +} + +func matchesWorkItemFilters(filters *WorkItemFilters, orchestration bool, name, version string) bool { + if filters == nil { + return true + } + candidates := filters.Activities + rejectAll := filters.RejectAllActivities + if orchestration { + candidates = filters.Orchestrations + rejectAll = filters.RejectAllOrchestrations + } + if rejectAll { + return false + } + if len(candidates) == 0 { + return true + } + for _, filter := range candidates { + if !strings.EqualFold(filter.Name, name) { + continue + } + if len(filter.Versions) == 0 || slices.ContainsFunc(filter.Versions, func(candidate string) bool { + return strings.EqualFold(candidate, version) + }) { + return true + } + } + return false +} + +func matchesEntityWorkItemFilters(filters *WorkItemFilters, name string) bool { + if filters == nil { + return true + } + if filters.RejectAllEntities { + return false + } + if len(filters.Entities) == 0 { + return true + } + return slices.Contains(filters.Entities, name) +} + +func (w *TaskHubGrpcWorker) abandonOrchestration( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, +) { + if completionToken == "" { + w.logger.Warn("cannot abandon orchestration work item without a completion token") + return + } + if err := w.executeRPCWithRetry(ctx, "abandon orchestration task", func(callCtx context.Context) error { + _, callErr := client.AbandonTaskOrchestratorWorkItem(callCtx, &protos.AbandonOrchestrationTaskRequest{ + CompletionToken: completionToken, + }) + return callErr + }); err != nil { + w.logger.Errorf("failed to abandon orchestration work item: %v", err) + } +} + +func (w *TaskHubGrpcWorker) abandonActivity( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, +) { + if completionToken == "" { + w.logger.Warn("cannot abandon activity work item without a completion token") + return + } + if err := w.executeRPCWithRetry(ctx, "abandon activity task", func(callCtx context.Context) error { + _, callErr := client.AbandonTaskActivityWorkItem(callCtx, &protos.AbandonActivityTaskRequest{ + CompletionToken: completionToken, + }) + return callErr + }); err != nil { + w.logger.Errorf("failed to abandon activity work item: %v", err) + } +} + +func (w *TaskHubGrpcWorker) abandonEntity( + ctx context.Context, + client protos.TaskHubSidecarServiceClient, + completionToken string, +) { + if completionToken == "" { + w.logger.Warn("cannot abandon unsupported entity work item without a completion token") + return + } + abandonCtx := context.WithoutCancel(ctx) + if err := w.executeRPCWithRetry(abandonCtx, "abandon entity task", func(callCtx context.Context) error { + _, callErr := client.AbandonTaskEntityWorkItem(callCtx, &protos.AbandonEntityTaskRequest{ + CompletionToken: completionToken, + }) + return callErr + }); err != nil { + w.logger.Errorf("failed to abandon unsupported entity work item: %v", err) + } +} + +func (w *TaskHubGrpcWorker) executeRPCWithRetry( + ctx context.Context, + operation string, + action func(context.Context) error, +) error { + var lastErr error + for attempt := 1; attempt <= w.options.transientRetryMaxAttempts; attempt++ { + callCtx, cancel := context.WithTimeout(ctx, w.options.rpcTimeout) + lastErr = action(callCtx) + cancel() + if lastErr == nil { + return nil + } + if ctx.Err() != nil { + return ctx.Err() + } + if !isTransientWorkerRPCError(lastErr) { + return fmt.Errorf("%s failed with a non-retryable error: %w", operation, lastErr) + } + if attempt == w.options.transientRetryMaxAttempts { + break + } + if err := w.wait(ctx, w.retryDelay(attempt)); err != nil { + return err + } + } + return fmt.Errorf("%s failed after %d attempts: %w", operation, w.options.transientRetryMaxAttempts, lastErr) +} + +// retryDelay returns the deterministic delay before the given attempt. Delays +// double from the configured base and never leave [base, max]. +func (w *TaskHubGrpcWorker) retryDelay(attempt int) time.Duration { + delay := w.options.transientRetryBaseDelay + for i := 1; i < attempt; i++ { + next := doubleDurationBounded(delay, w.options.transientRetryMaxDelay) + if next == delay { + break + } + delay = next + } + return delay +} + +func isTransientWorkerRPCError(err error) bool { + if errors.Is(err, ErrStreamedHistoryLimitExceeded) { + return false + } + grpcStatus, ok := status.FromError(err) + if !ok { + return false + } + if grpcStatus.Code() == codes.NotFound { + return false + } + return isTransientWorkerGRPCStatus(grpcStatus) +} + +func isWorkItemGone(err error) bool { + return status.Code(err) == codes.NotFound +} diff --git a/client/grpc_worker_test.go b/client/grpc_worker_test.go new file mode 100644 index 00000000..79ed0c85 --- /dev/null +++ b/client/grpc_worker_test.go @@ -0,0 +1,1793 @@ +package client + +import ( + "context" + "errors" + "io" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type fakeWorkItemResult struct { + item *protos.WorkItem + err error +} + +type fakeWorkItemsStream struct { + protos.TaskHubSidecarService_GetWorkItemsClient + ctx context.Context + results chan fakeWorkItemResult +} + +func (s *fakeWorkItemsStream) Recv() (*protos.WorkItem, error) { + select { + case <-s.ctx.Done(): + return nil, s.ctx.Err() + case result := <-s.results: + return result.item, result.err + } +} + +type fakeHistoryStream struct { + protos.TaskHubSidecarService_StreamInstanceHistoryClient + ctx context.Context + chunks []*protos.HistoryChunk + block bool + index int +} + +func (s *fakeHistoryStream) Recv() (*protos.HistoryChunk, error) { + if s.index < len(s.chunks) { + chunk := s.chunks[s.index] + s.index++ + return chunk, nil + } + if s.block { + <-s.ctx.Done() + return nil, s.ctx.Err() + } + return nil, io.EOF +} + +func TestMaximumTimerIntervalWorkerOption(t *testing.T) { + options := defaultTaskHubGrpcWorkerOptions() + require.Nil(t, options.maximumTimerInterval) + require.Empty(t, options.executorOptions()) + require.NoError(t, WithTaskExecutorOptions(task.WithMaximumTimerInterval(time.Hour))(&options)) + require.Len(t, options.executorOptions(), 1) + require.Error(t, WithMaximumTimerInterval(-time.Second)(&options)) + require.NoError(t, WithMaximumTimerInterval(0)(&options)) + require.Equal(t, task.DefaultMaximumTimerInterval, *options.maximumTimerInterval) + require.Len(t, options.executorOptions(), 2) + require.NoError(t, WithMaximumTimerInterval(2*time.Hour)(&options)) + require.Equal(t, 2*time.Hour, *options.maximumTimerInterval) +} + +type fakeSchedulerClient struct { + protos.TaskHubSidecarServiceClient + + mu sync.Mutex + + helloErr error + stream *fakeWorkItemsStream + request *protos.GetWorkItemsRequest + + history []*protos.HistoryChunk + historyBlocks bool + + orchestrationCompletions []*protos.OrchestratorResponse + activityCompletions []*protos.ActivityResponse + entityCompletions []*protos.EntityBatchResult + activityCompletionErr error + entityCompletionErr error + orchestrationAbandons int + activityAbandons int + entityAbandonAttempts int + entityAbandonFailures int + entityAbandonContextErr error +} + +func (c *fakeSchedulerClient) Hello(context.Context, *emptypb.Empty, ...grpc.CallOption) (*emptypb.Empty, error) { + return &emptypb.Empty{}, c.helloErr +} + +func (c *fakeSchedulerClient) GetWorkItems( + ctx context.Context, + request *protos.GetWorkItemsRequest, + _ ...grpc.CallOption, +) (protos.TaskHubSidecarService_GetWorkItemsClient, error) { + c.mu.Lock() + c.request = request + c.stream.ctx = ctx + c.mu.Unlock() + return c.stream, nil +} + +func (c *fakeSchedulerClient) CompleteOrchestratorTask( + _ context.Context, + response *protos.OrchestratorResponse, + _ ...grpc.CallOption, +) (*protos.CompleteTaskResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.orchestrationCompletions = append(c.orchestrationCompletions, response) + return &protos.CompleteTaskResponse{}, nil +} + +func (c *fakeSchedulerClient) CompleteActivityTask( + _ context.Context, + response *protos.ActivityResponse, + _ ...grpc.CallOption, +) (*protos.CompleteTaskResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.activityCompletions = append(c.activityCompletions, response) + return &protos.CompleteTaskResponse{}, c.activityCompletionErr +} + +func (c *fakeSchedulerClient) CompleteEntityTask( + _ context.Context, + response *protos.EntityBatchResult, + _ ...grpc.CallOption, +) (*protos.CompleteTaskResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.entityCompletions = append(c.entityCompletions, response) + return &protos.CompleteTaskResponse{}, c.entityCompletionErr +} + +func (c *fakeSchedulerClient) StreamInstanceHistory( + ctx context.Context, + _ *protos.StreamInstanceHistoryRequest, + _ ...grpc.CallOption, +) (protos.TaskHubSidecarService_StreamInstanceHistoryClient, error) { + return &fakeHistoryStream{ctx: ctx, chunks: c.history, block: c.historyBlocks}, nil +} + +func (c *fakeSchedulerClient) AbandonTaskOrchestratorWorkItem( + _ context.Context, + _ *protos.AbandonOrchestrationTaskRequest, + _ ...grpc.CallOption, +) (*protos.AbandonOrchestrationTaskResponse, error) { + c.mu.Lock() + c.orchestrationAbandons++ + c.mu.Unlock() + return &protos.AbandonOrchestrationTaskResponse{}, nil +} + +func (c *fakeSchedulerClient) AbandonTaskActivityWorkItem( + _ context.Context, + _ *protos.AbandonActivityTaskRequest, + _ ...grpc.CallOption, +) (*protos.AbandonActivityTaskResponse, error) { + c.mu.Lock() + c.activityAbandons++ + c.mu.Unlock() + return &protos.AbandonActivityTaskResponse{}, nil +} + +func (c *fakeSchedulerClient) AbandonTaskEntityWorkItem( + ctx context.Context, + _ *protos.AbandonEntityTaskRequest, + _ ...grpc.CallOption, +) (*protos.AbandonEntityTaskResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + c.entityAbandonContextErr = ctx.Err() + c.entityAbandonAttempts++ + if c.entityAbandonAttempts <= c.entityAbandonFailures { + return nil, status.Error(codes.Unavailable, "transient") + } + return &protos.AbandonEntityTaskResponse{}, nil +} + +func TestTaskHubGrpcWorkerAbandonsEntityWithIndependentContext(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + worker.abandonEntity(ctx, client, "token") + + require.Equal(t, 1, client.entityAbandonAttempts) + require.NoError(t, client.entityAbandonContextErr) +} + +type recordingExecutor struct { + executeOrchestrator func(context.Context, api.InstanceID, []*protos.HistoryEvent, []*protos.HistoryEvent) (*task.ExecutionResults, error) + executeActivity func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) + executeEntity func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) + entityParameters *protos.OrchestratorEntityParameters +} + +func (e *recordingExecutor) ExecuteOrchestrator( + ctx context.Context, + id api.InstanceID, + pastEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*task.ExecutionResults, error) { + e.entityParameters = entityParameters + return e.executeOrchestrator(ctx, id, pastEvents, newEvents) +} + +func (e *recordingExecutor) ExecuteActivity( + ctx context.Context, + id api.InstanceID, + event *protos.HistoryEvent, +) (*protos.HistoryEvent, error) { + return e.executeActivity(ctx, id, event) +} + +func (*recordingExecutor) Shutdown(context.Context) error { + return nil +} + +func (e *recordingExecutor) ExecuteEntity( + ctx context.Context, + request *protos.EntityBatchRequest, +) (*protos.EntityBatchResult, error) { + return e.executeEntity(ctx, request) +} + +func newFakeWorker(t *testing.T, client *fakeSchedulerClient, opts ...TaskHubGrpcWorkerOption) *TaskHubGrpcWorker { + t.Helper() + registry := task.NewTaskRegistry() + options := []TaskHubGrpcWorkerOption{ + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(time.Second), + WithWorkerRPCTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + WithWorkerTransientRetryPolicy(3, time.Millisecond, 5*time.Millisecond), + } + options = append(options, opts...) + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + }, + registry, + api.DefaultLogger(), + options..., + ) + require.NoError(t, err) + return worker +} + +func newFakeWorkItemStream(buffer int) *fakeWorkItemsStream { + return &fakeWorkItemsStream{results: make(chan fakeWorkItemResult, buffer)} +} + +type panickingWorkerResult struct{} + +func (panickingWorkerResult) MarshalJSON() ([]byte, error) { + panic("worker result serialization panic") +} + +func TestTaskHubGrpcWorkerContinuesAfterFinalizationPanic(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("panicking-result", func(*task.OrchestrationContext) (any, error) { + return panickingWorkerResult{}, nil + })) + require.NoError(t, registry.AddOrchestratorN("healthy", func(*task.OrchestrationContext) (any, error) { + return "done", nil + })) + stream := newFakeWorkItemStream(2) + scheduler := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, scheduler, WithMaxConcurrentOrchestrationWorkItems(1)) + worker.executor = task.NewTaskExecutor(registry) + for _, name := range []string{"panicking-result", "healthy"} { + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + CompletionToken: name, + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: name, + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(name, name, nil, nil, nil, nil), + }, + }}, + }} + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, worker.Start(ctx)) + t.Cleanup(func() { + shutdownCtx, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + require.NoError(t, worker.Shutdown(shutdownCtx)) + }) + require.Eventually(t, func() bool { + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + return len(scheduler.orchestrationCompletions) == 2 + }, time.Second, time.Millisecond) + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + failed := scheduler.orchestrationCompletions[0] + require.Equal(t, "panicking-result", failed.CompletionToken) + require.Len(t, failed.Actions, 1) + require.Equal(t, string(api.ErrorTypeOrchestratorPanic), + failed.Actions[0].GetCompleteOrchestration().GetFailureDetails().GetErrorType()) + completed := scheduler.orchestrationCompletions[1] + require.Equal(t, "healthy", completed.CompletionToken) + require.Len(t, completed.Actions, 1) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completed.Actions[0].GetCompleteOrchestration().OrchestrationStatus) + require.Equal(t, `"done"`, completed.Actions[0].GetCompleteOrchestration().GetResult().GetValue()) +} + +func TestTaskHubGrpcWorkerContinuesAfterActivityFailureProviderPanic(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("failing", func(task.ActivityContext) (any, error) { + return nil, errors.New("activity failed") + })) + require.NoError(t, registry.AddActivityN("healthy", func(task.ActivityContext) (any, error) { + return "done", nil + })) + var providerCalls atomic.Int32 + executor := task.NewTaskExecutor(registry, task.WithErrorPropertiesProvider( + api.ErrorPropertiesProviderFunc(func(error) map[string]any { + providerCalls.Add(1) + panic("failure provider panicked") + }), + )) + stream := newFakeWorkItemStream(2) + scheduler := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, scheduler, WithMaxConcurrentActivityWorkItems(1)) + worker.executor = executor + for i, name := range []string{"failing", "healthy"} { + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + CompletionToken: name, + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: name, + TaskId: int32(i), + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + }} + } + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, worker.Start(ctx)) + t.Cleanup(func() { + shutdownCtx, stop := context.WithTimeout(context.Background(), time.Second) + defer stop() + require.NoError(t, worker.Shutdown(shutdownCtx)) + }) + require.Eventually(t, func() bool { + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + return len(scheduler.activityCompletions) == 2 + }, time.Second, time.Millisecond) + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + require.EqualValues(t, 1, providerCalls.Load()) + failed, completed := scheduler.activityCompletions[0], scheduler.activityCompletions[1] + require.Equal(t, "failing", failed.CompletionToken) + require.Equal(t, string(api.ErrorTypeActivityPanic), failed.GetFailureDetails().GetErrorType()) + require.Contains(t, failed.GetFailureDetails().GetErrorMessage(), "failure provider panicked") + require.Equal(t, "healthy", completed.CompletionToken) + require.Nil(t, completed.GetFailureDetails()) + require.Equal(t, `"done"`, completed.GetResult().GetValue()) + require.Zero(t, scheduler.activityAbandons) +} + +func TestTaskHubGrpcWorkerAdvertisesCapabilitiesAndCompletesActivity(t *testing.T) { + stream := newFakeWorkItemStream(2) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker( + t, + client, + WithMaxConcurrentOrchestrationWorkItems(2), + WithMaxConcurrentActivityWorkItems(3), + WithMaxConcurrentEntityWorkItems(4), + WithWorkItemFilters(&WorkItemFilters{ + Entities: []string{"Counter"}, + }), + ) + worker.executor = &recordingExecutor{ + executeActivity: func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) { + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{ + TaskCompleted: &protos.TaskCompletedEvent{Result: wrapperspb.String(`"done"`)}, + }, + }, nil + }, + } + + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}, + }} + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + TaskId: 7, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "activity-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.activityCompletions) == 1 + }, time.Second, time.Millisecond) + + client.mu.Lock() + require.EqualValues(t, 2, client.request.MaxConcurrentOrchestrationWorkItems) + require.EqualValues(t, 3, client.request.MaxConcurrentActivityWorkItems) + require.EqualValues(t, 4, client.request.MaxConcurrentEntityWorkItems) + require.Equal(t, "counter", client.request.WorkItemFilters.Entities[0].Name) + require.Equal(t, []protos.WorkerCapability{ + protos.WorkerCapability_WORKER_CAPABILITY_HISTORY_STREAMING, + }, client.request.Capabilities) + require.Equal(t, "activity-token", client.activityCompletions[0].CompletionToken) + client.mu.Unlock() + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestPopulateOrchestratorActionTraceContexts(t *testing.T) { + parent := &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + TraceState: wrapperspb.String("vendor=value"), + } + started := helpers.NewExecutionStartedEvent( + "orchestration", + "instance", + nil, + nil, + parent, + nil, + ) + actions := []*protos.OrchestratorAction{ + helpers.NewScheduleTaskAction(1, "activity", nil), + helpers.NewCreateSubOrchestrationAction(2, "child", "child-id", nil), + } + + require.NoError(t, populateOrchestratorActionTraceContexts( + actions, + []*protos.HistoryEvent{started}, + nil, + )) + activityTrace := actions[0].GetScheduleTask().GetParentTraceContext() + childTrace := actions[1].GetCreateSubOrchestration().GetParentTraceContext() + require.NotNil(t, activityTrace) + require.NotNil(t, childTrace) + require.Contains(t, activityTrace.TraceParent, "0123456789abcdef0123456789abcdef") + require.Contains(t, childTrace.TraceParent, "0123456789abcdef0123456789abcdef") + require.NotEqual(t, parent.TraceParent, activityTrace.TraceParent) + require.NotEqual(t, activityTrace.TraceParent, childTrace.TraceParent) + require.Equal(t, "vendor=value", activityTrace.GetTraceState().GetValue()) + + existing := helpers.NewScheduleTaskAction(3, "existing", nil) + existing.GetScheduleTask().ParentTraceContext = parent + require.NoError(t, populateOrchestratorActionTraceContexts( + []*protos.OrchestratorAction{existing}, + []*protos.HistoryEvent{started}, + nil, + )) + require.Same(t, parent, existing.GetScheduleTask().GetParentTraceContext()) +} + +func TestTaskHubGrpcWorkerSendsNumEventsProcessedForPartialTurn(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("partial", func(ctx *task.OrchestrationContext) (any, error) { + ctx.CallActivity("activity") + return nil, ctx.WaitForSingleEvent("second", -1).Await(nil) + })) + require.NoError(t, registry.AddActivityN("activity", func(task.ActivityContext) (any, error) { + return nil, nil + })) + stream := newFakeWorkItemStream(1) + scheduler := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, scheduler) + worker.executor = task.NewTaskExecutor( + registry, + task.WithOrchestrationOptions(task.OrchestrationOptions{MaxEventsPerTurn: 1}), + ) + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: "partial-instance", + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("partial", "partial-instance", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("first", wrapperspb.String(`"one"`)), + helpers.NewEventRaisedEvent("second", wrapperspb.String(`"two"`)), + }, + }}, + CompletionToken: "partial-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + return len(scheduler.orchestrationCompletions) == 1 + }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + + scheduler.mu.Lock() + defer scheduler.mu.Unlock() + response := scheduler.orchestrationCompletions[0] + require.Equal(t, "partial-token", response.CompletionToken) + require.EqualValues(t, 1, response.GetNumEventsProcessed().GetValue()) + require.Len(t, response.Actions, 1) + require.NotNil(t, response.Actions[0].GetScheduleTask()) +} + +func TestTaskHubGrpcWorkerDoesNotAbandonExpiredActivityLease(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{ + stream: stream, + activityCompletionErr: status.Error(codes.NotFound, "work item not found"), + } + worker := newFakeWorker(t, client) + worker.executor = &recordingExecutor{ + executeActivity: func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) { + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{ + TaskCompleted: &protos.TaskCompletedEvent{Result: wrapperspb.String(`"done"`)}, + }, + }, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + TaskId: 7, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "expired-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.activityCompletions) == 1 + }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + + client.mu.Lock() + defer client.mu.Unlock() + require.Len(t, client.activityCompletions, 1) + require.Zero(t, client.activityAbandons) +} + +// Entity work-item filters are normalized with the same invariant rule as entity +// instance IDs, so a filter keeps matching its own entity name. +func TestEntityWorkItemFiltersUseInvariantCasing(t *testing.T) { + filters, err := cloneWorkItemFilters(&WorkItemFilters{Entities: []string{"İstanbul"}}) + require.NoError(t, err) + entityID, err := api.EntityIDFromString(api.NewEntityID("İstanbul", "key").String()) + require.NoError(t, err) + require.True(t, matchesEntityWorkItemFilters(filters, entityID.Name)) +} + +func TestWorkItemFiltersApplyIndependentlyByKind(t *testing.T) { + filters, err := cloneWorkItemFilters(&WorkItemFilters{ + Entities: []string{"Counter"}, + }) + require.NoError(t, err) + require.True(t, matchesWorkItemFilters(filters, true, "orchestration", "v1")) + require.True(t, matchesWorkItemFilters(filters, false, "activity", "v1")) + require.True(t, matchesEntityWorkItemFilters(filters, "counter")) + require.False(t, matchesEntityWorkItemFilters(filters, "other")) + + filters, err = cloneWorkItemFilters(&WorkItemFilters{RejectAllActivities: true}) + require.NoError(t, err) + require.False(t, matchesWorkItemFilters(filters, false, "activity", "v1")) + require.True(t, matchesWorkItemFilters(filters, true, "orchestration", "v1")) + require.True(t, matchesEntityWorkItemFilters(filters, "counter")) + wire := workItemFiltersToProto(&WorkItemFilters{ + RejectAllOrchestrations: true, + RejectAllActivities: true, + RejectAllEntities: true, + }) + require.Equal(t, helpers.RejectAllWorkItemFilterName, wire.Orchestrations[0].Name) + require.Equal(t, helpers.RejectAllWorkItemFilterName, wire.Activities[0].Name) + require.Equal(t, helpers.RejectAllWorkItemFilterName, wire.Entities[0].Name) +} + +func TestStrictAutoFiltersPreserveAllowedUnversionedOrchestrator(t *testing.T) { + filters := workItemFiltersFromRegistry( + task.TaskRegistrySnapshot{ + Orchestrators: []task.TaskRegistration{ + {Name: "system"}, + {Name: "application", Version: "1.0"}, + }, + }, + &task.VersioningOptions{Version: "1.0", MatchStrategy: task.VersionMatchStrict}, + map[string]struct{}{"system": {}}, + nil, + ) + require.Contains(t, filters.Orchestrations, WorkItemFilter{Name: "system", Versions: []string{""}}) + require.Contains(t, filters.Orchestrations, WorkItemFilter{Name: "application", Versions: []string{"1.0"}}) +} + +// TestStrictAutoFiltersPreserveAllowedUnversionedActivity keeps a system +// component's unversioned activities routable under strict worker versioning. +// An activity inherits its caller's version, so an unversioned system +// orchestration schedules unversioned activities. +func TestStrictAutoFiltersPreserveAllowedUnversionedActivity(t *testing.T) { + snapshot := task.TaskRegistrySnapshot{ + Activities: []task.TaskRegistration{ + {Name: "SystemActivity"}, + {Name: "application", Version: "1.0"}, + }, + } + versioning := &task.VersioningOptions{Version: "1.0", MatchStrategy: task.VersionMatchStrict} + + // Without the allow-list the worker demands its own version for the system + // activity, so the service never dispatches the unversioned work item. + blocked := workItemFiltersFromRegistry(snapshot, versioning, nil, nil) + require.Contains(t, blocked.Activities, WorkItemFilter{Name: "SystemActivity", Versions: []string{"1.0"}}) + + allowed := workItemFiltersFromRegistry( + snapshot, versioning, nil, map[string]struct{}{"systemactivity": {}}) + require.Contains(t, allowed.Activities, WorkItemFilter{Name: "SystemActivity", Versions: []string{""}}) + require.Contains(t, allowed.Activities, WorkItemFilter{Name: "application", Versions: []string{"1.0"}}) +} + +func TestWithUnversionedActivityNamesRejectsBlankNames(t *testing.T) { + options := defaultTaskHubGrpcWorkerOptions() + require.Error(t, WithUnversionedActivityNames(" ")(&options)) + require.NoError(t, WithUnversionedActivityNames("Alpha", "beta")(&options)) + require.Contains(t, options.unversionedActivities, "alpha") + require.Contains(t, options.unversionedActivities, "beta") +} + +func TestWorkItemFiltersFromRegistryMatchVersionedFallbackRules(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("legacy", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("mixed", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("mixed", "v2", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddActivityNVersion("activity", "V1", func(task.ActivityContext) (any, error) { + return nil, nil + })) + + filters := workItemFiltersFromRegistry(registry.Snapshot(), nil, nil, nil) + require.Equal(t, []WorkItemFilter{ + {Name: "legacy", Versions: []string{}}, + {Name: "mixed", Versions: []string{"", "v2"}}, + }, filters.Orchestrations) + require.Equal(t, []WorkItemFilter{{Name: "activity", Versions: []string{"V1"}}}, filters.Activities) + + strict := workItemFiltersFromRegistry(registry.Snapshot(), &task.VersioningOptions{ + Version: "v3", + MatchStrategy: task.VersionMatchStrict, + }, nil, nil) + require.Equal(t, []string{"v3"}, strict.Orchestrations[0].Versions) + require.Equal(t, []string{"v3"}, strict.Activities[0].Versions) + require.Error(t, validateStrictAutoFilters(registry.Snapshot(), &task.VersioningOptions{ + Version: "v3", + MatchStrategy: task.VersionMatchStrict, + })) + require.True(t, filters.RejectAllEntities) +} + +func TestWorkerVersioningOptionOverridesGenericExecutorVersioning(t *testing.T) { + worker := newFakeWorker( + t, + &fakeSchedulerClient{}, + WithTaskVersioning(task.VersioningOptions{ + Version: "1.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureReject, + }), + WithTaskExecutorOptions(task.WithVersioning(task.VersioningOptions{ + Version: "2.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureReject, + })), + ) + + accepted, err := worker.executor.ExecuteActivity( + context.Background(), + "instance", + helpers.NewTaskScheduledEvent(1, "activity", wrapperspb.String("1.0"), nil, nil), + ) + require.NoError(t, err) + require.NotNil(t, accepted.GetTaskFailed(), "v1 should pass version acceptance and reach registry dispatch") + + _, err = worker.executor.ExecuteActivity( + context.Background(), + "instance", + helpers.NewTaskScheduledEvent(1, "activity", wrapperspb.String("2.0"), nil, nil), + ) + var mismatch *task.VersionMismatchError + require.ErrorAs(t, err, &mismatch) + require.Equal(t, "1.0", mismatch.WorkerVersion) +} + +func TestExplicitWorkItemFiltersValidateRegisteredNames(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("known", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + err := validateWorkItemFilters( + &WorkItemFilters{Orchestrations: []WorkItemFilter{{Name: "unknown"}}}, + registry.Snapshot(), + ) + require.ErrorContains(t, err, "not registered") +} + +func TestStrictAutoFiltersValidateNamedRegistrationsWithWildcard(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorNVersion("known", "v1", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("*", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + require.Error(t, validateStrictAutoFilters(registry.Snapshot(), &task.VersioningOptions{ + Version: "v2", + MatchStrategy: task.VersionMatchStrict, + })) +} + +func TestTaskHubGrpcWorkerAdvertisesExplicitCapabilitiesAndFilters(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker( + t, + client, + WithScheduledTaskCapability(true), + WithWorkItemFilters(&WorkItemFilters{ + Orchestrations: []WorkItemFilter{{Name: "orchestration", Versions: []string{"v1", "v2"}}}, + Activities: []WorkItemFilter{{Name: "activity", Versions: []string{"v3"}}}, + }), + ) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.request != nil + }, time.Second, time.Millisecond) + + client.mu.Lock() + require.Equal(t, []protos.WorkerCapability{ + protos.WorkerCapability_WORKER_CAPABILITY_HISTORY_STREAMING, + protos.WorkerCapability_WORKER_CAPABILITY_SCHEDULED_TASKS, + }, client.request.Capabilities) + require.Equal(t, []*protos.OrchestrationFilter{{ + Name: "orchestration", + Versions: []string{"v1", "v2"}, + }}, client.request.WorkItemFilters.Orchestrations) + require.Equal(t, []*protos.ActivityFilter{{ + Name: "activity", + Versions: []string{"v3"}, + }}, client.request.WorkItemFilters.Activities) + client.mu.Unlock() + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerLocallyRejectsFilteredWorkItems(t *testing.T) { + stream := newFakeWorkItemStream(2) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker( + t, + client, + WithWorkItemFilters(&WorkItemFilters{ + Orchestrations: []WorkItemFilter{{Name: "allowed-orchestration", Versions: []string{"v1"}}}, + Activities: []WorkItemFilter{{Name: "allowed-activity", Versions: []string{"v1"}}}, + }), + ) + var executionCount atomic.Int32 + worker.executor = &recordingExecutor{ + executeOrchestrator: func( + context.Context, + api.InstanceID, + []*protos.HistoryEvent, + []*protos.HistoryEvent, + ) (*task.ExecutionResults, error) { + executionCount.Add(1) + return &task.ExecutionResults{Response: &protos.OrchestratorResponse{}}, nil + }, + executeActivity: func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) { + executionCount.Add(1) + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{ + TaskCompleted: &protos.TaskCompletedEvent{}, + }, + }, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: "instance", + NewEvents: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{ + Name: "other-orchestration", + Version: wrapperspb.String("v1"), + }, + }, + }}, + }}, + CompletionToken: "orchestration-token", + }} + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "allowed-activity", + Version: wrapperspb.String("v2"), + TaskId: 7, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "activity-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.orchestrationAbandons == 1 && client.activityAbandons == 1 + }, time.Second, time.Millisecond) + require.Zero(t, executionCount.Load()) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerHydratesAndExternalizesLargePayloads(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + input, err := largepayload.Externalize(context.Background(), options, wrapperspb.String(`"large-input"`)) + require.NoError(t, err) + + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client, WithWorkerLargePayloads(options)) + worker.executor = &recordingExecutor{ + executeActivity: func(_ context.Context, _ api.InstanceID, event *protos.HistoryEvent) (*protos.HistoryEvent, error) { + require.Equal(t, `"large-input"`, event.GetTaskScheduled().GetInput().GetValue()) + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{ + TaskCompleted: &protos.TaskCompletedEvent{Result: wrapperspb.String(`"large-output"`)}, + }, + }, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + Input: input, + TaskId: 7, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "activity-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.activityCompletions) == 1 + }, time.Second, time.Millisecond) + + client.mu.Lock() + require.Contains(t, client.request.Capabilities, protos.WorkerCapability_WORKER_CAPABILITY_LARGE_PAYLOADS) + result := client.activityCompletions[0].Result + client.mu.Unlock() + require.NotEqual(t, `"large-output"`, result.GetValue()) + hydrated, err := largepayload.Hydrate(context.Background(), options, result) + require.NoError(t, err) + require.Equal(t, `"large-output"`, hydrated.GetValue()) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerStreamsRequiredHistory(t *testing.T) { + stream := newFakeWorkItemStream(1) + pastEvent := &protos.HistoryEvent{EventId: 1} + newEvent := &protos.HistoryEvent{EventId: 2} + client := &fakeSchedulerClient{ + stream: stream, + history: []*protos.HistoryChunk{{Events: []*protos.HistoryEvent{pastEvent}}}, + } + worker := newFakeWorker(t, client) + + executed := make(chan struct{}) + worker.executor = &recordingExecutor{ + executeOrchestrator: func( + _ context.Context, + _ api.InstanceID, + pastEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + ) (*task.ExecutionResults, error) { + require.Equal(t, []*protos.HistoryEvent{pastEvent}, pastEvents) + require.Equal(t, []*protos.HistoryEvent{newEvent}, newEvents) + close(executed) + return &task.ExecutionResults{Response: &protos.OrchestratorResponse{}}, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: "instance", + NewEvents: []*protos.HistoryEvent{newEvent}, + RequiresHistoryStreaming: true, + }}, + CompletionToken: "orchestration-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + select { + case <-executed: + return true + default: + return false + } + }, time.Second, time.Millisecond) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.orchestrationCompletions) == 1 + }, time.Second, time.Millisecond) + + client.mu.Lock() + require.Equal(t, "orchestration-token", client.orchestrationCompletions[0].CompletionToken) + client.mu.Unlock() + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerAbandonsSilentHistoryStream(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{ + stream: stream, + historyBlocks: true, + } + worker := newFakeWorker(t, client, WithWorkerSilentDisconnectTimeout(20*time.Millisecond)) + + var executions atomic.Int32 + worker.executor = &recordingExecutor{ + executeOrchestrator: func( + context.Context, + api.InstanceID, + []*protos.HistoryEvent, + []*protos.HistoryEvent, + ) (*task.ExecutionResults, error) { + executions.Add(1) + return &task.ExecutionResults{Response: &protos.OrchestratorResponse{}}, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: "instance", + RequiresHistoryStreaming: true, + }}, + CompletionToken: "orchestration-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.orchestrationAbandons == 1 + }, time.Second, time.Millisecond) + require.Zero(t, executions.Load()) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerDelaysAbandonAfterStreamedHistoryLimit(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{ + stream: stream, + history: []*protos.HistoryChunk{{ + Events: []*protos.HistoryEvent{{EventId: 1}, {EventId: 2}}, + }}, + } + waits := &recordedWaits{} + worker := newFakeWorker( + t, + client, + WithMaxStreamedHistoryEvents(1), + withRecordedWaits(waits), + ) + var executions atomic.Int32 + worker.executor = &recordingExecutor{ + executeOrchestrator: func( + context.Context, + api.InstanceID, + []*protos.HistoryEvent, + []*protos.HistoryEvent, + ) (*task.ExecutionResults, error) { + executions.Add(1) + return &task.ExecutionResults{Response: &protos.OrchestratorResponse{}}, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_OrchestratorRequest{OrchestratorRequest: &protos.OrchestratorRequest{ + InstanceId: "instance", + RequiresHistoryStreaming: true, + }}, + CompletionToken: "orchestration-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.orchestrationAbandons == 1 + }, time.Second, time.Millisecond) + require.Zero(t, executions.Load()) + require.Contains(t, waits.snapshot(), 5*time.Millisecond) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerAppliesActivityBackpressure(t *testing.T) { + stream := newFakeWorkItemStream(2) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client, WithMaxConcurrentActivityWorkItems(1)) + + started := make(chan int32, 2) + release := make(chan struct{}) + worker.executor = &recordingExecutor{ + executeActivity: func(_ context.Context, _ api.InstanceID, event *protos.HistoryEvent) (*protos.HistoryEvent, error) { + started <- event.EventId + <-release + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{TaskCompleted: &protos.TaskCompletedEvent{}}, + }, nil + }, + } + for taskID := int32(1); taskID <= 2; taskID++ { + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + TaskId: taskID, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "token", + }} + } + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.EqualValues(t, 1, <-started) + select { + case taskID := <-started: + t.Fatalf("second activity %d started before the first completed", taskID) + case <-time.After(25 * time.Millisecond): + } + release <- struct{}{} + require.EqualValues(t, 2, <-started) + release <- struct{}{} + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.activityCompletions) == 2 + }, time.Second, time.Millisecond) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { + stream := newFakeWorkItemStream(2) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client, WithMaxConcurrentEntityWorkItems(1)) + worker.executor = &recordingExecutor{ + executeEntity: func(_ context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + require.Equal(t, "@counter@key", request.InstanceId) + require.Len(t, request.Operations, 1) + return &protos.EntityBatchResult{ + Results: []*protos.OperationResult{{ + ResultType: &protos.OperationResult_Success{ + Success: &protos.OperationResultSuccess{Result: wrapperspb.String("1")}, + }, + }}, + EntityState: wrapperspb.String("1"), + }, nil + }, + } + legacyRequestID := uuid.NewString() + v2RequestID := uuid.NewString() + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequest{EntityRequest: &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "add", RequestId: legacyRequestID}}, + }}, + CompletionToken: "legacy-token", + }} + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: &protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: v2RequestID, + Operation: "add", + ParentInstanceId: wrapperspb.String("caller"), + }, + }, + }}, + }}, + CompletionToken: "v2-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return len(client.entityCompletions) == 2 + }, time.Second, time.Millisecond) + client.mu.Lock() + require.Equal(t, "legacy-token", client.entityCompletions[0].CompletionToken) + require.Equal(t, "v2-token", client.entityCompletions[1].CompletionToken) + require.Len(t, client.entityCompletions[1].OperationInfos, 1) + require.Equal(t, v2RequestID, client.entityCompletions[1].OperationInfos[0].RequestId) + require.Equal(t, "caller", client.entityCompletions[1].OperationInfos[0].ResponseDestination.InstanceId) + client.mu.Unlock() + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerForwardsEntityParameters(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + executor := &recordingExecutor{ + executeOrchestrator: func( + context.Context, + api.InstanceID, + []*protos.HistoryEvent, + []*protos.HistoryEvent, + ) (*task.ExecutionResults, error) { + return &task.ExecutionResults{Response: &protos.OrchestratorResponse{}}, nil + }, + } + worker.executor = executor + parameters := &protos.OrchestratorEntityParameters{ + EntityMessageReorderWindow: durationpb.New(0), + } + worker.processOrchestration(context.Background(), client, "token", &protos.OrchestratorRequest{ + InstanceId: "instance", + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("orchestration", "instance", nil, nil, nil, nil), + }, + EntityParameters: parameters, + }) + require.Same(t, parameters, executor.entityParameters) +} + +func TestTaskHubGrpcWorkerAbandonsUnsupportedEntityParameters(t *testing.T) { + tests := map[string]*protos.OrchestratorEntityParameters{ + "missing-window": {}, + "negative-window": { + EntityMessageReorderWindow: durationpb.New(-time.Second), + }, + "positive-window": { + EntityMessageReorderWindow: durationpb.New(time.Second), + }, + "malformed-window": { + EntityMessageReorderWindow: &durationpb.Duration{Seconds: 1, Nanos: -1}, + }, + } + for name, parameters := range tests { + t.Run(name, func(t *testing.T) { + client := new(fakeSchedulerClient) + waits := new(recordedWaits) + worker := newFakeWorker(t, client, withRecordedWaits(waits)) + worker.processOrchestration(context.Background(), client, "token", &protos.OrchestratorRequest{ + InstanceId: "instance", + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("orchestration", "instance", nil, nil, nil, nil), + }, + EntityParameters: parameters, + }) + require.Equal(t, 1, client.orchestrationAbandons) + require.Empty(t, client.orchestrationCompletions) + require.NotEmpty(t, waits.snapshot()) + }) + } +} + +func TestTaskHubGrpcWorkerCompletesEntityFrameworkFailure(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + worker.executor = &recordingExecutor{ + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return nil, errors.New("framework failed") + }, + } + + worker.processEntityBatch( + context.Background(), + client, + "entity-token", + &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "get"}}, + }, + nil, + ) + + require.Len(t, client.entityCompletions, 1) + completion := client.entityCompletions[0] + require.Equal(t, "entity-token", completion.CompletionToken) + require.Contains(t, completion.FailureDetails.ErrorMessage, "framework failed") + require.Empty(t, completion.Results) + require.Empty(t, completion.Actions) + require.Empty(t, completion.OperationInfos) + require.Zero(t, client.entityAbandonAttempts) +} + +func TestTaskHubGrpcWorkerCompletesEntityExecutorPanicAndNilResult(t *testing.T) { + tests := []struct { + name string + executeEntity func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) + errorContains string + }{ + { + name: "panic", + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + panic("executor panic") + }, + errorContains: "executor panic", + }, + { + name: "nil result", + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return nil, nil + }, + errorContains: "returned no result", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + worker.executor = &recordingExecutor{executeEntity: test.executeEntity} + worker.processEntityBatch( + context.Background(), + client, + "entity-token", + &protos.EntityBatchRequest{InstanceId: "@counter@key"}, + nil, + ) + require.Len(t, client.entityCompletions, 1) + require.Contains(t, client.entityCompletions[0].FailureDetails.ErrorMessage, test.errorContains) + }) + } +} + +type failingPayloadBackend struct { + storeErr error + resolveErr error +} + +func (backend *failingPayloadBackend) Store(context.Context, []byte) (string, error) { + return "", backend.storeErr +} + +func (backend *failingPayloadBackend) Resolve(context.Context, string) ([]byte, error) { + return nil, backend.resolveErr +} + +func TestTaskHubGrpcWorkerCompletesEntityPayloadFailures(t *testing.T) { + makeExternalizedInput := func(t *testing.T) (*payload.MemoryStore, *wrapperspb.StringValue) { + t.Helper() + store := payload.NewMemoryStore() + input, err := largepayload.Externalize( + context.Background(), + &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + }, + wrapperspb.String(`"large"`), + ) + require.NoError(t, err) + return store, input + } + + t.Run("hydration", func(t *testing.T) { + store, input := makeExternalizedInput(t) + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: store, + Resolver: &failingPayloadBackend{resolveErr: errors.New("hydrate failed")}, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + })) + worker.processEntityBatch( + context.Background(), + client, + "token", + &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "run", Input: input}}, + }, + nil, + ) + require.Len(t, client.entityCompletions, 1) + require.Contains(t, client.entityCompletions[0].FailureDetails.ErrorMessage, "hydrate failed") + }) + + t.Run("hydration cancellation", func(t *testing.T) { + store, input := makeExternalizedInput(t) + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: store, + Resolver: &failingPayloadBackend{resolveErr: context.Canceled}, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + })) + worker.processEntityBatch( + context.Background(), + client, + "token", + &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "run", Input: input}}, + }, + nil, + ) + require.Empty(t, client.entityCompletions) + require.Equal(t, 1, client.entityAbandonAttempts) + require.NoError(t, client.entityAbandonContextErr) + }) + + t.Run("externalization fallback", func(t *testing.T) { + backend := &failingPayloadBackend{ + storeErr: errors.New("externalize failed"), + resolveErr: errors.New("resolve failed"), + } + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: backend, + Resolver: backend, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + })) + worker.executor = &recordingExecutor{ + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return &protos.EntityBatchResult{EntityState: wrapperspb.String("large")}, nil + }, + } + worker.processEntityBatch( + context.Background(), + client, + "token", + &protos.EntityBatchRequest{InstanceId: "@counter@key"}, + nil, + ) + require.Len(t, client.entityCompletions, 1) + completion := client.entityCompletions[0] + require.Contains(t, completion.FailureDetails.ErrorMessage, "externalize failed") + require.Nil(t, completion.EntityState) + }) +} + +func TestTaskHubGrpcWorkerAbandonsWhenEntityFailureFallbackCannotComplete(t *testing.T) { + backend := &failingPayloadBackend{ + storeErr: errors.New("externalize failed"), + resolveErr: errors.New("resolve failed"), + } + client := &fakeSchedulerClient{entityCompletionErr: status.Error(codes.InvalidArgument, "cannot complete")} + worker := newFakeWorker(t, client, WithWorkerLargePayloads(&api.LargePayloadOptions{ + Store: backend, + Resolver: backend, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + })) + worker.executor = &recordingExecutor{ + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return &protos.EntityBatchResult{EntityState: wrapperspb.String("large")}, nil + }, + } + + worker.processEntityBatch( + context.Background(), + client, + "token", + &protos.EntityBatchRequest{InstanceId: "@counter@key"}, + nil, + ) + require.Equal(t, 1, client.entityAbandonAttempts) +} + +func TestTaskHubGrpcWorkerConvertsInvalidEntityResultsToBatchFailure(t *testing.T) { + tests := []struct { + name string + result *protos.EntityBatchResult + operationInfos []*protos.OperationInfo + errorContains string + }{ + { + name: "V1 result count mismatch", + result: &protos.EntityBatchResult{}, + errorContains: "does not match operation count", + }, + { + name: "V2 result count mismatch", + result: &protos.EntityBatchResult{}, + operationInfos: []*protos.OperationInfo{{RequestId: uuid.NewString()}}, + errorContains: "does not match operation count", + }, + { + name: "batch failure with partial state", + result: &protos.EntityBatchResult{ + FailureDetails: &protos.TaskFailureDetails{ErrorMessage: "failed"}, + EntityState: wrapperspb.String("partial"), + }, + errorContains: "must not contain partial effects", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + worker.executor = &recordingExecutor{ + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + return test.result, nil + }, + } + + worker.processEntityBatch( + context.Background(), + client, + "entity-token", + &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "get"}}, + }, + test.operationInfos, + ) + + require.Len(t, client.entityCompletions, 1) + completion := client.entityCompletions[0] + require.Contains(t, completion.FailureDetails.ErrorMessage, test.errorContains) + require.Empty(t, completion.OperationInfos) + require.Nil(t, completion.EntityState) + }) + } +} + +func TestTaskHubGrpcWorkerAbandonsInvalidV2EntityWithBoundedRetry(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream, entityAbandonFailures: 2} + worker := newFakeWorker(t, client) + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: &protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{ + RequestId: "not-a-guid", + Operation: "add", + }, + }, + }}, + }}, + CompletionToken: "entity-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.entityAbandonAttempts == 3 + }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerRejectsMismatchedTaskVersion(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client, WithTaskExecutorOptions(task.WithVersioning(task.VersioningOptions{ + Version: "1.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureReject, + }))) + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + Version: wrapperspb.String("2.0"), + TaskId: 1, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "version-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + startedAt := time.Now() + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.activityAbandons == 1 + }, 2*time.Second, time.Millisecond) + require.GreaterOrEqual(t, time.Since(startedAt), 900*time.Millisecond) + client.mu.Lock() + require.Empty(t, client.activityCompletions) + client.mu.Unlock() + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerRecreatesConnectionAfterTransientDisconnect(t *testing.T) { + firstStream := newFakeWorkItemStream(1) + firstStream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "disconnect")} + secondStream := newFakeWorkItemStream(1) + secondStream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}, + }} + clients := []*fakeSchedulerClient{ + {stream: firstStream}, + {stream: secondStream}, + } + var factoryCalls atomic.Int32 + + registry := task.NewTaskRegistry() + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + index := int(factoryCalls.Add(1)) - 1 + if index >= len(clients) { + return clients[len(clients)-1], nil, nil + } + return clients[index], nil, nil + }, + registry, + api.DefaultLogger(), + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + return factoryCalls.Load() >= 2 + }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerReconnectsAfterAuthenticationError(t *testing.T) { + firstStream := newFakeWorkItemStream(1) + firstStream.results <- fakeWorkItemResult{err: status.Error(codes.Unauthenticated, "expired token")} + secondStream := newFakeWorkItemStream(0) + thirdStream := newFakeWorkItemStream(1) + thirdStream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}, + }} + clients := []*fakeSchedulerClient{ + {stream: firstStream}, + {stream: secondStream, helloErr: status.Error(codes.Unauthenticated, "refresh pending")}, + {stream: thirdStream}, + } + var factoryCalls atomic.Int32 + + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + index := min(int(factoryCalls.Add(1))-1, len(clients)-1) + return clients[index], nil, nil + }, + task.NewTaskRegistry(), + api.DefaultLogger(), + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { return factoryCalls.Load() >= 3 }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) +} + +func TestTaskHubGrpcWorkerGracefulDrainKeepsCompletionContextAlive(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client, WithMaxConcurrentActivityWorkItems(1)) + + started := make(chan struct{}) + release := make(chan struct{}) + worker.executor = &recordingExecutor{ + executeActivity: func(ctx context.Context, _ api.InstanceID, _ *protos.HistoryEvent) (*protos.HistoryEvent, error) { + close(started) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-release: + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{TaskCompleted: &protos.TaskCompletedEvent{}}, + }, nil + } + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + TaskId: 1, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "token", + }} + + runCtx, cancelRun := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(runCtx)) + <-started + cancelRun() + close(release) + + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), time.Second) + defer cancelShutdown() + require.NoError(t, worker.Shutdown(shutdownCtx)) + client.mu.Lock() + defer client.mu.Unlock() + require.Len(t, client.activityCompletions, 1) +} + +func TestTaskHubGrpcWorkerHelloFailsFast(t *testing.T) { + client := &fakeSchedulerClient{ + stream: newFakeWorkItemStream(0), + helloErr: status.Error(codes.PermissionDenied, "forbidden"), + } + worker := newFakeWorker(t, client) + err := worker.Start(context.Background()) + require.ErrorContains(t, err, "Hello") + require.False(t, worker.Running()) +} + +func TestTaskHubGrpcWorkerOptionValidation(t *testing.T) { + registry := task.NewTaskRegistry() + _, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return nil, nil, errors.New("unused") + }, + registry, + api.DefaultLogger(), + WithMaxConcurrentActivityWorkItems(0), + ) + require.Error(t, err) +} + +func TestTransientWorkerGRPCCodes(t *testing.T) { + for _, code := range []codes.Code{ + codes.Canceled, + codes.DeadlineExceeded, + codes.NotFound, + codes.Unauthenticated, + codes.PermissionDenied, + codes.ResourceExhausted, + codes.Aborted, + codes.Internal, + codes.Unavailable, + codes.Unknown, + } { + require.True(t, isTransientWorkerGRPCCode(code), code.String()) + } + for _, code := range []codes.Code{ + codes.InvalidArgument, + } { + require.False(t, isTransientWorkerGRPCCode(code), code.String()) + } + tooLarge := status.Error(codes.ResourceExhausted, "grpc: received message larger than max") + require.False(t, isTransientWorkerError(tooLarge)) + require.False(t, isTransientWorkerRPCError(tooLarge)) + tooLarge = status.Error(codes.ResourceExhausted, "trying to send message larger than max") + require.False(t, isTransientWorkerError(tooLarge)) + require.False(t, isTransientWorkerRPCError(tooLarge)) + historyLimit := newStreamedHistoryLimitError("streamed history exceeds limit") + require.False(t, isTransientWorkerError(historyLimit)) + require.False(t, isTransientWorkerRPCError(historyLimit)) + require.False(t, isTransientWorkerError( + status.Error(codes.Canceled, "grpc: the client connection is closing"), + )) +} + +func TestFailOversizedOrchestratorResponsePreservesValidResponse(t *testing.T) { + response := &protos.OrchestratorResponse{ + InstanceId: "instance", + CompletionToken: "token", + Actions: []*protos.OrchestratorAction{ + helpers.NewScheduleTaskAction(1, "small", wrapperspb.String("input")), + }, + } + require.Same(t, response, failOversizedOrchestratorResponse(response, DefaultMaxOrchestratorCompletionBytes)) +} + +func TestFailOversizedOrchestratorResponseFailsLargeFanout(t *testing.T) { + const maxBytes = 1024 + response := &protos.OrchestratorResponse{ + InstanceId: "fanout", + CompletionToken: "token", + OrchestrationTraceContext: &protos.OrchestrationTraceContext{SpanID: wrapperspb.String("0123456789abcdef")}, + Actions: []*protos.OrchestratorAction{ + helpers.NewScheduleTaskAction(1, "one", wrapperspb.String(strings.Repeat("a", 600))), + helpers.NewScheduleTaskAction(2, "two", wrapperspb.String(strings.Repeat("b", 600))), + }, + } + require.Greater(t, proto.Size(response), maxBytes) + + bounded := failOversizedOrchestratorResponse(response, maxBytes) + require.NotSame(t, response, bounded) + require.LessOrEqual(t, proto.Size(bounded), maxBytes) + require.Equal(t, response.InstanceId, bounded.InstanceId) + require.Equal(t, response.CompletionToken, bounded.CompletionToken) + require.Same(t, response.OrchestrationTraceContext, bounded.OrchestrationTraceContext) + require.Len(t, bounded.Actions, 1) + failure := bounded.Actions[0].GetCompleteOrchestration() + require.NotNil(t, failure) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, failure.GetOrchestrationStatus()) + require.Equal(t, string(api.ErrorTypeOrchestratorResponseTooLarge), failure.GetFailureDetails().GetErrorType()) + require.True(t, failure.GetFailureDetails().GetIsNonRetriable()) + require.Contains(t, failure.GetFailureDetails().GetErrorMessage(), "reduce per-turn fan-out") +} + +func TestFailOversizedOrchestratorResponseIdentifiesOversizedAction(t *testing.T) { + const maxBytes = 256 + response := &protos.OrchestratorResponse{ + InstanceId: "instance", + CompletionToken: "token", + Actions: []*protos.OrchestratorAction{ + helpers.NewScheduleTaskAction(42, "oversized", wrapperspb.String(strings.Repeat("x", maxBytes*2))), + }, + } + bounded := failOversizedOrchestratorResponse(response, maxBytes) + failure := bounded.Actions[0].GetCompleteOrchestration().GetFailureDetails() + require.Contains(t, failure.GetErrorMessage(), "action 42") + require.Contains(t, failure.GetErrorMessage(), "large-payload externalization") +} diff --git a/client/grpc_worker_transport_test.go b/client/grpc_worker_transport_test.go new file mode 100644 index 00000000..eafcda4f --- /dev/null +++ b/client/grpc_worker_transport_test.go @@ -0,0 +1,929 @@ +package client + +import ( + "context" + "errors" + "io" + "math" + "runtime" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +// recordedWaits captures the deterministic retry schedule without sleeping. +type recordedWaits struct { + mu sync.Mutex + delays []time.Duration +} + +func (r *recordedWaits) wait(ctx context.Context, delay time.Duration) error { + r.mu.Lock() + r.delays = append(r.delays, delay) + r.mu.Unlock() + return ctx.Err() +} + +func (r *recordedWaits) snapshot() []time.Duration { + r.mu.Lock() + defer r.mu.Unlock() + return append([]time.Duration(nil), r.delays...) +} + +func (r *recordedWaits) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.delays) +} + +// withRecordedWaits installs the wait seam so reconnect and RPC retry delays are +// observable without sleeping. +func withRecordedWaits(waits *recordedWaits) TaskHubGrpcWorkerOption { + return func(options *taskHubGrpcWorkerOptions) error { + options.waitFn = waits.wait + options.reconnectRandom = nil + return nil + } +} + +func TestWorkerBackoffScheduleStaysWithinConfiguredBounds(t *testing.T) { + schedule := newWorkerBackoff(200*time.Millisecond, time.Second, nil) + delays := make([]time.Duration, 0, 6) + for i := 0; i < 6; i++ { + delays = append(delays, schedule.Next()) + } + require.Equal(t, []time.Duration{ + 200 * time.Millisecond, + 400 * time.Millisecond, + 800 * time.Millisecond, + time.Second, + time.Second, + time.Second, + }, delays) + + schedule.Reset() + require.Equal(t, 200*time.Millisecond, schedule.Next()) + + repeat := newWorkerBackoff(200*time.Millisecond, time.Second, nil) + for _, expected := range delays { + require.Equal(t, expected, repeat.Next()) + } +} + +func TestWorkerBackoffSaturatesInsteadOfOverflowing(t *testing.T) { + for _, random := range []randomInt64N{ + nil, + func(n int64) int64 { return n - 1 }, + } { + schedule := newWorkerBackoff(time.Hour, time.Duration(math.MaxInt64), random) + previous := time.Duration(0) + for i := range 200 { + delay := schedule.Next() + require.Positive(t, delay, "delay %d overflowed to a non-positive duration", i) + require.GreaterOrEqual(t, delay, time.Hour) + require.LessOrEqual(t, delay, time.Duration(math.MaxInt64)) + if random == nil { + require.GreaterOrEqual(t, delay, previous) + } + previous = delay + } + if random == nil { + require.Equal(t, time.Duration(math.MaxInt64), previous) + } + } + + require.Equal(t, time.Second, doubleDurationBounded(time.Second, time.Second)) + require.Equal(t, 2*time.Second, doubleDurationBounded(time.Second, time.Minute)) + require.Equal( + t, + time.Duration(math.MaxInt64), + doubleDurationBounded(time.Duration(math.MaxInt64)-1, time.Duration(math.MaxInt64)), + ) +} + +func TestWorkerBackoffJitterUsesTheFullBoundedRange(t *testing.T) { + tests := []struct { + name string + random randomInt64N + expected []time.Duration + }{ + { + name: "lower bound", + random: func(int64) int64 { return 0 }, + expected: []time.Duration{200 * time.Millisecond, 300 * time.Millisecond, 600 * time.Millisecond, 750 * time.Millisecond}, + }, + { + name: "upper bound", + random: func(n int64) int64 { return n - 1 }, + expected: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, time.Second, time.Second}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + schedule := newWorkerBackoff(200*time.Millisecond, time.Second, test.random) + for _, expected := range test.expected { + require.Equal(t, expected, schedule.Next()) + } + }) + } +} + +func TestWorkerTransientRetryDelayIsDeterministicAndBounded(t *testing.T) { + worker := &TaskHubGrpcWorker{options: taskHubGrpcWorkerOptions{ + transientRetryBaseDelay: 100 * time.Millisecond, + transientRetryMaxDelay: 500 * time.Millisecond, + }} + delays := make([]time.Duration, 0, 6) + for attempt := 1; attempt <= 6; attempt++ { + delays = append(delays, worker.retryDelay(attempt)) + } + require.Equal(t, []time.Duration{ + 100 * time.Millisecond, + 200 * time.Millisecond, + 400 * time.Millisecond, + 500 * time.Millisecond, + 500 * time.Millisecond, + 500 * time.Millisecond, + }, delays) + + overflowing := &TaskHubGrpcWorker{options: taskHubGrpcWorkerOptions{ + transientRetryBaseDelay: time.Hour, + transientRetryMaxDelay: time.Duration(math.MaxInt64), + }} + for attempt := 1; attempt <= 200; attempt++ { + require.GreaterOrEqual(t, overflowing.retryDelay(attempt), time.Hour) + } +} + +func TestWorkerTransientRPCRetryFollowsTheDeterministicSchedule(t *testing.T) { + waits := &recordedWaits{} + worker := newFakeWorker( + t, + &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + WithWorkerTransientRetryPolicy(4, 10*time.Millisecond, 40*time.Millisecond), + withRecordedWaits(waits), + ) + + var attempts atomic.Int32 + err := worker.executeRPCWithRetry(context.Background(), "test rpc", func(context.Context) error { + attempts.Add(1) + return status.Error(codes.Unavailable, "transient") + }) + require.ErrorContains(t, err, "test rpc failed after 4 attempts") + require.EqualValues(t, 4, attempts.Load()) + require.Equal(t, []time.Duration{ + 10 * time.Millisecond, + 20 * time.Millisecond, + 40 * time.Millisecond, + }, waits.snapshot()) +} + +func TestWorkerDoesNotRetryExpiredWorkItemLease(t *testing.T) { + waits := &recordedWaits{} + worker := newFakeWorker( + t, + &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + WithWorkerTransientRetryPolicy(4, 10*time.Millisecond, 40*time.Millisecond), + withRecordedWaits(waits), + ) + + var attempts atomic.Int32 + err := worker.executeRPCWithRetry(context.Background(), "complete task", func(context.Context) error { + attempts.Add(1) + return status.Error(codes.NotFound, "work item not found") + }) + require.ErrorContains(t, err, "non-retryable") + require.EqualValues(t, 1, attempts.Load()) + require.Empty(t, waits.snapshot()) + require.True(t, isWorkItemGone(err)) +} + +// replayStream delivers a fixed number of health pings and then reports the +// silent-disconnect sentinel, standing in for a stream that ends without ever +// producing a server-side error. +type replayStream struct { + protos.TaskHubSidecarService_GetWorkItemsClient + remaining int +} + +func (s *replayStream) Recv() (*protos.WorkItem, error) { + if s.remaining > 0 { + s.remaining-- + return &protos.WorkItem{Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}}, nil + } + return nil, errSilentDisconnect +} + +// replaySchedulerClient hands out a fresh replayStream per stream generation. +type replaySchedulerClient struct { + *fakeSchedulerClient + messages int +} + +func (c *replaySchedulerClient) GetWorkItems( + context.Context, + *protos.GetWorkItemsRequest, + ...grpc.CallOption, +) (protos.TaskHubSidecarService_GetWorkItemsClient, error) { + return &replayStream{remaining: c.messages}, nil +} + +func newRunLoopTestRun() *grpcWorkerRun { + run := &grpcWorkerRun{ + orchestrationSlots: make(chan struct{}, 1), + activitySlots: make(chan struct{}, 1), + entitySlots: make(chan struct{}, 1), + } + run.intakeCtx, run.cancelIntake = context.WithCancel(context.Background()) + run.processingCtx, run.cancelProcessing = context.WithCancel(context.Background()) + return run +} + +// runLoopUntilWaits drives runLoop until the wait seam has observed the wanted +// number of delays, then cancels intake so the loop returns deterministically. +func runLoopUntilWaits( + t *testing.T, + worker *TaskHubGrpcWorker, + waits *recordedWaits, + wanted int, + stream workItemsStream, +) { + t.Helper() + run := newRunLoopTestRun() + defer run.cancelIntake() + defer run.cancelProcessing() + + watcher := make(chan struct{}) + go func() { + defer close(watcher) + for waits.count() < wanted { + time.Sleep(time.Millisecond) + } + run.cancelIntake() + }() + + _, cancelStream := context.WithCancel(context.Background()) + defer cancelStream() + require.NoError(t, worker.runLoop(run, &grpcWorkerConnection{ + stream: stream, + cancelStream: cancelStream, + })) + <-watcher + run.pending.Wait() + run.cancelProcessing() + run.retired.Wait() +} + +func TestPoisonedSilenceEscalatesWhileDrainAfterFirstMessageResets(t *testing.T) { + for _, testCase := range []struct { + name string + messages int + expected []time.Duration + }{ + { + name: "silence before the first message keeps escalating", + messages: 0, + expected: []time.Duration{ + 10 * time.Millisecond, + 20 * time.Millisecond, + 40 * time.Millisecond, + 80 * time.Millisecond, + 80 * time.Millisecond, + }, + }, + { + name: "a drain after the first message restarts at the base delay", + messages: 1, + expected: []time.Duration{ + 10 * time.Millisecond, + 10 * time.Millisecond, + 10 * time.Millisecond, + 10 * time.Millisecond, + 10 * time.Millisecond, + }, + }, + } { + t.Run(testCase.name, func(t *testing.T) { + waits := &recordedWaits{} + worker := newFakeWorker( + t, + &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + WithWorkerReconnectBackoff(10*time.Millisecond, 80*time.Millisecond), + withRecordedWaits(waits), + ) + worker.clientFactory = func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return &replaySchedulerClient{ + fakeSchedulerClient: &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + messages: testCase.messages, + }, nil, nil + } + + runLoopUntilWaits( + t, + worker, + waits, + len(testCase.expected), + &replayStream{remaining: testCase.messages}, + ) + + delays := waits.snapshot() + require.GreaterOrEqual(t, len(delays), len(testCase.expected)) + require.Equal(t, testCase.expected, delays[:len(testCase.expected)]) + }) + } +} + +func TestTransientConnectFailuresEscalateWithinTheReconnectLoop(t *testing.T) { + waits := &recordedWaits{} + worker := newFakeWorker( + t, + &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + WithWorkerReconnectBackoff(10*time.Millisecond, 40*time.Millisecond), + withRecordedWaits(waits), + ) + worker.clientFactory = func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return nil, nil, status.Error(codes.Unavailable, "cannot dial") + } + + runLoopUntilWaits(t, worker, waits, 4, &replayStream{}) + require.Equal(t, []time.Duration{ + 10 * time.Millisecond, + 20 * time.Millisecond, + 40 * time.Millisecond, + 40 * time.Millisecond, + }, waits.snapshot()[:4]) +} + +// racingStream stays silent until the worker cancels the stream and then +// delivers a work item, reproducing a message that races the silence timeout. +type racingStream struct { + protos.TaskHubSidecarService_GetWorkItemsClient + ctx context.Context + item *protos.WorkItem + delivered atomic.Bool +} + +func (s *racingStream) Recv() (*protos.WorkItem, error) { + <-s.ctx.Done() + if s.delivered.Swap(true) { + return nil, status.FromContextError(s.ctx.Err()).Err() + } + return s.item, nil +} + +type racingSchedulerClient struct { + *fakeSchedulerClient + racing *racingStream +} + +func (c *racingSchedulerClient) GetWorkItems( + ctx context.Context, + _ *protos.GetWorkItemsRequest, + _ ...grpc.CallOption, +) (protos.TaskHubSidecarService_GetWorkItemsClient, error) { + c.racing.ctx = ctx + return c.racing, nil +} + +func TestSilenceTimeoutDoesNotDropAConcurrentlyDeliveredWorkItem(t *testing.T) { + inner := &fakeSchedulerClient{stream: newFakeWorkItemStream(0)} + client := &racingSchedulerClient{ + fakeSchedulerClient: inner, + racing: &racingStream{item: &protos.WorkItem{ + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "activity", + TaskId: 1, + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "instance"}, + }}, + CompletionToken: "raced-token", + }}, + } + worker := newFakeWorker(t, inner, WithWorkerSilentDisconnectTimeout(10*time.Millisecond)) + worker.clientFactory = func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return client, nil, nil + } + worker.executor = &recordingExecutor{ + executeActivity: func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) { + return &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_TaskCompleted{TaskCompleted: &protos.TaskCompletedEvent{}}, + }, nil + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + inner.mu.Lock() + defer inner.mu.Unlock() + return len(inner.activityCompletions) == 1 + }, 5*time.Second, time.Millisecond, "the work item delivered with the silence timeout was dropped") + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + inner.mu.Lock() + defer inner.mu.Unlock() + require.Equal(t, "raced-token", inner.activityCompletions[0].CompletionToken) + require.Zero(t, inner.activityAbandons) +} + +// silencedRecv blocks until the silence timer cancels the stream and then +// reports the supplied terminal error, which makes the timer race deterministic. +func silencedRecv(ctx context.Context, err error) func() (*protos.WorkItem, error) { + return func() (*protos.WorkItem, error) { + <-ctx.Done() + return nil, err + } +} + +func TestSilenceTimeoutOnlyRewritesItsOwnCancellation(t *testing.T) { + for _, testCase := range []struct { + name string + err error + want error + }{ + { + name: "stream cancellation becomes the silent disconnect sentinel", + err: status.Error(codes.Canceled, "context canceled"), + want: errSilentDisconnect, + }, + { + name: "bare context cancellation becomes the silent disconnect sentinel", + err: context.Canceled, + want: errSilentDisconnect, + }, + { + name: "unauthenticated is propagated", + err: status.Error(codes.Unauthenticated, "token expired"), + want: status.Error(codes.Unauthenticated, "token expired"), + }, + { + name: "permission denied is propagated", + err: status.Error(codes.PermissionDenied, "task hub forbidden"), + want: status.Error(codes.PermissionDenied, "task hub forbidden"), + }, + { + name: "unavailable is propagated", + err: status.Error(codes.Unavailable, "endpoint restarting"), + want: status.Error(codes.Unavailable, "endpoint restarting"), + }, + } { + t.Run(testCase.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + _, err := recvBeforeSilenceTimeout(silencedRecv(ctx, testCase.err), cancel, time.Millisecond) + require.Equal(t, testCase.want, err) + }) + } +} + +// TestSilenceTimeoutRacingNonRetryableStatusStopsTheWorker asserts a genuine +// non-retryable status that races the silence timer ends the run loop instead +// of being masked as a retryable silent disconnect and reconnected forever. +func TestSilenceTimeoutRacingNonRetryableStatusStopsTheWorker(t *testing.T) { + streamCtx, cancelStream := context.WithCancel(context.Background()) + defer cancelStream() + authFailure := status.Error(codes.InvalidArgument, "invalid worker request") + + worker := newFakeWorker( + t, + &fakeSchedulerClient{stream: newFakeWorkItemStream(0)}, + WithWorkerSilentDisconnectTimeout(time.Millisecond), + withRecordedWaits(&recordedWaits{}), + ) + + run := newRunLoopTestRun() + defer run.cancelIntake() + defer run.cancelProcessing() + + err := worker.runLoop(run, &grpcWorkerConnection{ + stream: &silencedStream{ctx: streamCtx, err: authFailure}, + cancelStream: cancelStream, + }) + require.ErrorContains(t, err, "non-retryable error") + require.Equal(t, codes.InvalidArgument, status.Code(err)) + require.NotErrorIs(t, err, errSilentDisconnect) + run.retired.Wait() +} + +// silencedStream never delivers a message and reports a terminal error only +// once the silence timer has canceled the stream. +type silencedStream struct { + protos.TaskHubSidecarService_GetWorkItemsClient + ctx context.Context + err error +} + +func (s *silencedStream) Recv() (*protos.WorkItem, error) { + <-s.ctx.Done() + return nil, s.err +} + +func TestStreamHistoryHandlesManySingleEventChunksWithoutQuadraticCopying(t *testing.T) { + const chunks = 20000 + history := make([]*protos.HistoryChunk, 0, chunks) + for i := 0; i < chunks; i++ { + history = append(history, &protos.HistoryChunk{ + Events: []*protos.HistoryEvent{{EventId: int32(i)}}, + }) + } + client := &fakeSchedulerClient{stream: newFakeWorkItemStream(0), history: history} + worker := newFakeWorker(t, client, WithWorkerSilentDisconnectTimeout(time.Minute)) + + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + events, err := worker.streamHistory( + context.Background(), + client, + &protos.OrchestratorRequest{InstanceId: "instance"}, + ) + runtime.ReadMemStats(&after) + + require.NoError(t, err) + require.Len(t, events, chunks) + for i, event := range events { + require.EqualValues(t, i, event.EventId, "history chunks must be accumulated in service order") + } + + // Amortized appends allocate a small constant per event. Reallocating an + // exactly sized slice per chunk is quadratic and allocates on the order of + // chunks/2 pointers per event, which is several orders of magnitude more. + bytesPerEvent := (after.TotalAlloc - before.TotalAlloc) / chunks + require.Less( + t, + bytesPerEvent, + uint64(8192), + "history accumulation allocated %d bytes per event, which suggests quadratic copying", + bytesPerEvent, + ) +} + +func TestStreamHistoryEnforcesEventAndByteLimitsBeforeRetainingChunk(t *testing.T) { + tests := []struct { + name string + history []*protos.HistoryChunk + option TaskHubGrpcWorkerOption + message string + }{ + { + name: "events", + history: []*protos.HistoryChunk{ + {Events: []*protos.HistoryEvent{{EventId: 1}, {EventId: 2}}}, + {Events: []*protos.HistoryEvent{{EventId: 3}}}, + }, + option: WithMaxStreamedHistoryEvents(2), + message: "2 events", + }, + { + name: "bytes", + history: []*protos.HistoryChunk{ + {Events: []*protos.HistoryEvent{{EventId: 1}}}, + }, + option: WithMaxStreamedHistoryBytes(1), + message: "1 bytes", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := &fakeSchedulerClient{ + stream: newFakeWorkItemStream(0), + history: test.history, + } + worker := newFakeWorker(t, client, test.option) + + events, err := worker.streamHistory( + context.Background(), + client, + &protos.OrchestratorRequest{InstanceId: "instance"}, + ) + require.Nil(t, events) + require.ErrorIs(t, err, ErrStreamedHistoryLimitExceeded) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) + require.ErrorContains(t, err, test.message) + }) + } +} + +// unrepresentableConcurrency returns the smallest concurrency that cannot be +// advertised on the 32-bit GetWorkItems fields. It is computed at runtime +// because the constant expression math.MaxInt32+1 does not fit in an int on +// 32-bit targets, where no int can exceed the limit and the case cannot exist. +func unrepresentableConcurrency() (int, bool) { + limit := int64(math.MaxInt32) + 1 + if limit > int64(maxInt) { + return 0, false + } + return int(limit), true +} + +// maxInt is the largest value of the platform int type. +const maxInt = int(^uint(0) >> 1) + +func TestWorkerLimitOptionValidation(t *testing.T) { + for _, testCase := range []struct { + name string + option TaskHubGrpcWorkerOption + }{ + {"zero orchestrations", WithMaxConcurrentOrchestrationWorkItems(0)}, + {"negative orchestrations", WithMaxConcurrentOrchestrationWorkItems(-1)}, + {"zero activities", WithMaxConcurrentActivityWorkItems(0)}, + {"negative activities", WithMaxConcurrentActivityWorkItems(-1)}, + {"zero entities", WithMaxConcurrentEntityWorkItems(0)}, + {"negative entities", WithMaxConcurrentEntityWorkItems(-1)}, + {"zero streamed history events", WithMaxStreamedHistoryEvents(0)}, + {"excessive streamed history events", WithMaxStreamedHistoryEvents(MaxStreamedHistoryEvents + 1)}, + {"zero streamed history bytes", WithMaxStreamedHistoryBytes(0)}, + {"excessive streamed history bytes", WithMaxStreamedHistoryBytes(MaxStreamedHistoryBytes + 1)}, + {"small orchestrator completion bytes", WithMaxOrchestratorCompletionBytes(minOrchestratorCompletionBytes - 1)}, + {"excessive orchestrator completion bytes", WithMaxOrchestratorCompletionBytes(DefaultMaxOrchestratorCompletionBytes + 1)}, + } { + t.Run(testCase.name, func(t *testing.T) { + options := defaultTaskHubGrpcWorkerOptions() + require.Error(t, testCase.option(&options)) + }) + } + + if unrepresentable, ok := unrepresentableConcurrency(); ok { + for _, testCase := range []struct { + name string + option TaskHubGrpcWorkerOption + }{ + {"unrepresentable orchestrations", WithMaxConcurrentOrchestrationWorkItems(unrepresentable)}, + {"unrepresentable activities", WithMaxConcurrentActivityWorkItems(unrepresentable)}, + {"unrepresentable entities", WithMaxConcurrentEntityWorkItems(unrepresentable)}, + } { + t.Run(testCase.name, func(t *testing.T) { + options := defaultTaskHubGrpcWorkerOptions() + require.Error(t, testCase.option(&options)) + }) + } + } + + options := defaultTaskHubGrpcWorkerOptions() + require.NoError(t, WithMaxConcurrentOrchestrationWorkItems(math.MaxInt32)(&options)) + require.EqualValues(t, math.MaxInt32, int32(options.maxConcurrentOrchestrations)) + require.Equal(t, DefaultMaxStreamedHistoryEvents, options.maxStreamedHistoryEvents) + require.Equal(t, DefaultMaxStreamedHistoryBytes, options.maxStreamedHistoryBytes) + require.Equal(t, DefaultMaxOrchestratorCompletionBytes, options.maxOrchestratorCompletionBytes) + require.NoError(t, WithMaxOrchestratorCompletionBytes(64*1024)(&options)) + require.NoError(t, WithMaxOrchestratorCompletionBytes(DefaultMaxOrchestratorCompletionBytes)(&options)) + require.Equal(t, 64*1024, options.maxOrchestratorCompletionBytes) +} + +func TestWorkerOptionValidationRejectsInvalidConfiguration(t *testing.T) { + for _, testCase := range []struct { + name string + option TaskHubGrpcWorkerOption + }{ + {"non-positive hello timeout", WithWorkerHelloTimeout(0)}, + {"non-positive silent disconnect timeout", WithWorkerSilentDisconnectTimeout(-time.Second)}, + {"non-positive rpc timeout", WithWorkerRPCTimeout(0)}, + {"non-positive reconnect base delay", WithWorkerReconnectBackoff(0, time.Second)}, + {"inverted reconnect delays", WithWorkerReconnectBackoff(time.Second, time.Millisecond)}, + {"non-positive retry attempts", WithWorkerTransientRetryPolicy(0, time.Second, time.Second)}, + {"non-positive retry base delay", WithWorkerTransientRetryPolicy(3, 0, time.Second)}, + {"inverted retry delays", WithWorkerTransientRetryPolicy(3, time.Second, time.Millisecond)}, + {"negative maximum timer interval", WithMaximumTimerInterval(-time.Second)}, + {"nil task executor option", WithTaskExecutorOptions(nil)}, + {"unsupported capability", WithWorkerCapabilities(WorkerCapability(9999))}, + {"unspecified capability", WithWorkerCapabilities(protos.WorkerCapability_WORKER_CAPABILITY_UNSPECIFIED)}, + {"blank unversioned orchestrator name", WithUnversionedOrchestratorNames(" ")}, + {"empty orchestration filter name", WithWorkItemFilters(&WorkItemFilters{ + Orchestrations: []WorkItemFilter{{Name: ""}}, + })}, + {"empty activity filter name", WithWorkItemFilters(&WorkItemFilters{ + Activities: []WorkItemFilter{{Name: ""}}, + })}, + {"blank entity filter name", WithWorkItemFilters(&WorkItemFilters{Entities: []string{" "}})}, + {"contradictory orchestration rejection", WithWorkItemFilters(&WorkItemFilters{ + Orchestrations: []WorkItemFilter{{Name: "orchestration"}}, + RejectAllOrchestrations: true, + })}, + {"contradictory activity rejection", WithWorkItemFilters(&WorkItemFilters{ + Activities: []WorkItemFilter{{Name: "activity"}}, + RejectAllActivities: true, + })}, + {"contradictory entity rejection", WithWorkItemFilters(&WorkItemFilters{ + Entities: []string{"counter"}, + RejectAllEntities: true, + })}, + {"duplicate orchestration filter", WithWorkItemFilters(&WorkItemFilters{ + Orchestrations: []WorkItemFilter{{Name: "Orchestration"}, {Name: "orchestration"}}, + })}, + {"duplicate activity filter", WithWorkItemFilters(&WorkItemFilters{ + Activities: []WorkItemFilter{{Name: "Activity"}, {Name: "activity"}}, + })}, + {"duplicate entity filter", WithWorkItemFilters(&WorkItemFilters{ + Entities: []string{"Counter", "counter"}, + })}, + } { + t.Run(testCase.name, func(t *testing.T) { + options := defaultTaskHubGrpcWorkerOptions() + require.Error(t, testCase.option(&options)) + }) + } +} + +func TestWorkerConstructorPropagatesOptionValidationErrors(t *testing.T) { + factory := func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + return nil, nil, errors.New("unused") + } + _, err := newTaskHubGrpcWorker( + factory, + task.NewTaskRegistry(), + api.DefaultLogger(), + WithTaskExecutorOptions(nil), + ) + require.ErrorContains(t, err, "task executor option cannot be nil") + + _, err = newTaskHubGrpcWorker( + factory, + task.NewTaskRegistry(), + api.DefaultLogger(), + WithWorkerCapabilities(WorkerCapabilityLargePayloads), + ) + require.ErrorContains(t, err, "large-payload capability requires worker large-payload options") + + _, err = NewTaskHubGrpcWorker(nil, task.NewTaskRegistry(), api.DefaultLogger()) + require.ErrorContains(t, err, "gRPC connection is required") + + _, err = NewTaskHubGrpcWorkerWithConnectionFactory(nil, task.NewTaskRegistry(), api.DefaultLogger()) + require.ErrorContains(t, err, "connection factory is required") + + _, err = newTaskHubGrpcWorker(factory, nil, api.DefaultLogger()) + require.ErrorContains(t, err, "task registry is required") +} + +// countingCloser records how many times a retired connection was closed. +type countingCloser struct { + closes atomic.Int32 +} + +func (c *countingCloser) Close() error { + c.closes.Add(1) + return nil +} + +func TestConnectionFactoryRejectsNilConnections(t *testing.T) { + closer := &countingCloser{} + worker, err := NewTaskHubGrpcWorkerWithConnectionFactory( + func(context.Context) (grpc.ClientConnInterface, io.Closer, error) { + return nil, closer, nil + }, + task.NewTaskRegistry(), + api.DefaultLogger(), + ) + require.NoError(t, err) + err = worker.Start(context.Background()) + require.ErrorContains(t, err, "returned a nil connection") + require.False(t, worker.Running()) + require.EqualValues(t, 1, closer.closes.Load(), "a rejected connection must still be closed") +} + +func TestOwnedConnectionFactoryRecreatesAndClosesRetiredConnections(t *testing.T) { + firstStream := newFakeWorkItemStream(1) + firstStream.results <- fakeWorkItemResult{err: status.Error(codes.Unavailable, "disconnect")} + secondStream := newFakeWorkItemStream(1) + secondStream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}, + }} + clients := []*fakeSchedulerClient{{stream: firstStream}, {stream: secondStream}} + closers := []*countingCloser{{}, {}} + + var generations atomic.Int32 + worker, err := newTaskHubGrpcWorker( + func(context.Context) (protos.TaskHubSidecarServiceClient, io.Closer, error) { + index := int(generations.Add(1)) - 1 + if index >= len(clients) { + index = len(clients) - 1 + } + return clients[index], closers[index], nil + }, + task.NewTaskRegistry(), + api.DefaultLogger(), + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + return closers[0].closes.Load() == 1 + }, 5*time.Second, time.Millisecond, "the retired owned connection was never closed") + require.Eventually(t, func() bool { + return generations.Load() >= 2 + }, 5*time.Second, time.Millisecond) + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + require.EqualValues(t, 1, closers[0].closes.Load(), "retired connections must be closed exactly once") +} + +// fakeClientConn is a borrowed grpc.ClientConnInterface that records how many +// streams the worker opened on it and whether it was ever closed. +type fakeClientConn struct { + mu sync.Mutex + streams int + items []*protos.WorkItem + closes atomic.Int32 +} + +func (c *fakeClientConn) Invoke(context.Context, string, any, any, ...grpc.CallOption) error { + return nil +} + +func (c *fakeClientConn) NewStream( + ctx context.Context, + _ *grpc.StreamDesc, + _ string, + _ ...grpc.CallOption, +) (grpc.ClientStream, error) { + c.mu.Lock() + c.streams++ + stream := &fakeClientStream{ctx: ctx, items: c.items, generation: c.streams} + c.mu.Unlock() + return stream, nil +} + +func (c *fakeClientConn) Close() error { + c.closes.Add(1) + return nil +} + +func (c *fakeClientConn) streamCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.streams +} + +type fakeClientStream struct { + ctx context.Context + items []*protos.WorkItem + generation int + index int +} + +func (s *fakeClientStream) Header() (metadata.MD, error) { return nil, nil } +func (s *fakeClientStream) Trailer() metadata.MD { return nil } +func (s *fakeClientStream) CloseSend() error { return nil } +func (s *fakeClientStream) Context() context.Context { return s.ctx } +func (s *fakeClientStream) SendMsg(any) error { return nil } + +// RecvMsg lets the first stream generation deliver its items and then +// disconnect; later generations stay open so the worker settles. +func (s *fakeClientStream) RecvMsg(message any) error { + if s.generation == 1 { + if s.index < len(s.items) { + proto.Merge(message.(*protos.WorkItem), s.items[s.index]) + s.index++ + return nil + } + return status.Error(codes.Unavailable, "disconnect") + } + <-s.ctx.Done() + return status.FromContextError(s.ctx.Err()).Err() +} + +func TestBorrowedConnectionIsReusedAcrossReconnectsAndNeverClosed(t *testing.T) { + connection := &fakeClientConn{items: []*protos.WorkItem{{ + Request: &protos.WorkItem_HealthPing{HealthPing: &protos.HealthPing{}}, + }}} + worker, err := NewTaskHubGrpcWorker( + connection, + task.NewTaskRegistry(), + api.DefaultLogger(), + WithWorkerHelloTimeout(time.Second), + WithWorkerSilentDisconnectTimeout(time.Second), + WithWorkerReconnectBackoff(time.Millisecond, 5*time.Millisecond), + ) + require.NoError(t, err) + + ctx, cancel := context.WithCancel(context.Background()) + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + return connection.streamCount() >= 2 + }, 5*time.Second, time.Millisecond, "the worker did not reopen a stream after a transient disconnect") + + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + + // Documented limitation: a borrowed connection is never replaced or closed, + // so every reconnect reuses the caller's channel however wedged it is. Only + // the connection-factory worker, which durabletaskscheduler.NewWorker uses, + // can obtain a fresh channel and close the retired one. + require.Zero(t, connection.closes.Load(), "the worker must never close a borrowed connection") + require.GreaterOrEqual(t, connection.streamCount(), 2) +} diff --git a/client/worker_grpc.go b/client/worker_grpc.go deleted file mode 100644 index 69c1bea0..00000000 --- a/client/worker_grpc.go +++ /dev/null @@ -1,212 +0,0 @@ -package client - -import ( - "context" - "errors" - "fmt" - "io" - "time" - - "github.com/cenkalti/backoff/v4" - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/microsoft/durabletask-go/task" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" - "google.golang.org/protobuf/types/known/emptypb" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -type workItemsStream interface { - Recv() (*protos.WorkItem, error) -} - -func (c *TaskHubGrpcClient) StartWorkItemListener(ctx context.Context, r *task.TaskRegistry) error { - executor := task.NewTaskExecutor(r) - - var stream workItemsStream - - initStream := func() error { - _, err := c.client.Hello(ctx, &emptypb.Empty{}) - if err != nil { - return fmt.Errorf("failed to connect to task hub service: %w", err) - } - - req := protos.GetWorkItemsRequest{} - stream, err = c.client.GetWorkItems(ctx, &req) - if err != nil { - return fmt.Errorf("failed to get work item stream: %w", err) - } - return nil - } - - c.logger.Infof("connecting work item listener stream") - err := initStream() - if err != nil { - return err - } - - go func() { - c.logger.Info("starting background processor") - defer func() { - c.logger.Info("stopping background processor") - // We must use a background context here as the stream's context is likely canceled - shutdownErr := executor.Shutdown(context.Background()) - if shutdownErr != nil { - c.logger.Warnf("error while shutting down background processor: %v", shutdownErr) - } - }() - for { - // TODO: Manage concurrency - workItem, err := stream.Recv() - - if err != nil { - // user wants to stop the listener - if ctx.Err() != nil { - c.logger.Infof("stopping background processor: %v", err) - return - } - - retriable := false - - c.logger.Errorf("background processor received stream error: %v", err) - - if errors.Is(err, io.EOF) { - retriable = true - } else if grpcStatus, ok := status.FromError(err); ok { - c.logger.Warnf("received grpc error code %v", grpcStatus.Code().String()) - switch grpcStatus.Code() { - case codes.Unavailable, codes.Canceled: - retriable = true - default: - retriable = true - } - } - - if !retriable { - c.logger.Infof("stopping background processor, non retriable error: %v", err) - return - } - - err = backoff.Retry( - func() error { - // user wants to stop the listener - if ctx.Err() != nil { - return backoff.Permanent(ctx.Err()) - } - - c.logger.Infof("reconnecting work item listener stream") - streamErr := initStream() - if streamErr != nil { - c.logger.Errorf("error initializing work item listener stream %v", streamErr) - return streamErr - } - return nil - }, - // retry forever since we don't have a way of asynchronously return errors to the user - newInfiniteRetries(), - ) - if err != nil { - c.logger.Infof("stopping background processor, unable to reconnect stream: %v", err) - return - } - c.logger.Infof("successfully reconnected work item listener stream...") - // continue iterating - continue - } - - if orchReq := workItem.GetOrchestratorRequest(); orchReq != nil { - go c.processOrchestrationWorkItem(ctx, executor, orchReq) - } else if actReq := workItem.GetActivityRequest(); actReq != nil { - go c.processActivityWorkItem(ctx, executor, actReq) - } else { - c.logger.Warnf("received unknown work item type: %v", workItem) - } - } - }() - return nil -} - -func (c *TaskHubGrpcClient) processOrchestrationWorkItem( - ctx context.Context, - executor backend.Executor, - workItem *protos.OrchestratorRequest, -) { - results, err := executor.ExecuteOrchestrator(ctx, api.InstanceID(workItem.InstanceId), workItem.PastEvents, workItem.NewEvents) - - resp := protos.OrchestratorResponse{InstanceId: workItem.InstanceId} - if err != nil { - // NOTE: At the time of writing, there's no known case where this error is returned. - // We add error handling here anyways, just in case. - failureAction := helpers.NewCompleteOrchestrationAction( - -1, - protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, - wrapperspb.String("An internal error occured while executing the orchestration."), - nil, - &protos.TaskFailureDetails{ - ErrorType: fmt.Sprintf("%T", err), - ErrorMessage: err.Error(), - }) - resp.Actions = []*protos.OrchestratorAction{failureAction} - } else { - resp.Actions = results.Response.Actions - resp.CustomStatus = results.Response.GetCustomStatus() - } - - if _, err = c.client.CompleteOrchestratorTask(ctx, &resp); err != nil { - if ctx.Err() != nil { - c.logger.Warn("failed to complete orchestration task: context canceled") - } else { - c.logger.Errorf("failed to complete orchestration task: %v", err) - } - } -} - -func (c *TaskHubGrpcClient) processActivityWorkItem( - ctx context.Context, - executor backend.Executor, - req *protos.ActivityRequest, -) { - var tc *protos.TraceContext = nil // TODO: How to populate trace context? - event := helpers.NewTaskScheduledEvent(req.TaskId, req.Name, req.Version, req.Input, tc) - result, err := executor.ExecuteActivity(ctx, api.InstanceID(req.OrchestrationInstance.InstanceId), event) - - resp := protos.ActivityResponse{InstanceId: req.OrchestrationInstance.InstanceId, TaskId: req.TaskId} - if err != nil { - // NOTE: At the time of writing, there's no known case where this error is returned. - // We add error handling here anyways, just in case. - resp.FailureDetails = &protos.TaskFailureDetails{ - ErrorType: fmt.Sprintf("%T", err), - ErrorMessage: err.Error(), - } - } else if tc := result.GetTaskCompleted(); tc != nil { - resp.Result = tc.Result - } else if tf := result.GetTaskFailed(); tf != nil { - resp.FailureDetails = tf.FailureDetails - } else { - resp.FailureDetails = &protos.TaskFailureDetails{ - ErrorType: "UnknownTaskResult", - ErrorMessage: "Unknown task result", - } - } - - if _, err = c.client.CompleteActivityTask(ctx, &resp); err != nil { - if ctx.Err() != nil { - c.logger.Warn("failed to complete activity task: context canceled") - } else { - c.logger.Errorf("failed to complete activity task: %v", err) - } - } -} - -func newInfiniteRetries() *backoff.ExponentialBackOff { - b := backoff.NewExponentialBackOff() - // max wait of 15 seconds between retries - b.MaxInterval = 15 * time.Second - // retry forever - b.MaxElapsedTime = 0 - b.Reset() - return b -} diff --git a/client/workitem.go b/client/workitem.go new file mode 100644 index 00000000..9b3ba328 --- /dev/null +++ b/client/workitem.go @@ -0,0 +1,10 @@ +package client + +import "time" + +// workItemAbandonDelayError is an optional marker a work-item processing error +// can implement to ask the worker to defer redelivery of that work item. +type workItemAbandonDelayError interface { + error + WorkItemAbandonDelay() time.Duration +} diff --git a/cmd/orchestratorvet/README.md b/cmd/orchestratorvet/README.md new file mode 100644 index 00000000..5c5121bc --- /dev/null +++ b/cmd/orchestratorvet/README.md @@ -0,0 +1,261 @@ +# orchestratorvet + +`orchestratorvet` is a [`go vet`](https://pkg.go.dev/cmd/vet)-compatible driver +for the `orchestratorgo` analyzer. The analyzer reports replay hazards in the +orchestrator functions a package registers with `task.TaskRegistry`. + +## Running it + +```bash +cd cmd/orchestratorvet && go build -o ../../bin/orchestratorvet . +cd ../.. +go vet -vettool=$PWD/bin/orchestratorvet ./... +``` + +Files ending in `_test.go` are excluded by default because test suites often +register intentionally invalid or nondeterministic orchestrators. Include them +when desired: + +```bash +go vet -vettool=$PWD/bin/orchestratorvet -orchestratorgo.test-files ./... +``` + +The binary also runs standalone under any +[`unitchecker`](https://pkg.go.dev/golang.org/x/tools/go/analysis/unitchecker) +driver, and the analyzer itself is available as +`github.com/microsoft/durabletask-go/cmd/orchestratorvet/analysis/orchestratorgo.Analyzer` +for embedding in a multichecker. + +## What it analyzes + +Orchestrator code is replayed from history on every turn, so it must produce the +same actions in the same order every time it runs. The analyzer therefore starts +from registrations rather than from signatures: a function is only analyzed when +the package under analysis passes it to one of + +- `(*task.TaskRegistry).AddOrchestrator` +- `(*task.TaskRegistry).AddOrchestratorN` +- `(*task.TaskRegistry).AddOrchestratorVersion` +- `(*task.TaskRegistry).AddOrchestratorNVersion` + +From each of those roots it follows the whole-package call graph through +same-package named functions, methods, function variables whose target can be +proven, and nested function literals. Recursion terminates because each function +is visited once, and a helper reached from several orchestrators is reported +once. Activity bodies, entity bodies, and any function not reachable from a +registered orchestrator are never reported. + +## Checks + +### Wall clock + +`time.Now`, `time.Since`, and `time.Until` read the host clock, which differs on +every replay. `time.Sleep`, `time.After`, `time.Tick`, `time.NewTimer`, +`time.NewTicker`, and `time.AfterFunc` block or fire on host time instead of +durable time. + +Use `ctx.CurrentTimeUtc` for the current orchestration time and +`ctx.CreateTimer` for a durable delay. + +### Nondeterministic identifiers and randomness + +Reported: the `github.com/google/uuid` random and time-based constructors +(`New`, `NewString`, `NewRandom`, `NewRandomFromReader`, `NewUUID`), every +`crypto/rand` function, the `math/rand` and `math/rand/v2` package-level global +source, and methods on a `*rand.Rand` value whose seed the analyzer proves +reaches a source that varies between runs, such as the host clock, the process +environment, or another random source. + +Not reported: `uuid.NewSHA1`, `uuid.NewMD5`, `uuid.Parse`, a generator seeded +from a constant (including a constant held in a variable), and a generator +seeded from orchestration input, which is restored from history and therefore +replays identically. A seed the analyzer cannot follow to either kind of source +is left alone rather than guessed at. + +Use `ctx.NewGuid` for a deterministic identifier, or generate the value in an +activity and pass it back. + +### Unsafe parallelism and synchronization + +Reported: `go` statements, `sync` and `sync/atomic` primitives, channel +creation, send, receive, range, and close, and `select` statements. The check +keys on the package that declares a method, so an embedded `sync.Mutex` is +recognized through the embedding type. + +Not reported: `ctx.Go`, `ctx.NewWaitGroup`, `ctx.Select` with `task.OnTask` or +`task.OnEvent`, `task.NewEventChannel`, and `ctx.WaitForSingleEvent`. These are +scheduled by the deterministic orchestration scheduler. + +### External I/O + +Reported: `os` filesystem, environment, and process functions, `net/http` +package functions and `*http.Client` request methods, `net.Dial*`, `net.Listen*`, +`net.Lookup*`, and `net.Resolve*`, `os/exec` and `*exec.Cmd` execution methods, +and every `syscall` function. + +Move the side effect into an activity, whose result is recorded in history. +Activities are delivered at least once and must be idempotent because a worker +can finish the side effect but lose its completion response before DTS records it. + +### Replay-unsafe logging + +Reported: the `log` package functions and `*log.Logger` methods, the +package-level `log/slog` functions, `slog.Default` and `slog.SetDefault`, which +reach the process-wide logger, `fmt.Print`, `fmt.Printf`, `fmt.Println`, and +`fmt.Fprint*` targeting `os.Stdout` or `os.Stderr`. + +Not reported: methods on a `*slog.Logger`, because `ctx.Logger()` returns a +`*slog.Logger` that suppresses output while replaying. `fmt.Sprintf` and the +other pure formatting helpers are also left alone. + +### Unbounded orchestration loops + +A loop with no condition, or with a constant `true` condition, is reported only +when the whole-package loop and call graph proves that the body can neither +leave the loop nor make durable progress. Any `return`, `break`, `goto`, +labelled statement, closure, `go` statement, `defer`, or `panic` in the loop +body ends the proof, as does any call the analyzer cannot follow, except calls +to packages it explicitly models as pure. Any interaction with the durable task +package counts as possible progress and suppresses the report. + +Await a task, wait for an event, or call `ctx.ContinueAsNew` to make the loop +durable. + +### Unresolved activity and sub-orchestration names + +`ctx.CallActivity` and `ctx.CallSubOrchestrator` accept either a name or a +function, and both resolve to a registered name at runtime. When the package's +own registration set for that namespace is complete, a referenced name that is +absent from it is reported. + +A registration set is only treated as complete when the package registers at +least one task in that namespace, every registered name is proven statically, +no `"*"` wildcard registration exists, and no typed registry escapes to opaque +code or comes from an unknown origin, including caller-owned parameters and +tuple-returning factories. Passing a registry to a helper, returning +or storing it outside a local variable, erasing its type behind an interface, or +taking a registration method value +ends the completeness proof for both namespaces: a helper's name does not limit +what it can register. Registration validation and orchestrator discovery still +run even when absence cannot be proven. + +### Registration forms + +`task.TaskRegistry` rejects a registration outright when the handler is nil, +when the name is the empty string, or when the version is a non-empty string of +only whitespace. Each of those is reported, because the call always returns an +error. + +A whitespace-only name is not rejected: the registry lowercases it and stores it +verbatim, so the task really is registered under a name every caller has to +reproduce exactly. That is reported separately, as a name to replace rather than +a call that fails. + +Also reported: a duplicate name and version pair on the same registry value, a +name derived by reflection from a function literal or a method value (which +produces a compiler-generated name such as `func1` or `Run-fm`), and a +registered orchestrator invoked as a plain Go call instead of through +`ctx.CallSubOrchestrator`. + +Duplicate detection compares direct expression or assignment statements in the +same block or switch/select clause, on a variable initialized once by +`task.NewTaskRegistry()`. Reassignment (including tuple and range writes), +aliasing (including registry pointers stored as map keys), escapes, unknown +initializers, conditional expressions, and separate functions or closures +prevent a proof of duplication. Functions containing `goto` are excluded rather +than assuming that registrations in one block execute on compatible paths. Its key mirrors the +registry's own normalization, which lowercases the name and the version and +trims neither, so `"1.0"` and `" 1.0"` are distinct registrations just as they +are at runtime. A registration whose version is computed at runtime is left out +of the comparison entirely, because it could land on any key. + +## Suggested fixes + +Two diagnostics carry an `analysis.SuggestedFix` that `gopls` and `go vet -fix` +can apply: + +- `time.Now()` becomes `ctx.CurrentTimeUtc`, when a non-blank + `*task.OrchestrationContext` parameter is in scope and neither shadowed nor + reassigned. The visible binding must be the original parameter object, not + merely another value of the same type. One atomic + fix, offered on the first fixable clock diagnostic in each file, rewrites all + fixable reads in that file using each call's own context binding. It removes + the `time` import only if no other uses remain, while retaining all diagnostics. + Standalone call statements and direct `go`/`defer` calls are not rewritten, + since a field read is not a valid replacement there. +- `go func() { ... }()` becomes `ctx.Go(func(*task.OrchestrationContext) { ... })`, + when the goroutine is an immediately invoked literal with no parameters, no + results, and no arguments, and the file imports the durable task package under + a usable, unshadowed name and the original, unmodified context parameter is + visible. Only the text around the braces is rewritten, so the body and + every comment in it are preserved exactly. + +Every other diagnostic is left without a fix because no single rewrite is always +correct. + +Before file-level batching, the pinned x/tools v0.49 CLI already handled the +multiple-`time.Now()` unused-import case: `go vet -fix` merges edits and removes +unused imports internally. That case is not a reproduced CLI failure. The atomic +clock fix deliberately offers a stronger raw-edit guarantee for consumers applying +our `TextEdits` without cleanup. x/tools does not promise that arbitrary fixes +from different diagnostics compose into compiling code. + +## False-positive guardrails + +The analyzer reports only what it can prove, and stays silent otherwise. + +- Only registered orchestrators and the code they reach are analyzed. +- A function variable assigned more than once is treated as unresolved, and its + bodies are not followed. +- Calls into `github.com/microsoft/durabletask-go/task` are never followed, + because the channels, goroutines, and locks there implement the deterministic + orchestration scheduler. Without this the durable task package would report + its own primitives when analyzing itself. +- Dynamic, wildcard, and cross-package registration all suppress the unresolved + name check. +- Unbounded loop reporting requires a complete proof; any unmodelled call, exit + path, or durable interaction suppresses it. +- A random generator is only reported when its seed provably reaches a source + that varies between runs. A constant seed, a seed taken from orchestration + input, and a seed the analyzer cannot follow are all left alone. +- Every diagnostic is deduplicated by source position and category, so a shared + helper is reported once no matter how many orchestrators reach it. + +## Limitations + +- The analyzer runs one package at a time. Helpers, activity registrations, and + orchestrator bodies in other packages are invisible to it, so hazards there + are not reported. +- Calls through interface values, struct fields, and function parameters cannot + be resolved to a body and are not followed. +- Version-aware name resolution is approximated: a task name is considered + registered when any version of it is registered. +- The checks are syntactic and type-based. A hazard reached only through + reflection, `unsafe`, code generation, or an indirection the analyzer cannot + resolve is not reported. +- Nondeterminism that is legal Go and legal durable code, such as iterating a + map without sorting, is out of scope. + +## Testing and benchmarking + +The analyzer's `analysistest` fixtures live in +`cmd/orchestratorvet/analysis/orchestratorgo/testdata/src`, split one package per scenario, +with a dedicated package of negative cases. The scenario list is discovered from +that directory, so a new fixture package is exercised as soon as it is added. + +The fix fixture packages carry `.golden` files for suggested fixes. The suite +type-checks both the goldens and the actual `TextEdits`, applied individually and +combined. Raw-edit checks do not remove unused imports or merge overlapping +edits: they enforce this analyzer's self-contained edit policy, not a general +`analysis.SuggestedFix` requirement. Golden tests exercise the pinned framework's +merge-and-import-cleanup behavior separately. + +```bash +cd cmd/orchestratorvet && go test ./... +cd cmd/orchestratorvet && go test -run '^$' -bench . ./analysis/orchestratorgo/... +``` + +The benchmarks measure orchestrator-count scaling, call-graph depth, and the +early exit taken by the overwhelming majority of packages, which register no +orchestrators at all. They type-check against the same durable task stub the +fixtures use, so the two cannot drift apart. diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go new file mode 100644 index 00000000..3b131c91 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go @@ -0,0 +1,81 @@ +// Package orchestratorgo reports replay hazards in the orchestrator functions a +// package registers with task.TaskRegistry. +// +// The analyzer starts from every proven registration in the package under +// analysis, follows the call graph through same-package named functions, +// methods, resolvable function variables, and nested function literals, and then +// reports only constructs whose nondeterminism is provable from that syntax and +// type information alone. +// +// It reports: +// +// - wall-clock reads and host timers (time.Now, time.Sleep, time.After, and +// friends) that are not routed through the orchestration APIs; +// - nondeterministic identifier and random sources (google/uuid random +// constructors, crypto/rand, the math/rand global source, and generators +// whose seed provably reaches a value that varies between runs); +// - raw goroutines and unsafe parallelism: go statements, sync and sync/atomic +// primitives, channel operations, and select, while allowing the durable task +// primitives that replace them; +// - direct filesystem, network, process, and environment I/O; +// - replay-unsafe logging through the log, log/slog, and fmt global writers, +// while allowing (*task.OrchestrationContext).Logger; +// - condition-free loops that the whole-package loop and call graph proves can +// neither exit nor make durable progress; +// - activity and sub-orchestration names that the package's own complete +// registration set proves are missing; +// - registration forms that task.TaskRegistry rejects or that derive unstable +// names, and registered orchestrators invoked as plain Go calls. +// +// The analyzer stays silent whenever proof is unavailable. Dynamic registration, +// wildcard registration, cross-package registration, functions reached only +// through values it cannot resolve, and code outside orchestrator reachability +// are all left alone. +package orchestratorgo + +import ( + "golang.org/x/tools/go/analysis" +) + +var includeTestFiles bool + +// Analyzer reports replay hazards in registered durable task orchestrators. +var Analyzer = &analysis.Analyzer{ + Name: "orchestratorgo", + Doc: "report nondeterministic and replay-unsafe code in durable task orchestrators", + URL: "https://github.com/microsoft/durabletask-go/tree/main/cmd/orchestratorvet/analysis/orchestratorgo", + Run: run, +} + +func init() { + Analyzer.Flags.BoolVar( + &includeTestFiles, + "test-files", + false, + "analyze orchestrators declared in _test.go files", + ) +} + +func run(pass *analysis.Pass) (any, error) { + index := newPackageIndex(pass) + registry := collectRegistrations(pass, index) + if len(registry.roots) == 0 { + return nil, nil + } + reach := reachableFunctions(index, registry.roots) + + check := &checker{ + pass: pass, + index: index, + registry: registry, + reported: make(map[reportKey]bool), + } + for _, node := range reach.order { + if reach.lexicallyCovered(index, node) { + continue + } + check.checkFunction(node) + } + check.reportClockDiagnostics() + return nil, nil +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go new file mode 100644 index 00000000..fb8de818 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go @@ -0,0 +1,250 @@ +package orchestratorgo_test + +import ( + "bytes" + "encoding/json" + "fmt" + "go/token" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/cmd/orchestratorvet/analysis/orchestratorgo" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/analysistest" + "golang.org/x/tools/go/packages" +) + +// fixPackages are the fixture packages that carry a .golden file and are +// therefore driven by RunWithSuggestedFixes rather than by Run. +var fixPackages = []string{ + "fixes", "fixesimport", "fixesstandalone", "fixesstandalonealias", + "fixesbatch", "fixesbatchstandalone", "fixesbatchalias", "fixespartial", "fixesinline", + "fixbindings", +} + +// stubPackageRoot is the fixture tree holding the stand-in dependencies the +// scenario packages import. It is not a scenario itself. +const stubPackageRoot = "github.com" + +const fixtureLoadMode = packages.NeedName | + packages.NeedFiles | + packages.NeedCompiledGoFiles | + packages.NeedImports | + packages.NeedTypes | + packages.NeedSyntax | + packages.NeedTypesInfo | + packages.NeedDeps + +func fixtureEnvironment(root string) []string { + return append(os.Environ(), "GOPATH="+root, "GO111MODULE=off", "GOWORK=off") +} + +// scenarioPackages discovers the analysistest fixture packages, one per +// concern, so a new fixture directory is exercised without also having to be +// listed here. Discovery failing is a test failure: a silently empty list would +// make the whole suite pass without analyzing anything. +func scenarioPackages(t *testing.T) []string { + t.Helper() + root := filepath.Join(analysistest.TestData(), "src") + entries, err := os.ReadDir(root) + if err != nil { + t.Fatalf("read fixture root %s: %v", root, err) + } + var scenarios []string + for _, entry := range entries { + if !entry.IsDir() || entry.Name() == stubPackageRoot { + continue + } + if slices.Contains(fixPackages, entry.Name()) { + continue + } + scenarios = append(scenarios, entry.Name()) + } + if len(scenarios) == 0 { + t.Fatalf("no fixture packages found under %s", root) + } + slices.Sort(scenarios) + return scenarios +} + +func TestAnalyzer(t *testing.T) { + for _, scenario := range scenarioPackages(t) { + t.Run(scenario, func(t *testing.T) { + analysistest.Run(t, analysistest.TestData(), orchestratorgo.Analyzer, scenario) + }) + } +} + +func TestAnalyzerSuggestedFixes(t *testing.T) { + analysistest.RunWithSuggestedFixes(t, analysistest.TestData(), orchestratorgo.Analyzer, fixPackages...) +} + +// TestSuggestedFixesCompile checks that the expected golden files compile. +// TestSuggestedFixTextEditsCompile separately checks the actual edits, without +// analysistest's automatic removal of unused imports. +func TestSuggestedFixesCompile(t *testing.T) { + for _, name := range fixPackages { + t.Run(name, func(t *testing.T) { + testdata := analysistest.TestData() + source := filepath.Join(testdata, "src", name, name+".go") + fixed, err := os.ReadFile(source + ".golden") + if err != nil { + t.Fatalf("read golden: %v", err) + } + checkFixedPackage(t, name, map[string][]byte{source: fixed}) + }) + } +} + +// TestSuggestedFixTextEditsCompile enforces this analyzer's stronger raw-edit +// policy. Unlike this test, x/tools' CLI resolves merges and removes unused +// imports; raw composition failing alone does not establish a CLI failure. +func TestSuggestedFixTextEditsCompile(t *testing.T) { + results := analysistest.Run(t, analysistest.TestData(), orchestratorgo.Analyzer, fixPackages...) + for _, result := range results { + if result.Pass.Pkg.Name() == "main" && strings.HasSuffix(result.Pass.Pkg.Path(), ".test") { + // Runtime fixtures also load a generated test main, not a fix scenario. + continue + } + t.Run(result.Pass.Pkg.Name(), func(t *testing.T) { + var combined []analysis.TextEdit + clockFixes := make(map[string]int) + for i, diagnostic := range result.Diagnostics { + for j, fix := range diagnostic.SuggestedFixes { + if strings.HasPrefix(diagnostic.Message, "time.Now ") { + clockFixes[result.Pass.Fset.Position(diagnostic.Pos).Filename]++ + } + t.Run(fmt.Sprintf("individual_%d_%d", i, j), func(t *testing.T) { + checkFixedPackage(t, result.Pass.Pkg.Path(), applyTextEdits(t, result.Pass.Fset, fix.TextEdits)) + }) + combined = append(combined, fix.TextEdits...) + } + } + for file, count := range clockFixes { + if count != 1 { + t.Errorf("%s: got %d clock fixes, want one atomic file-level fix", file, count) + } + } + if len(combined) == 0 { + t.Fatal("fixture produced no suggested fixes") + } + t.Run("combined", func(t *testing.T) { + checkFixedPackage(t, result.Pass.Pkg.Path(), applyTextEdits(t, result.Pass.Fset, combined)) + }) + }) + } +} + +func TestSuggestedFixesPreserveContextBindings(t *testing.T) { + results := analysistest.Run(t, analysistest.TestData(), orchestratorgo.Analyzer, "fixbindings") + var edits []analysis.TextEdit + for _, diagnostic := range results[0].Diagnostics { + if strings.HasPrefix(diagnostic.Message, "time.Now ") { + for _, fix := range diagnostic.SuggestedFixes { + edits = append(edits, fix.TextEdits...) + } + } + } + if len(edits) == 0 { + t.Fatal("fixture produced no clock fix") + } + replacements := make(map[string]string) + for original, content := range applyTextEdits(t, results[0].Pass.Fset, edits) { + fixed := filepath.Join(t.TempDir(), "fixed.go") + if err := os.WriteFile(fixed, content, 0600); err != nil { + t.Fatal(err) + } + replacements[original] = fixed + } + overlay, err := json.Marshal(struct{ Replace map[string]string }{replacements}) + if err != nil { + t.Fatal(err) + } + overlayFile := filepath.Join(t.TempDir(), "overlay.json") + if err := os.WriteFile(overlayFile, overlay, 0600); err != nil { + t.Fatal(err) + } + command := exec.CommandContext(t.Context(), "go", "test", "-vet=off", "-overlay", overlayFile, + "-run", "^TestContextBindingsRemainSafe$", "-count=1", "-timeout=30s", "fixbindings") + command.Dir = analysistest.TestData() + command.Env = fixtureEnvironment(command.Dir) + if output, err := command.CombinedOutput(); err != nil { + t.Fatalf("fixed context binding changed runtime behavior: %v\n%s", err, output) + } +} + +func checkFixedPackage(t *testing.T, name string, overlay map[string][]byte) { + t.Helper() + testdata := analysistest.TestData() + loaded, err := packages.Load(&packages.Config{ + Mode: fixtureLoadMode, + Dir: testdata, + Env: fixtureEnvironment(testdata), + Overlay: overlay, + }, name) + if err != nil { + t.Fatalf("load fixed package: %v", err) + } + if len(loaded) != 1 { + t.Fatalf("loaded %d packages, want 1", len(loaded)) + } + for _, loadError := range loaded[0].Errors { + t.Errorf("fixed %s does not compile: %v", name, loadError) + } +} + +// applyTextEdits applies only the edits supplied by the analyzer. In particular, +// it neither cleans imports nor coalesces overlapping edits from separate fixes. +func applyTextEdits(t *testing.T, fset *token.FileSet, edits []analysis.TextEdit) map[string][]byte { + t.Helper() + byFile := make(map[*token.File][]analysis.TextEdit) + for _, edit := range edits { + file := fset.File(edit.Pos) + if file == nil || fset.File(edit.End) != file { + t.Fatalf("invalid edit range: %v", edit) + } + byFile[file] = append(byFile[file], edit) + } + overlay := make(map[string][]byte) + for file, edits := range byFile { + source, err := os.ReadFile(file.Name()) + if err != nil { + t.Fatal(err) + } + slices.SortFunc(edits, func(a, b analysis.TextEdit) int { return int(a.Pos - b.Pos) }) + var fixed bytes.Buffer + offset := 0 + for _, edit := range edits { + start, end := file.Offset(edit.Pos), file.Offset(edit.End) + if start < offset || end < start || end > len(source) { + t.Fatalf("%s: overlapping or invalid edit at %d:%d", file.Name(), start, end) + } + fixed.Write(source[offset:start]) + fixed.Write(edit.NewText) + offset = end + } + fixed.Write(source[offset:]) + overlay[file.Name()] = fixed.Bytes() + } + return overlay +} + +// TestAnalyzerMetadata guards the identity the vet tool and documentation use. +func TestAnalyzerMetadata(t *testing.T) { + if orchestratorgo.Analyzer.Name != "orchestratorgo" { + t.Fatalf("analyzer name = %q, want %q", orchestratorgo.Analyzer.Name, "orchestratorgo") + } + if orchestratorgo.Analyzer.Doc == "" { + t.Fatal("analyzer doc must be set so go vet can describe the check") + } + if orchestratorgo.Analyzer.Run == nil { + t.Fatal("analyzer must have a run function") + } + if !strings.Contains(orchestratorgo.Analyzer.URL, "orchestratorgo") { + t.Fatalf("analyzer URL = %q, want it to point at the analyzer", orchestratorgo.Analyzer.URL) + } +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/bench_test.go b/cmd/orchestratorvet/analysis/orchestratorgo/bench_test.go new file mode 100644 index 00000000..5492edf5 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/bench_test.go @@ -0,0 +1,164 @@ +package orchestratorgo_test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/cmd/orchestratorvet/analysis/orchestratorgo" + "golang.org/x/tools/go/analysis" + "golang.org/x/tools/go/analysis/analysistest" + "golang.org/x/tools/go/packages" +) + +// taskStubPath is the durable task stand-in the analysistest fixtures already +// type-check against. The benchmarks reuse that one file so the surface they +// measure cannot drift away from the surface the fixtures assert on. +var taskStubPath = filepath.Join( + "testdata", "src", "github.com", "microsoft", "durabletask-go", "task", "task.go", +) + +func taskStubSource(b *testing.B) string { + b.Helper() + source, err := os.ReadFile(taskStubPath) + if err != nil { + b.Fatalf("read task stub: %v", err) + } + return string(source) +} + +// generateBenchmarkPackage builds a synthetic package with the requested number +// of orchestrators and helper-chain depth. Every orchestrator body mixes +// reported and clean constructs so the benchmark measures the full check +// pipeline rather than an early exit. When register is false the functions are +// declared but never registered, which is the shape of almost every package the +// analyzer sees. +func generateBenchmarkPackage(orchestrators, depth int, register bool) string { + var builder strings.Builder + builder.WriteString("package bench\n\nimport (\n\t\"os\"\n\t\"time\"\n\n") + builder.WriteString("\t\"github.com/microsoft/durabletask-go/task\"\n)\n\n") + + for i := 0; i < orchestrators; i++ { + fmt.Fprintf(&builder, ` +func orchestrator%d(ctx *task.OrchestrationContext) (any, error) { + total := 0 + for j := 0; j < 4; j++ { + total += j + } + if err := ctx.CallActivity("work").Await(nil); err != nil { + return nil, err + } + go func() {}() + _ = ctx.CurrentTimeUtc + _ = ctx.CreateTimer(time.Second) + return helper%d_0(total), nil +} +`, i, i) + for level := 0; level < depth; level++ { + next := fmt.Sprintf("helper%d_%d(value + 1)", i, level+1) + if level == depth-1 { + next = `len(os.Getenv("REGION")) + value` + } + fmt.Fprintf(&builder, ` +func helper%d_%d(value int) int { + if value%%2 == 0 { + value = value * 3 + } + return %s +} +`, i, level, next) + } + } + + builder.WriteString("\nfunc register() {\n\tregistry := task.NewTaskRegistry()\n") + builder.WriteString("\t_ = registry\n") + if register { + builder.WriteString("\t_ = registry.AddActivityN(\"work\", func(task.ActivityContext) (any, error) { return nil, nil })\n") + } + for i := 0; i < orchestrators; i++ { + if register { + fmt.Fprintf(&builder, "\t_ = registry.AddOrchestrator(orchestrator%d)\n", i) + } else { + fmt.Fprintf(&builder, "\t_ = orchestrator%d\n", i) + } + } + builder.WriteString("}\n") + return builder.String() +} + +// loadBenchmarkPass type-checks a generated package once and returns a pass the +// benchmark loop can run the analyzer against repeatedly. +func loadBenchmarkPass(b *testing.B, source string) *analysis.Pass { + b.Helper() + dir, cleanup, err := analysistest.WriteFiles(map[string]string{ + "github.com/microsoft/durabletask-go/task/task.go": taskStubSource(b), + "bench/bench.go": source, + }) + if err != nil { + b.Fatalf("write benchmark files: %v", err) + } + b.Cleanup(cleanup) + + config := &packages.Config{ + Mode: fixtureLoadMode | packages.NeedTypesSizes, + Dir: dir, + Env: fixtureEnvironment(dir), + } + loaded, err := packages.Load(config, "bench") + if err != nil { + b.Fatalf("load benchmark package: %v", err) + } + if len(loaded) != 1 || len(loaded[0].Errors) > 0 { + b.Fatalf("load benchmark package: %d packages, errors %v", len(loaded), loaded[0].Errors) + } + pkg := loaded[0] + return &analysis.Pass{ + Analyzer: orchestratorgo.Analyzer, + Fset: pkg.Fset, + Files: pkg.Syntax, + Pkg: pkg.Types, + TypesInfo: pkg.TypesInfo, + TypesSizes: pkg.TypesSizes, + Report: func(analysis.Diagnostic) {}, + ResultOf: map[*analysis.Analyzer]any{}, + } +} + +func runAnalyzerBenchmark(b *testing.B, source string) { + pass := loadBenchmarkPass(b, source) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, err := orchestratorgo.Analyzer.Run(pass); err != nil { + b.Fatalf("run analyzer: %v", err) + } + } +} + +// BenchmarkAnalyzerOrchestratorScaling measures how analysis cost grows with the +// number of registered orchestrators at a fixed call-graph depth. +func BenchmarkAnalyzerOrchestratorScaling(b *testing.B) { + for _, orchestrators := range []int{1, 8, 32, 128} { + b.Run(fmt.Sprintf("orchestrators=%d", orchestrators), func(b *testing.B) { + runAnalyzerBenchmark(b, generateBenchmarkPackage(orchestrators, 4, true)) + }) + } +} + +// BenchmarkAnalyzerCallGraphDepth measures how analysis cost grows with the +// length of the same-package helper chain each orchestrator reaches through. +func BenchmarkAnalyzerCallGraphDepth(b *testing.B) { + for _, depth := range []int{1, 4, 16, 64} { + b.Run(fmt.Sprintf("depth=%d", depth), func(b *testing.B) { + runAnalyzerBenchmark(b, generateBenchmarkPackage(4, depth, true)) + }) + } +} + +// BenchmarkAnalyzerNoOrchestrators measures the cost of the early exit taken by +// the overwhelming majority of packages, which register nothing at all. +func BenchmarkAnalyzerNoOrchestrators(b *testing.B) { + runAnalyzerBenchmark(b, generateBenchmarkPackage(8, 4, false)) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/checks.go b/cmd/orchestratorvet/analysis/orchestratorgo/checks.go new file mode 100644 index 00000000..f3c09601 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/checks.go @@ -0,0 +1,677 @@ +package orchestratorgo + +import ( + "fmt" + "go/ast" + "go/token" + "go/types" + "strings" + + "golang.org/x/tools/go/analysis" +) + +// Diagnostic categories keep one report per source position per concern, so a +// helper reached from several orchestrators is only reported once. +const ( + categoryGoroutine = "goroutine" + categoryWallClock = "wallclock" + categoryRandom = "random" + categorySync = "sync" + categoryChannel = "channel" + categoryIO = "io" + categoryLogging = "logging" + categoryLoop = "loop" + categoryUnregistered = "unregistered" + categoryDirectInvoke = "directinvoke" +) + +type reportKey struct { + pos token.Pos + category string +} + +type checker struct { + pass *analysis.Pass + index *packageIndex + registry *registrySet + reported map[reportKey]bool + + // Clock fixes are reported after all reachable functions have been checked, + // so each file's replacements and import cleanup form one atomic fix. + clockDiagnostics []analysis.Diagnostic +} + +func (c *checker) report( + pos token.Pos, + category string, + fixes []analysis.SuggestedFix, + format string, + args ...any, +) { + key := reportKey{pos: pos, category: category} + if c.reported[key] { + return + } + c.reported[key] = true + diagnostic := analysis.Diagnostic{ + Pos: pos, + Message: fmt.Sprintf(format, args...), + SuggestedFixes: fixes, + } + if category == categoryWallClock && len(fixes) > 0 { + c.clockDiagnostics = append(c.clockDiagnostics, diagnostic) + return + } + c.pass.Report(diagnostic) +} + +// checkFunction walks one reachable function and reports every replay hazard it +// contains, including hazards inside nested function literals. +func (c *checker) checkFunction(node ast.Node) { + body := funcBody(node) + if body == nil { + return + } + file := c.index.fileOf[node] + stack := []ast.Node{node} + ast.Inspect(body, func(current ast.Node) bool { + if current == nil { + stack = stack[:len(stack)-1] + return true + } + stack = append(stack, current) + c.inspect(current, stack, file) + return true + }) +} + +func (c *checker) inspect(node ast.Node, stack []ast.Node, file *ast.File) { + switch node := node.(type) { + case *ast.GoStmt: + c.checkGoStmt(node, stack, file) + case *ast.SelectStmt: + c.report( + node.Select, + categoryChannel, + nil, + "select statement is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).Select with task.OnTask or task.OnEvent", + ) + case *ast.SendStmt: + c.report( + node.Arrow, + categoryChannel, + nil, + "channel send is not deterministic in an orchestrator; "+ + "use task.NewEventChannel or (*task.OrchestrationContext).SendEvent", + ) + case *ast.UnaryExpr: + if node.Op == token.ARROW { + c.report( + node.OpPos, + categoryChannel, + nil, + "channel receive is not deterministic in an orchestrator; "+ + "use task.NewEventChannel or (*task.OrchestrationContext).WaitForSingleEvent", + ) + } + case *ast.RangeStmt: + if isChannel(c.pass.TypesInfo.TypeOf(node.X)) { + c.report( + node.TokPos, + categoryChannel, + nil, + "ranging over a channel is not deterministic in an orchestrator; "+ + "use task.NewEventChannel", + ) + } + case *ast.ForStmt: + c.checkUnboundedLoop(node) + case *ast.CallExpr: + c.checkCall(node, stack, file) + } +} + +func (c *checker) checkGoStmt(node *ast.GoStmt, stack []ast.Node, file *ast.File) { + c.report( + node.Go, + categoryGoroutine, + c.goStatementFix(node, contextParameter(c.pass, stack), file), + "raw go statement is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).Go", + ) +} + +func (c *checker) checkCall(call *ast.CallExpr, stack []ast.Node, file *ast.File) { + if c.checkBuiltinCall(call) { + return + } + function := staticFunc(c.pass, call.Fun) + if function == nil { + return + } + c.checkDirectOrchestratorCall(call, function) + if owner, name, ok := methodOwner(function); ok { + c.checkMethodCall(call, function, owner, name) + return + } + c.checkPackageCall(call, function, stack, file) +} + +func (c *checker) checkBuiltinCall(call *ast.CallExpr) bool { + identifier, ok := call.Fun.(*ast.Ident) + if !ok { + return false + } + builtin, ok := c.pass.TypesInfo.ObjectOf(identifier).(*types.Builtin) + if !ok { + return false + } + switch builtin.Name() { + case "make": + if len(call.Args) > 0 && isChannel(c.pass.TypesInfo.TypeOf(call.Args[0])) { + c.report( + call.Lparen, + categoryChannel, + nil, + "creating a channel is not deterministic in an orchestrator; "+ + "use task.NewEventChannel or (*task.OrchestrationContext).NewWaitGroup", + ) + } + return true + case "close": + if len(call.Args) == 1 && isChannel(c.pass.TypesInfo.TypeOf(call.Args[0])) { + c.report( + call.Lparen, + categoryChannel, + nil, + "closing a channel is not deterministic in an orchestrator; use task.NewEventChannel", + ) + } + return true + } + return false +} + +// checkPackageCall handles calls to package-level functions. +func (c *checker) checkPackageCall( + call *ast.CallExpr, + function *types.Func, + stack []ast.Node, + file *ast.File, +) { + if function.Pkg() == nil { + return + } + path, name := function.Pkg().Path(), function.Name() + switch path { + case "time": + if replacement, ok := wallClockFunctions[name]; ok { + c.report( + call.Lparen, + categoryWallClock, + c.currentTimeFix(call, name, stack, file), + "time.%s is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).%s", + name, + replacement, + ) + } + case "crypto/rand": + c.report( + call.Lparen, + categoryRandom, + nil, + "crypto/rand.%s is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).NewGuid or generate the value in an activity", + name, + ) + case "math/rand", "math/rand/v2": + if randomConstructors[name] { + return + } + c.report( + call.Lparen, + categoryRandom, + nil, + "%s.%s uses the global random source and is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).NewGuid or generate the value in an activity", + path, + name, + ) + case "github.com/google/uuid": + if !nondeterministicUUIDFunctions[name] { + return + } + c.report( + call.Lparen, + categoryRandom, + nil, + "uuid.%s is not deterministic in an orchestrator; "+ + "use (*task.OrchestrationContext).NewGuid", + name, + ) + case "sync": + if !syncOnceFunctions[name] { + return + } + c.report( + call.Lparen, + categorySync, + nil, + "sync.%s is not replay safe in an orchestrator; orchestrator coroutines are "+ + "scheduled deterministically by the task runtime", + name, + ) + case "sync/atomic": + c.report( + call.Lparen, + categorySync, + nil, + "sync/atomic.%s is not replay safe in an orchestrator; orchestrator coroutines are "+ + "scheduled deterministically by the task runtime", + name, + ) + case "os": + if externalOSFunctions[name] { + c.reportIO(call, "os."+name) + } + case "os/exec": + if name == "Command" || name == "CommandContext" || name == "LookPath" { + c.reportIO(call, "exec."+name) + } + case "net": + if hasAnyPrefix(name, "Dial", "Listen", "Lookup", "Resolve") { + c.reportIO(call, "net."+name) + } + case "net/http": + if externalHTTPFunctions[name] { + c.reportIO(call, "http."+name) + } + case "syscall": + c.reportIO(call, "syscall."+name) + case "log": + if replayUnsafeLogFunctions[name] { + c.reportLogging(call, "log."+name) + } + case "log/slog": + if processLoggerFunctions[name] { + c.report( + call.Lparen, + categoryLogging, + nil, + "slog.%s reaches the process-wide logger, which writes on every replay; "+ + "use (*task.OrchestrationContext).Logger, which suppresses output while replaying", + name, + ) + return + } + if replayUnsafeSlogFunctions[name] { + c.reportLogging(call, "slog."+name) + } + case "fmt": + c.checkFmtCall(call, name) + } +} + +// checkMethodCall handles method calls, keyed on the package and named type that +// declares the method rather than the syntactic receiver, so embedded fields and +// interface values are classified correctly. +func (c *checker) checkMethodCall( + call *ast.CallExpr, + function *types.Func, + ownerPath string, + ownerName string, +) { + name := function.Name() + switch ownerPath { + case taskPackagePath: + if ownerName == "OrchestrationContext" { + c.checkOrchestrationContextCall(call, name) + } + return + case "sync": + c.report( + call.Lparen, + categorySync, + nil, + "(sync.%s).%s is not replay safe in an orchestrator; "+ + "use (*task.OrchestrationContext).NewWaitGroup and (*task.OrchestrationContext).Go", + ownerName, + name, + ) + case "sync/atomic": + c.report( + call.Lparen, + categorySync, + nil, + "(atomic.%s).%s is not replay safe in an orchestrator; orchestrator coroutines are "+ + "scheduled deterministically by the task runtime", + ownerName, + name, + ) + case "math/rand", "math/rand/v2": + if ownerName == "Rand" && c.usesNondeterministicSource(call) { + c.report( + call.Lparen, + categoryRandom, + nil, + "(*rand.Rand).%s is seeded from a nondeterministic source; "+ + "use (*task.OrchestrationContext).NewGuid or seed from orchestration input", + name, + ) + } + case "os": + if ownerName == "Process" { + c.reportIO(call, "(*os.Process)."+name) + } + case "os/exec": + if ownerName == "Cmd" && externalCmdMethods[name] { + c.reportIO(call, "(*exec.Cmd)."+name) + } + case "net/http": + if ownerName == "Client" && externalHTTPClientMethods[name] { + c.reportIO(call, "(*http.Client)."+name) + } + case "log": + if ownerName == "Logger" && replayUnsafeLogFunctions[name] { + c.reportLogging(call, "(*log.Logger)."+name) + } + } +} + +// checkOrchestrationContextCall validates durable task names referenced from an +// orchestrator against the registrations proven for this package. +func (c *checker) checkOrchestrationContextCall(call *ast.CallExpr, method string) { + var kind registrationKind + switch method { + case "CallActivity": + kind = activityKind + case "CallSubOrchestrator": + kind = orchestratorKind + default: + return + } + if len(call.Args) == 0 || !c.registry.provesAbsence(kind) { + return + } + name, ok := c.index.taskName(call.Args[0]) + if !ok || c.registry.registered(kind, name) { + return + } + label := "activity" + if kind == orchestratorKind { + label = "sub-orchestration" + } + c.report( + call.Args[0].Pos(), + categoryUnregistered, + nil, + "%s %q is not registered with task.TaskRegistry in this package", + label, + name, + ) +} + +func (c *checker) checkFmtCall(call *ast.CallExpr, name string) { + switch name { + case "Print", "Printf", "Println": + c.reportLogging(call, "fmt."+name) + case "Fprint", "Fprintf", "Fprintln": + if len(call.Args) > 0 && isStandardStream(c.pass, call.Args[0]) { + c.reportLogging(call, "fmt."+name) + } + } +} + +func (c *checker) reportIO(call *ast.CallExpr, label string) { + c.report( + call.Lparen, + categoryIO, + nil, + "%s performs external I/O that is not replay safe in an orchestrator; "+ + "move it into an activity", + label, + ) +} + +func (c *checker) reportLogging(call *ast.CallExpr, label string) { + c.report( + call.Lparen, + categoryLogging, + nil, + "%s writes on every replay; use (*task.OrchestrationContext).Logger, "+ + "which suppresses output while replaying", + label, + ) +} + +// checkDirectOrchestratorCall reports a registered orchestrator invoked as a +// plain Go function instead of through CallSubOrchestrator. +func (c *checker) checkDirectOrchestratorCall(call *ast.CallExpr, function *types.Func) { + name, ok := c.registry.orchestratorObjects[function] + if !ok { + return + } + if name == "" { + name = function.Name() + } + c.report( + call.Lparen, + categoryDirectInvoke, + nil, + "orchestrator %q is invoked directly; "+ + "use (*task.OrchestrationContext).CallSubOrchestrator so the call is durable", + name, + ) +} + +// usesNondeterministicSource reports whether the generator a *rand.Rand method +// is called on was provably built from a seed that differs between replays of +// the same orchestration. +// +// The proof is positive: a generator is only reported when its seed reaches a +// source the analyzer knows varies between runs, such as the host clock, the +// process environment, or another random source. A constant seed, a seed taken +// from orchestration input, and a seed the analyzer cannot follow are all left +// unreported, because all three replay identically or cannot be judged at all. +func (c *checker) usesNondeterministicSource(call *ast.CallExpr) bool { + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + identifier, ok := selector.X.(*ast.Ident) + if !ok { + // A generator reached through a field or another call cannot be proven + // deterministic or nondeterministic, so it is left alone. + return false + } + constructor, ok := c.generatorConstructor(identifier) + if !ok { + return false + } + seen := make(map[types.Object]struct{}) + for _, seed := range constructor.Args { + if c.isNondeterministicValue(seed, seen) { + return true + } + } + return false +} + +// generatorConstructor returns the rand.New call an identifier holds, taken from +// the single-assignment values the package index already recorded. An identifier +// written more than once has no proven value and yields nothing. +func (c *checker) generatorConstructor(identifier *ast.Ident) (*ast.CallExpr, bool) { + value := c.index.singleValue(c.pass.TypesInfo.ObjectOf(identifier)) + call, ok := value.(*ast.CallExpr) + if !ok { + return nil, false + } + function := staticFunc(c.pass, call.Fun) + if function == nil || function.Pkg() == nil || function.Name() != "New" { + return nil, false + } + if path := function.Pkg().Path(); path != "math/rand" && path != "math/rand/v2" { + return nil, false + } + return call, true +} + +// isNondeterministicValue reports whether an expression provably reaches a value +// that differs between runs. Identifiers are followed to their single assigned +// value, so a seed hoisted into a local is judged exactly like an inline one. +func (c *checker) isNondeterministicValue(expression ast.Expr, seen map[types.Object]struct{}) bool { + found := false + ast.Inspect(expression, func(node ast.Node) bool { + if found { + return false + } + switch node := node.(type) { + case *ast.Ident: + object := c.pass.TypesInfo.ObjectOf(node) + if object == nil { + return true + } + if _, ok := seen[object]; ok { + return true + } + value := c.index.singleValue(object) + if value == nil { + return true + } + seen[object] = struct{}{} + if c.isNondeterministicValue(value, seen) { + found = true + return false + } + case *ast.CallExpr: + if c.isNondeterministicCall(node) { + found = true + return false + } + } + return true + }) + return found +} + +// isNondeterministicCall reports whether a call yields a value that is not +// reproducible on replay. +func (c *checker) isNondeterministicCall(call *ast.CallExpr) bool { + function := staticFunc(c.pass, call.Fun) + if function == nil || function.Pkg() == nil { + return false + } + name := function.Name() + switch function.Pkg().Path() { + case "time": + _, ok := wallClockFunctions[name] + return ok + case "crypto/rand": + return true + case "math/rand", "math/rand/v2": + // A constructor only wraps the seed it is handed, so it is neutral; + // the seed itself is judged by the surrounding walk. + return !randomConstructors[name] + case "github.com/google/uuid": + return nondeterministicUUIDFunctions[name] + case "os": + return externalOSFunctions[name] + } + return false +} + +// contextParameter returns the object of the innermost *task.OrchestrationContext +// parameter in scope, which suggested fixes need to rewrite calls. +func contextParameter(pass *analysis.Pass, stack []ast.Node) types.Object { + for i := len(stack) - 1; i >= 0; i-- { + signature := funcType(stack[i]) + if signature == nil { + continue + } + if parameter := orchestrationContextParam(pass, signature); parameter != nil { + return parameter + } + } + return nil +} + +func orchestrationContextParam(pass *analysis.Pass, signature *ast.FuncType) types.Object { + if signature.Params == nil { + return nil + } + for _, field := range signature.Params.List { + if !isOrchestrationContextType(pass.TypesInfo.TypeOf(field.Type)) { + continue + } + for _, name := range field.Names { + if name.Name != "_" && name.Name != "" { + return pass.TypesInfo.Defs[name] + } + } + } + return nil +} + +func isOrchestrationContextType(value types.Type) bool { + pointer, ok := types.Unalias(value).(*types.Pointer) + if !ok { + return false + } + named, ok := types.Unalias(pointer.Elem()).(*types.Named) + if !ok { + return false + } + object := named.Obj() + return object.Pkg() != nil && + object.Pkg().Path() == taskPackagePath && + object.Name() == "OrchestrationContext" +} + +// methodOwner returns the package path and named type that declares a method. +func methodOwner(function *types.Func) (string, string, bool) { + signature, ok := function.Type().(*types.Signature) + if !ok || signature.Recv() == nil { + return "", "", false + } + receiver := types.Unalias(signature.Recv().Type()) + if pointer, ok := receiver.(*types.Pointer); ok { + receiver = types.Unalias(pointer.Elem()) + } + named, ok := receiver.(*types.Named) + if !ok || named.Obj().Pkg() == nil { + return "", "", false + } + return named.Obj().Pkg().Path(), named.Obj().Name(), true +} + +func isChannel(value types.Type) bool { + if value == nil { + return false + } + _, ok := value.Underlying().(*types.Chan) + return ok +} + +func isStandardStream(pass *analysis.Pass, expression ast.Expr) bool { + selector, ok := expression.(*ast.SelectorExpr) + if !ok { + return false + } + object := pass.TypesInfo.ObjectOf(selector.Sel) + variable, ok := object.(*types.Var) + if !ok || variable.Pkg() == nil || variable.Pkg().Path() != "os" { + return false + } + return variable.Name() == "Stdout" || variable.Name() == "Stderr" +} + +func hasAnyPrefix(value string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(value, prefix) { + return true + } + } + return false +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/fixes.go b/cmd/orchestratorvet/analysis/orchestratorgo/fixes.go new file mode 100644 index 00000000..f0c4542d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/fixes.go @@ -0,0 +1,265 @@ +package orchestratorgo + +import ( + "go/ast" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" +) + +// currentTimeFix rewrites time.Now() as ctx.CurrentTimeUtc when an orchestration +// context is in scope. Only the zero-argument clock reads have a one-to-one +// replacement, so no other wall-clock call is offered a fix. +// +// reportClockDiagnostics combines these per-call bindings before offering a fix. +func (c *checker) currentTimeFix( + call *ast.CallExpr, + name string, + stack []ast.Node, + file *ast.File, +) []analysis.SuggestedFix { + context := contextParameter(c.pass, stack) + if name != "Now" || len(call.Args) != 0 || !c.bindingAvailable(context, file, call.Pos()) { + return nil + } + // A field read cannot replace a call used as a statement, go, or defer. + for i := len(stack) - 2; i >= 0; i-- { + switch stack[i].(type) { + case *ast.ParenExpr: + continue + case *ast.ExprStmt, *ast.GoStmt, *ast.DeferStmt: + return nil + } + break + } + edits := []analysis.TextEdit{{ + Pos: call.Pos(), + End: call.End(), + NewText: []byte(context.Name() + ".CurrentTimeUtc"), + }} + return []analysis.SuggestedFix{{ + Message: "use (*task.OrchestrationContext).CurrentTimeUtc for fixable time.Now calls in this file", + TextEdits: edits, + }} +} + +// reportClockDiagnostics attaches each file's complete clock fix to only its +// first fixable diagnostic. Every other clock diagnostic remains visible, but +// offers no partial fix that could conflict with the file's import cleanup. +func (c *checker) reportClockDiagnostics() { + if len(c.clockDiagnostics) == 0 { + return + } + fixes := make(map[*token.File]*analysis.SuggestedFix) + for i := range c.clockDiagnostics { + diagnostic := &c.clockDiagnostics[i] + file := c.pass.Fset.File(diagnostic.Pos) + fix := &diagnostic.SuggestedFixes[0] + if first := fixes[file]; first != nil { + first.TextEdits = append(first.TextEdits, fix.TextEdits...) + diagnostic.SuggestedFixes = nil + } else { + fixes[file] = fix + } + } + for _, file := range c.pass.Files { + if fix := fixes[c.pass.Fset.File(file.Pos())]; fix != nil { + fix.TextEdits = append(fix.TextEdits, c.unusedTimeImportRemovals(file, fix.TextEdits)...) + } + } + for _, diagnostic := range c.clockDiagnostics { + c.pass.Report(diagnostic) + } +} + +// goStatementFix rewrites `go func() { ... }()` as an orchestration coroutine. +// It only applies to an immediately invoked literal with no parameters, no +// results, and no arguments, because any other form would change the meaning of +// the captured values or the call. +// +// The literal's body is never reprinted. Comments live on the file rather than +// on the statements they annotate, so printing the block would silently drop +// them; editing only the text around the braces leaves the body, and everything +// written inside it, exactly as the author wrote it. +func (c *checker) goStatementFix( + statement *ast.GoStmt, + context types.Object, + file *ast.File, +) []analysis.SuggestedFix { + if !c.bindingAvailable(context, file, statement.Pos()) { + return nil + } + literal, ok := statement.Call.Fun.(*ast.FuncLit) + if !ok || len(statement.Call.Args) != 0 || statement.Call.Ellipsis.IsValid() { + return nil + } + if literal.Body == nil || !literal.Body.Lbrace.IsValid() || !literal.Body.Rbrace.IsValid() { + return nil + } + signature := literal.Type + if signature.Params != nil && len(signature.Params.List) != 0 { + return nil + } + if signature.Results != nil && len(signature.Results.List) != 0 { + return nil + } + qualifier := taskImport(c.pass, file) + if qualifier == nil || !c.bindingAvailable(qualifier, file, statement.Pos()) { + return nil + } + return []analysis.SuggestedFix{{ + Message: "use (*task.OrchestrationContext).Go", + TextEdits: []analysis.TextEdit{ + { + // `go func() ` becomes `ctx.Go(func(*task.OrchestrationContext) `, + // stopping short of the brace so the body is untouched. + Pos: statement.Pos(), + End: literal.Body.Lbrace, + NewText: []byte(context.Name() + ".Go(func(*" + qualifier.Name() + ".OrchestrationContext) "), + }, + { + // The trailing `()` of the immediate invocation becomes the + // closing paren of the Go call. + Pos: literal.Body.Rbrace + 1, + End: statement.End(), + NewText: []byte(")"), + }, + }, + }} +} + +func (c *checker) bindingAvailable(object types.Object, file *ast.File, position token.Pos) bool { + if object == nil || file == nil || len(c.index.functionValues[object]) != 0 { + return false + } + scope := c.pass.TypesInfo.Scopes[file] + if scope == nil { + return false + } + scope = scope.Innermost(position) + if scope == nil { + return false + } + _, visible := scope.LookupParent(object.Name(), position) + // A shadow can have the same type while referring to an unrelated value. + return visible == object +} + +// unusedTimeImportRemovals checks the file once, ignoring only expressions the +// atomic clock fix replaces. Unreachable or unfixable uses keep their imports. +func (c *checker) unusedTimeImportRemovals( + file *ast.File, + replacements []analysis.TextEdit, +) []analysis.TextEdit { + replaced := make(map[token.Pos]token.Pos, len(replacements)) + for _, edit := range replacements { + replaced[edit.Pos] = edit.End + } + usedImports := make(map[*types.PkgName]bool) + usedPackages := make(map[*types.Package]bool) + ast.Inspect(file, func(node ast.Node) bool { + if node == nil { + return true + } + if replaced[node.Pos()] == node.End() { + return false + } + if identifier, ok := node.(*ast.Ident); ok { + object := c.pass.TypesInfo.Uses[identifier] + if name, ok := object.(*types.PkgName); ok { + usedImports[name] = true + } else if object != nil { + usedPackages[object.Pkg()] = true + } + } + return true + }) + var edits []analysis.TextEdit + for declarationIndex, declaration := range file.Decls { + importDeclaration, ok := declaration.(*ast.GenDecl) + if !ok || importDeclaration.Tok != token.IMPORT { + continue + } + var unused []int + for i, specification := range importDeclaration.Specs { + specification := specification.(*ast.ImportSpec) + object := c.pass.TypesInfo.Implicits[specification] + if specification.Name != nil { + object = c.pass.TypesInfo.Defs[specification.Name] + } + name, ok := object.(*types.PkgName) + if !ok || name.Imported().Path() != "time" || name.Name() == "_" || + usedImports[name] || name.Name() == "." && usedPackages[name.Imported()] { + continue + } + unused = append(unused, i) + } + if len(unused) == 0 { + continue + } + if len(unused) == len(importDeclaration.Specs) { + end := importDeclaration.End() + if tokenFile := c.pass.Fset.File(end); tokenFile != nil { + line := tokenFile.Line(end) + end = tokenFile.Pos(tokenFile.Size()) + if line < tokenFile.LineCount() { + end = tokenFile.LineStart(line + 1) + } + } + if declarationIndex+1 < len(file.Decls) && file.Decls[declarationIndex+1].Pos() < end { + end = file.Decls[declarationIndex+1].Pos() + } + edits = append(edits, analysis.TextEdit{ + Pos: importDeclaration.Pos(), + End: end, + }) + continue + } + for _, i := range unused { + end := importDeclaration.Rparen + if i+1 < len(importDeclaration.Specs) { + end = importDeclaration.Specs[i+1].Pos() + } + // Include separators, but never a grouped import's closing paren. + edits = append(edits, analysis.TextEdit{ + Pos: importDeclaration.Specs[i].Pos(), + End: end, + }) + } + } + // Ranges are in source order. Preserve comments swept up with import + // separators, including comments between declarations, in one linear pass. + i := 0 + for _, group := range file.Comments { + for _, comment := range group.List { + for i < len(edits) && edits[i].End <= comment.Pos() { + i++ + } + if i == len(edits) { + return edits + } + if edits[i].Pos <= comment.Pos() { + edits[i].NewText = append(edits[i].NewText, comment.Text...) + edits[i].NewText = append(edits[i].NewText, '\n') + } + } + } + return edits +} + +// taskImport returns the package binding the file uses for the durable task +// package, so generated code compiles under dot-free aliases too. +func taskImport(pass *analysis.Pass, file *ast.File) *types.PkgName { + for _, specification := range file.Imports { + object := pass.TypesInfo.Implicits[specification] + if specification.Name != nil { + object = pass.TypesInfo.Defs[specification.Name] + } + name, ok := object.(*types.PkgName) + if ok && name.Imported().Path() == taskPackagePath && name.Name() != "." && name.Name() != "_" { + return name + } + } + return nil +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/index.go b/cmd/orchestratorvet/analysis/orchestratorgo/index.go new file mode 100644 index 00000000..e63f441b --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/index.go @@ -0,0 +1,419 @@ +package orchestratorgo + +import ( + "go/ast" + "go/token" + "go/types" + "strings" + + "golang.org/x/tools/go/analysis" +) + +// packageIndex holds the whole-package declaration information the analyzer +// needs to follow call graphs without loading other packages. Everything it +// records is derived from the syntax and type information of a single +// [analysis.Pass], so the analyzer never depends on facts from dependencies. +type packageIndex struct { + pass *analysis.Pass + + // functions maps a declared function or method object to its declaration. + functions map[types.Object]*ast.FuncDecl + + // functionValues maps a variable or constant object to the expressions it is + // initialized or assigned with. Objects with more than one recorded + // expression are treated as unresolved so that dynamic reassignment never + // produces a diagnostic. A nil expression records an unknown/tuple write. + functionValues map[types.Object][]ast.Expr + + // fileOf maps every function declaration and literal the analyzer can reach + // to the file that contains it, which suggested fixes use to qualify types + // with the import name that file actually uses. + fileOf map[ast.Node]*ast.File + + // enclosingFunc maps every function literal to the nearest function + // declaration or literal that lexically contains it, and is absent for a + // literal written outside any function, such as a package-level variable + // initializer. Walking a function body already visits the literals nested + // in it, so this is what lets the checker walk each one exactly once. + enclosingFunc map[ast.Node]ast.Node + + // registrationCandidates holds, in source order, every call whose selector + // name matches a task.TaskRegistry registration method. Collecting them + // during the single indexing walk keeps packages that register nothing -- + // which is almost every package -- down to one pass over the syntax. + registrationCandidates []*ast.CallExpr + + // registrationScopes identifies direct registration statements in the same + // block or switch/select clause, never conditional subexpressions. + registrationScopes map[*ast.CallExpr]ast.Node + + // Opaque uses can register either namespace, or replace registry contents. + registrationsIncomplete bool + unstableRegistries map[types.Object]bool + + // Reuse the ancestor stack across bodies rather than allocating per function. + stack []ast.Node +} + +func newPackageIndex(pass *analysis.Pass) *packageIndex { + index := &packageIndex{ + pass: pass, + functions: make(map[types.Object]*ast.FuncDecl), + functionValues: make(map[types.Object][]ast.Expr), + fileOf: make(map[ast.Node]*ast.File), + enclosingFunc: make(map[ast.Node]ast.Node), + registrationScopes: make(map[*ast.CallExpr]ast.Node), + unstableRegistries: make(map[types.Object]bool), + } + for _, file := range pass.Files { + if !includeTestFiles && + strings.HasSuffix(pass.Fset.PositionFor(file.Pos(), false).Filename, "_test.go") { + continue + } + ast.Inspect(file, func(node ast.Node) bool { + if node == nil { + index.stack = index.stack[:len(index.stack)-1] + return true + } + switch node := node.(type) { + case *ast.FuncDecl: + index.fileOf[node] = file + if object := pass.TypesInfo.Defs[node.Name]; object != nil { + index.functions[object] = node + } + index.walkFunction(node, file, node.Body) + // walkFunction already indexed the body while tracking the + // enclosing function, so the outer traversal stops here. + return false + case *ast.FuncLit: + // A literal reached from file scope, such as a package-level + // variable initializer, has no enclosing function and is a + // root the checker must walk on its own. + index.fileOf[node] = file + index.walkFunction(node, file, node.Body) + return false + } + index.record(node) + index.stack = append(index.stack, node) + return true + }) + } + return index +} + +// walkFunction indexes one function body, recording the enclosing function of +// every literal nested in it. +func (index *packageIndex) walkFunction(owner ast.Node, file *ast.File, body *ast.BlockStmt) { + if body == nil { + return + } + if parameters := funcType(owner).Params; parameters != nil { + for _, field := range parameters.List { + if isTaskRegistry(index.pass.TypesInfo.TypeOf(field.Type)) { + for _, name := range field.Names { + index.recordValue(index.pass.TypesInfo.Defs[name], nil) + } + } + } + } + var registrations []*ast.CallExpr + hasGoto := false + index.stack = append(index.stack, owner) + ast.Inspect(body, func(node ast.Node) bool { + if node == nil { + index.stack = index.stack[:len(index.stack)-1] + return true + } + if literal, ok := node.(*ast.FuncLit); ok { + index.fileOf[literal] = file + index.enclosingFunc[literal] = owner + index.walkFunction(literal, file, literal.Body) + return false + } + index.record(node) + switch node := node.(type) { + case *ast.BranchStmt: + hasGoto = hasGoto || node.Tok == token.GOTO + case *ast.CallExpr: + if _, ok := index.registrationScopes[node]; ok { + registrations = append(registrations, node) + } + } + index.stack = append(index.stack, node) + return true + }) + index.stack = index.stack[:len(index.stack)-1] + if hasGoto { + // A jump can bypass another direct registration in this function. + for _, call := range registrations { + delete(index.registrationScopes, call) + } + } +} + +// record captures the declaration facts the analyzer needs from one node. +func (index *packageIndex) record(node ast.Node) { + pass := index.pass + switch node := node.(type) { + case *ast.ValueSpec: + if len(node.Values) == 0 { + return + } + for i, name := range node.Names { + var value ast.Expr + if len(node.Names) == len(node.Values) { + value = node.Values[i] + } + index.recordValue(pass.TypesInfo.Defs[name], value) + } + case *ast.AssignStmt: + for i, left := range node.Lhs { + var value ast.Expr + if len(node.Lhs) == len(node.Rhs) { + value = node.Rhs[i] + } + left = ast.Unparen(left) + identifier, ok := left.(*ast.Ident) + if !ok { + index.recordRegistryEscape(value) + if pointer, ok := left.(*ast.StarExpr); ok { + index.recordRegistryEscape(pointer.X) + } + continue + } + index.recordValue(pass.TypesInfo.ObjectOf(identifier), value) + } + case *ast.RangeStmt: + for _, target := range []ast.Expr{node.Key, node.Value} { + if identifier, ok := ast.Unparen(target).(*ast.Ident); ok { + index.recordValue(pass.TypesInfo.ObjectOf(identifier), nil) + } + } + case *ast.CallExpr: + if selector, ok := node.Fun.(*ast.SelectorExpr); ok { + if _, ok := registrationShapes[selector.Sel.Name]; ok { + index.registrationCandidates = append(index.registrationCandidates, node) + index.registrationScopes[node] = registrationScope(index.stack) + } + } + for _, argument := range node.Args { + index.recordRegistryEscape(argument) + } + if isTaskRegistry(pass.TypesInfo.TypeOf(node)) && !isRegistryConstructor(pass, node) { + index.registrationsIncomplete = true + } + case *ast.SelectorExpr: + if selection := pass.TypesInfo.Selections[node]; selection != nil && isTaskRegistry(selection.Recv()) { + if len(index.stack) > 0 { + if call, ok := index.stack[len(index.stack)-1].(*ast.CallExpr); ok && call.Fun == node { + if _, _, ok := registrationCall(pass, call); ok { + return + } + } + } + // A bound method can register tasks through an unindexed call. + index.recordRegistryEscape(node.X) + } + case *ast.UnaryExpr: + if node.Op == token.AND { + index.recordRegistryEscape(node.X) + } + case *ast.ReturnStmt: + for _, result := range node.Results { + index.recordRegistryEscape(result) + } + case *ast.SendStmt: + index.recordRegistryEscape(node.Value) + case *ast.CompositeLit: + for _, element := range node.Elts { + if keyed, ok := element.(*ast.KeyValueExpr); ok { + index.recordRegistryEscape(keyed.Key) + element = keyed.Value + } + index.recordRegistryEscape(element) + } + } +} + +func (index *packageIndex) recordValue(object types.Object, value ast.Expr) { + if object == nil || object.Name() == "_" { + return + } + index.functionValues[object] = append(index.functionValues[object], value) + if value == nil || !isTaskRegistry(index.pass.TypesInfo.TypeOf(value)) { + return + } + if identifier, ok := ast.Unparen(value).(*ast.Ident); ok { + // An alias permits writes the original variable's assignment count + // cannot see. Do not try to prove instance stability through aliases. + index.unstableRegistries[index.pass.TypesInfo.ObjectOf(identifier)] = true + } + if object.Parent() == index.pass.Pkg.Scope() || !isTaskRegistry(object.Type()) { + // Global storage and interface erasure hide subsequent registrations. + index.recordRegistryEscape(value) + index.unstableRegistries[object] = true + } +} + +func (index *packageIndex) recordRegistryEscape(expression ast.Expr) { + if expression == nil || !isTaskRegistry(index.pass.TypesInfo.TypeOf(expression)) { + return + } + index.registrationsIncomplete = true + if identifier, ok := ast.Unparen(expression).(*ast.Ident); ok { + index.unstableRegistries[index.pass.TypesInfo.ObjectOf(identifier)] = true + } +} + +// registrationScope accepts only an entire expression/assignment statement in +// a sequential statement list. In particular, nearest-block matching would +// conflate separate switch/select clauses and short-circuit operands. +func registrationScope(stack []ast.Node) ast.Node { + i := len(stack) - 1 + for i >= 0 { + if _, ok := stack[i].(*ast.ParenExpr); !ok { + break + } + i-- + } + if i < 1 { + return nil + } + switch stack[i].(type) { + case *ast.ExprStmt, *ast.AssignStmt: + switch scope := stack[i-1].(type) { + case *ast.BlockStmt, *ast.CaseClause, *ast.CommClause: + return scope + } + } + return nil +} + +// singleValue returns the only expression an object is ever initialized or +// assigned with, or nil when the object is unset or written more than once. +func (index *packageIndex) singleValue(object types.Object) ast.Expr { + if object == nil { + return nil + } + values := index.functionValues[object] + if len(values) != 1 { + return nil + } + return values[0] +} + +// resolveFunction returns the function declaration or literal that expression +// evaluates to, or nil when the target cannot be proven statically. +func (index *packageIndex) resolveFunction(expression ast.Expr, seen map[types.Object]struct{}) ast.Node { + switch expression := expression.(type) { + case *ast.FuncLit: + return expression + case *ast.ParenExpr: + return index.resolveFunction(expression.X, seen) + case *ast.Ident: + return index.resolveObject(index.pass.TypesInfo.ObjectOf(expression), seen) + case *ast.SelectorExpr: + if selection := index.pass.TypesInfo.Selections[expression]; selection != nil { + return index.resolveObject(selection.Obj(), seen) + } + return index.resolveObject(index.pass.TypesInfo.ObjectOf(expression.Sel), seen) + case *ast.IndexExpr: + // Instantiation of a generic function, such as helper[int]. + return index.resolveFunction(expression.X, seen) + case *ast.IndexListExpr: + return index.resolveFunction(expression.X, seen) + case *ast.CallExpr: + // Conversions such as task.Orchestrator(fn) forward to their operand. + if typeInfo, ok := index.pass.TypesInfo.Types[expression.Fun]; ok && + typeInfo.IsType() && + len(expression.Args) == 1 { + return index.resolveFunction(expression.Args[0], seen) + } + } + return nil +} + +func (index *packageIndex) resolveObject(object types.Object, seen map[types.Object]struct{}) ast.Node { + if object == nil { + return nil + } + if function := index.functions[object]; function != nil { + return function + } + if seen == nil { + seen = make(map[types.Object]struct{}) + } + if _, ok := seen[object]; ok { + return nil + } + seen[object] = struct{}{} + value := index.singleValue(object) + if value == nil { + // Unset, or reassigned more than once: the target is ambiguous. + return nil + } + return index.resolveFunction(value, seen) +} + +// callee returns the function declaration or literal invoked by call when that +// target is declared in the package under analysis. +// +// Calls into the durable task package are never followed. Its channels, +// goroutines, and locks implement the deterministic orchestration scheduler, so +// auditing them would report the very primitives the analyzer recommends. This +// only has an effect when the durable task package analyzes itself; for every +// other package the target is already outside the index. +func (index *packageIndex) callee(call *ast.CallExpr) ast.Node { + if function := staticFunc(index.pass, call.Fun); function != nil { + if pkg := function.Pkg(); pkg != nil && pkg.Path() == taskPackagePath { + return nil + } + } + return index.resolveFunction(call.Fun, nil) +} + +// staticFunc returns the function object an expression denotes, which may be a +// package-level function or a method. +func staticFunc(pass *analysis.Pass, expression ast.Expr) *types.Func { + switch expression := expression.(type) { + case *ast.ParenExpr: + return staticFunc(pass, expression.X) + case *ast.Ident: + function, _ := pass.TypesInfo.ObjectOf(expression).(*types.Func) + return function + case *ast.SelectorExpr: + if selection := pass.TypesInfo.Selections[expression]; selection != nil { + function, _ := selection.Obj().(*types.Func) + return function + } + function, _ := pass.TypesInfo.ObjectOf(expression.Sel).(*types.Func) + return function + case *ast.IndexExpr: + return staticFunc(pass, expression.X) + case *ast.IndexListExpr: + return staticFunc(pass, expression.X) + } + return nil +} + +// funcBody returns the body of a function declaration or literal. +func funcBody(node ast.Node) *ast.BlockStmt { + switch node := node.(type) { + case *ast.FuncDecl: + return node.Body + case *ast.FuncLit: + return node.Body + } + return nil +} + +// funcType returns the signature syntax of a function declaration or literal. +func funcType(node ast.Node) *ast.FuncType { + switch node := node.(type) { + case *ast.FuncDecl: + return node.Type + case *ast.FuncLit: + return node.Type + } + return nil +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/loops.go b/cmd/orchestratorvet/analysis/orchestratorgo/loops.go new file mode 100644 index 00000000..715643c5 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/loops.go @@ -0,0 +1,143 @@ +package orchestratorgo + +import ( + "go/ast" + "go/constant" + "go/token" + "go/types" + + "golang.org/x/tools/go/analysis" +) + +// checkUnboundedLoop reports a condition-free orchestrator loop only when the +// whole-package loop and call graph proves the body can neither leave the loop +// nor make durable progress. Any call the analyzer cannot follow, and any use of +// the task package, leaves the loop unreported. +func (c *checker) checkUnboundedLoop(loop *ast.ForStmt) { + if loop.Cond != nil && !isConstantTrue(c.pass, loop.Cond) { + return + } + if loop.Body == nil { + // `for {}` with an empty body is unambiguously a spin. + c.reportUnboundedLoop(loop) + return + } + probe := &loopProbe{ + checker: c, + visitedCallees: make(map[ast.Node]bool), + } + probe.scan(loop.Body, true) + if probe.inconclusive { + return + } + c.reportUnboundedLoop(loop) +} + +func (c *checker) reportUnboundedLoop(loop *ast.ForStmt) { + c.report( + loop.For, + categoryLoop, + nil, + "unbounded orchestrator loop never awaits a durable task, waits for an event, "+ + "or calls ContinueAsNew, so it cannot make progress or complete", + ) +} + +// loopProbe walks a loop body and everything it calls inside the package. +// inconclusive means the loop could not be proven to be a pure spin. +type loopProbe struct { + checker *checker + visitedCallees map[ast.Node]bool + inconclusive bool +} + +// scan inspects one block and everything it calls inside the package. Blocking +// operations and calls the analyzer cannot follow end the proof everywhere. +// +// inLoop distinguishes the statements written directly in the loop from the body +// of a function the loop calls. In the loop, a control transfer or a closure +// means the loop may exit or do something the probe cannot see, so the proof +// ends; in a callee, returning is ordinary control flow and says nothing about +// the loop. +func (p *loopProbe) scan(body *ast.BlockStmt, inLoop bool) { + ast.Inspect(body, func(node ast.Node) bool { + if p.inconclusive { + return false + } + switch node := node.(type) { + case *ast.SelectStmt, *ast.SendStmt, *ast.GoStmt: + p.inconclusive = true + return false + case *ast.ReturnStmt, *ast.BranchStmt, *ast.LabeledStmt, + *ast.FuncLit, *ast.DeferStmt: + if inLoop { + p.inconclusive = true + return false + } + case *ast.UnaryExpr: + if node.Op == token.ARROW { + p.inconclusive = true + return false + } + case *ast.RangeStmt: + if isChannel(p.checker.pass.TypesInfo.TypeOf(node.X)) { + p.inconclusive = true + return false + } + case *ast.CallExpr: + p.scanCall(node) + } + return true + }) +} + +func (p *loopProbe) scanCall(call *ast.CallExpr) { + if identifier, ok := call.Fun.(*ast.Ident); ok { + if builtin, ok := p.checker.pass.TypesInfo.ObjectOf(identifier).(*types.Builtin); ok { + switch builtin.Name() { + case "panic", "recover": + p.inconclusive = true + } + return + } + } + if callee := p.checker.index.callee(call); callee != nil { + p.scanCallee(callee) + return + } + function := staticFunc(p.checker.pass, call.Fun) + if function == nil || function.Pkg() == nil { + p.inconclusive = true + return + } + switch path := function.Pkg().Path(); { + case path == taskPackagePath: + // Any durable task interaction may make progress or block. + p.inconclusive = true + case pureLoopPackages[path]: + default: + p.inconclusive = true + } +} + +func (p *loopProbe) scanCallee(node ast.Node) { + if p.visitedCallees[node] { + // Recursion is safe because the function is already being scanned. + return + } + p.visitedCallees[node] = true + body := funcBody(node) + if body == nil { + p.inconclusive = true + return + } + p.scan(body, false) +} + +func isConstantTrue(pass *analysis.Pass, expression ast.Expr) bool { + typeInfo, ok := pass.TypesInfo.Types[expression] + if !ok || typeInfo.Value == nil || typeInfo.Value.Kind() != constant.Bool { + return false + } + return constant.BoolVal(typeInfo.Value) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/reach.go b/cmd/orchestratorvet/analysis/orchestratorgo/reach.go new file mode 100644 index 00000000..16dd9775 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/reach.go @@ -0,0 +1,71 @@ +package orchestratorgo + +import ( + "go/ast" +) + +// reachSet is the deterministic set of package functions reachable from the +// registered orchestrators. +type reachSet struct { + order []ast.Node + set map[ast.Node]bool +} + +// lexicallyCovered reports whether node is a function literal nested inside +// another reachable function. Walking that outer function already visits the +// literal's body, so checking the literal separately would walk it a second +// time. It would also lose context: a fix rewritten from the outer walk can see +// the enclosing orchestration context parameter, which is out of scope when the +// literal is walked on its own. +// +// A literal written outside any function, such as a package-level function +// variable, has no enclosing function and is therefore never covered. +func (reach *reachSet) lexicallyCovered(index *packageIndex, node ast.Node) bool { + for parent := index.enclosingFunc[node]; parent != nil; parent = index.enclosingFunc[parent] { + if reach.set[parent] { + return true + } + } + return false +} + +// reachableFunctions walks the whole-package call graph from roots, following +// named helpers, methods, resolvable function variables, and nested literals. +// Recursion terminates because each function node is added at most once. +func reachableFunctions(index *packageIndex, roots []ast.Node) *reachSet { + reach := &reachSet{set: make(map[ast.Node]bool)} + for _, root := range roots { + if root == nil || reach.set[root] { + continue + } + reach.set[root] = true + reach.order = append(reach.order, root) + } + for next := 0; next < len(reach.order); next++ { + current := reach.order[next] + forEachCall(current, func(call *ast.CallExpr) { + callee := index.callee(call) + if callee == nil || reach.set[callee] { + return + } + reach.set[callee] = true + reach.order = append(reach.order, callee) + }) + } + return reach +} + +// forEachCall visits every call expression lexically contained in a function, +// including calls inside nested function literals, go statements, and defers. +func forEachCall(node ast.Node, visit func(*ast.CallExpr)) { + body := funcBody(node) + if body == nil { + return + } + ast.Inspect(body, func(node ast.Node) bool { + if call, ok := node.(*ast.CallExpr); ok { + visit(call) + } + return true + }) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/registration.go b/cmd/orchestratorvet/analysis/orchestratorgo/registration.go new file mode 100644 index 00000000..f0405708 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/registration.go @@ -0,0 +1,396 @@ +package orchestratorgo + +import ( + "go/ast" + "go/constant" + "go/types" + "strings" + + "golang.org/x/tools/go/analysis" +) + +const taskPackagePath = "github.com/microsoft/durabletask-go/task" + +// registrationKind distinguishes the two task namespaces the analyzer tracks. +type registrationKind int + +const ( + orchestratorKind registrationKind = iota + activityKind +) + +func (kind registrationKind) String() string { + if kind == orchestratorKind { + return "orchestrator" + } + return "activity" +} + +// registrationShape describes where the name, version, and handler live in the +// argument list of one task.TaskRegistry registration method. +type registrationShape struct { + kind registrationKind + nameIndex int // -1 when the name is derived by reflection + handlerIndex int + versionIndex int // -1 when the method has no version parameter + arity int +} + +var registrationShapes = map[string]registrationShape{ + "AddOrchestrator": {kind: orchestratorKind, nameIndex: -1, handlerIndex: 0, versionIndex: -1, arity: 1}, + "AddOrchestratorN": {kind: orchestratorKind, nameIndex: 0, handlerIndex: 1, versionIndex: -1, arity: 2}, + "AddOrchestratorVersion": {kind: orchestratorKind, nameIndex: -1, handlerIndex: 1, versionIndex: 0, arity: 2}, + "AddOrchestratorNVersion": {kind: orchestratorKind, nameIndex: 0, handlerIndex: 2, versionIndex: 1, arity: 3}, + "AddActivity": {kind: activityKind, nameIndex: -1, handlerIndex: 0, versionIndex: -1, arity: 1}, + "AddActivityN": {kind: activityKind, nameIndex: 0, handlerIndex: 1, versionIndex: -1, arity: 2}, + "AddActivityVersion": {kind: activityKind, nameIndex: -1, handlerIndex: 1, versionIndex: 0, arity: 2}, + "AddActivityNVersion": {kind: activityKind, nameIndex: 0, handlerIndex: 2, versionIndex: 1, arity: 3}, +} + +// registration is a single proven call to a task.TaskRegistry Add* method. +type registration struct { + // name is the registered task name when it can be proven statically. + name string + // nameKnown reports whether name was proven rather than guessed. + nameKnown bool + // version is the registered version when it is a string literal. + version string + // versionKnown reports whether the registration's version was proven. A + // version computed at runtime leaves it false, which keeps the registration + // out of duplicate detection: two calls with different computed versions + // are not a conflict. + versionKnown bool +} + +// registrySet is the whole-package view of task.TaskRegistry registrations. +type registrySet struct { + // names holds the lowercased proven names per namespace. + names map[registrationKind]map[string]struct{} + // dynamic reports that at least one registration in the namespace has a name + // the analyzer could not prove, so absence from names proves nothing. + dynamic map[registrationKind]bool + // incomplete covers opaque registry uses that can register either kind. + incomplete bool + + // orchestratorObjects holds the objects of registered orchestrator + // declarations, used to detect direct orchestrator invocation. + orchestratorObjects map[types.Object]string + // roots holds every function node registered as an orchestrator. + roots []ast.Node +} + +func (set *registrySet) registered(kind registrationKind, name string) bool { + _, ok := set.names[kind][strings.ToLower(name)] + return ok +} + +// provesAbsence reports whether the package's registrations are complete enough +// that a missing name is a real error rather than an artifact of cross-package +// or dynamic registration. +func (set *registrySet) provesAbsence(kind registrationKind) bool { + if set.incomplete || set.dynamic[kind] || set.registered(kind, "*") { + return false + } + return len(set.names[kind]) > 0 +} + +// collectRegistrations walks the package's registration calls and reports +// registration forms that task.TaskRegistry rejects or that derive an unstable +// name. The candidate calls were gathered during package indexing, so packages +// that register nothing cost nothing here. +func collectRegistrations(pass *analysis.Pass, index *packageIndex) *registrySet { + set := ®istrySet{ + names: map[registrationKind]map[string]struct{}{orchestratorKind: {}, activityKind: {}}, + dynamic: map[registrationKind]bool{}, + incomplete: index.registrationsIncomplete, + orchestratorObjects: make(map[types.Object]string), + } + + // Only compatible direct statements on a stable registry are compared. + // The key mirrors the registry's normalization: lowercase, without trimming. + type duplicateKey struct { + registry types.Object + scope ast.Node + kind registrationKind + name string + version string + } + duplicates := make(map[duplicateKey]bool) + + seenRoots := make(map[ast.Node]bool) + for _, call := range index.registrationCandidates { + shape, receiver, ok := registrationCall(pass, call) + if !ok { + continue + } + if !isRegistryConstructor(pass, index.singleValue(receiver)) { + // Incoming, tuple-returned, or otherwise unproven registries may + // already contain registrations invisible to this package. + set.incomplete = true + } + + handler := call.Args[shape.handlerIndex] + if isNilExpr(pass, handler) { + pass.Reportf( + call.Lparen, + "task.TaskRegistry registration with a nil %s always returns an error", + shape.kind, + ) + continue + } + handlerNode := index.resolveFunction(handler, nil) + handlerObj := handlerObject(pass, handler) + + // rejected records that this call provably fails, so it contributes no + // name to the registry and cannot conflict with another registration. + version, versionKnown, versionRejected := registrationVersion(pass, call, shape) + name, nameKnown, nameRejected := registrationName(pass, call, shape, handlerNode) + entry := registration{ + name: name, + nameKnown: nameKnown, + version: version, + versionKnown: versionKnown, + } + rejected := versionRejected || nameRejected + + if entry.nameKnown { + // A rejected call registers nothing, but its name is still recorded + // so the unresolved-name check does not pile a second diagnostic + // onto a call site whose registration was already reported here. + set.names[shape.kind][strings.ToLower(entry.name)] = struct{}{} + // A registration whose version is computed at runtime may land on + // any key, so it neither proves nor disproves a conflict. + scope := index.registrationScopes[call] + if scope != nil && isRegistryConstructor(pass, index.singleValue(receiver)) && + !index.unstableRegistries[receiver] && entry.versionKnown && !rejected { + key := duplicateKey{ + registry: receiver, + scope: scope, + kind: shape.kind, + name: strings.ToLower(entry.name), + version: strings.ToLower(entry.version), + } + if duplicates[key] { + pass.Reportf( + call.Lparen, + "%s %q is registered more than once on the same task.TaskRegistry; "+ + "the duplicate registration returns an error", + shape.kind, + entry.name, + ) + } + duplicates[key] = true + } + } else { + set.dynamic[shape.kind] = true + } + + if handlerObj != nil && shape.kind == orchestratorKind { + set.orchestratorObjects[handlerObj] = entry.name + } + + if shape.kind == orchestratorKind && handlerNode != nil && !seenRoots[handlerNode] { + seenRoots[handlerNode] = true + set.roots = append(set.roots, handlerNode) + } + } + return set +} + +// registrationVersion returns the literal version, whether it is known, and +// whether the registry will reject it. Methods without a version argument have +// a known empty version. +func registrationVersion( + pass *analysis.Pass, + call *ast.CallExpr, + shape registrationShape, +) (version string, known, rejected bool) { + if shape.versionIndex < 0 { + return "", true, false + } + version, known = stringLiteral(pass, call.Args[shape.versionIndex]) + if !known || version == "" || strings.TrimSpace(version) != "" { + return version, known, false + } + pass.Reportf( + call.Args[shape.versionIndex].Pos(), + "task.TaskRegistry rejects an %s version that is only whitespace", + shape.kind, + ) + return version, true, true +} + +// registrationName returns a statically proven name and reports explicit names +// the registry rejects or reflection-derived names that are unstable. +func registrationName( + pass *analysis.Pass, + call *ast.CallExpr, + shape registrationShape, + handler ast.Node, +) (name string, known, rejected bool) { + if shape.nameIndex >= 0 { + name, known = stringLiteral(pass, call.Args[shape.nameIndex]) + if !known { + return "", false, false + } + switch { + case name == "": + pass.Reportf( + call.Args[shape.nameIndex].Pos(), + "task.TaskRegistry rejects an empty %s name", + shape.kind, + ) + return "", false, true + case strings.TrimSpace(name) == "": + // The registry accepts this name verbatim, so the task is + // registered under a name no caller can type readably. + pass.Reportf( + call.Args[shape.nameIndex].Pos(), + "task.TaskRegistry registers this %s under a name that is only whitespace, "+ + "which callers must reproduce exactly; use a non-blank name", + shape.kind, + ) + } + return name, true, false + } + + declaration, ok := handler.(*ast.FuncDecl) + if ok { + if declaration.Recv == nil { + return declaration.Name.Name, true, false + } + pass.Reportf( + call.Lparen, + "task.TaskRegistry derives the %s name from a method value, which produces a "+ + "name with a compiler-generated \"-fm\" suffix; register it with an explicit name instead", + shape.kind, + ) + return "", false, false + } + if handler != nil { + pass.Reportf( + call.Lparen, + "task.TaskRegistry derives the %s name from a function literal, "+ + "which produces a compiler-generated name; register it with an explicit name instead", + shape.kind, + ) + } + return "", false, false +} + +// registrationCall matches a call to a task.TaskRegistry Add* method and returns +// its shape plus the receiver variable when the receiver is a plain identifier. +func registrationCall(pass *analysis.Pass, call *ast.CallExpr) (registrationShape, types.Object, bool) { + selector, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return registrationShape{}, nil, false + } + selection := pass.TypesInfo.Selections[selector] + if selection == nil || !isTaskRegistry(selection.Recv()) { + return registrationShape{}, nil, false + } + method, ok := selection.Obj().(*types.Func) + if !ok || method.Pkg() == nil || method.Pkg().Path() != taskPackagePath { + return registrationShape{}, nil, false + } + shape, ok := registrationShapes[method.Name()] + if !ok || len(call.Args) != shape.arity { + return registrationShape{}, nil, false + } + var receiver types.Object + if identifier, ok := selector.X.(*ast.Ident); ok { + receiver = pass.TypesInfo.ObjectOf(identifier) + } + return shape, receiver, true +} + +func isTaskRegistry(value types.Type) bool { + value = types.Unalias(value) + if pointer, ok := value.(*types.Pointer); ok { + value = types.Unalias(pointer.Elem()) + } + named, ok := value.(*types.Named) + if !ok { + return false + } + object := named.Obj() + return object.Pkg() != nil && + object.Pkg().Path() == taskPackagePath && + object.Name() == "TaskRegistry" +} + +func isRegistryConstructor(pass *analysis.Pass, expression ast.Expr) bool { + call, ok := ast.Unparen(expression).(*ast.CallExpr) + if !ok { + return false + } + function := staticFunc(pass, call.Fun) + return function != nil && function.Pkg() != nil && + function.Pkg().Path() == taskPackagePath && function.Name() == "NewTaskRegistry" +} + +// handlerObject returns the declared function object a handler expression names. +func handlerObject(pass *analysis.Pass, expression ast.Expr) types.Object { + switch expression := expression.(type) { + case *ast.ParenExpr: + return handlerObject(pass, expression.X) + case *ast.Ident: + if function, ok := pass.TypesInfo.ObjectOf(expression).(*types.Func); ok { + return function + } + case *ast.SelectorExpr: + if selection := pass.TypesInfo.Selections[expression]; selection != nil { + if function, ok := selection.Obj().(*types.Func); ok { + return function + } + return nil + } + if function, ok := pass.TypesInfo.ObjectOf(expression.Sel).(*types.Func); ok { + return function + } + case *ast.CallExpr: + if typeInfo, ok := pass.TypesInfo.Types[expression.Fun]; ok && + typeInfo.IsType() && + len(expression.Args) == 1 { + return handlerObject(pass, expression.Args[0]) + } + } + return nil +} + +// stringLiteral returns the constant string value of an expression. +func stringLiteral(pass *analysis.Pass, expression ast.Expr) (string, bool) { + typeInfo, ok := pass.TypesInfo.Types[expression] + if !ok || typeInfo.Value == nil || typeInfo.Value.Kind() != constant.String { + return "", false + } + return constant.StringVal(typeInfo.Value), true +} + +func isNilExpr(pass *analysis.Pass, expression ast.Expr) bool { + identifier, ok := expression.(*ast.Ident) + if !ok || identifier.Name != "nil" { + return false + } + _, isNil := pass.TypesInfo.ObjectOf(identifier).(*types.Nil) + return isNil +} + +// taskName reports the durable task name an expression passed to CallActivity or +// CallSubOrchestrator resolves to, mirroring helpers.GetTaskFunctionName. +// +// Method values are excluded because reflection appends a compiler-generated +// "-fm" suffix to their name, which cannot be derived from the syntax. +func (index *packageIndex) taskName(expression ast.Expr) (string, bool) { + if name, ok := stringLiteral(index.pass, expression); ok { + return name, true + } + function, ok := handlerObject(index.pass, expression).(*types.Func) + if !ok { + return "", false + } + if signature, ok := function.Type().(*types.Signature); ok && signature.Recv() != nil { + return "", false + } + return function.Name(), true +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/rules.go b/cmd/orchestratorvet/analysis/orchestratorgo/rules.go new file mode 100644 index 00000000..592cbf40 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/rules.go @@ -0,0 +1,113 @@ +package orchestratorgo + +// wallClockFunctions maps time package entry points that read or wait on the +// host clock to the orchestration API that replaces them. +var wallClockFunctions = map[string]string{ + "Now": "CurrentTimeUtc", + "Since": "CurrentTimeUtc", + "Until": "CurrentTimeUtc", + "After": "CreateTimer", + "AfterFunc": "CreateTimer", + "Sleep": "CreateTimer", + "Tick": "CreateTimer", + "NewTicker": "CreateTimer", + "NewTimer": "CreateTimer", +} + +// randomConstructors are math/rand entry points that only build a generator. +// They are deterministic on their own; what matters is the seed they receive. +var randomConstructors = map[string]bool{ + "New": true, + "NewChaCha8": true, + "NewPCG": true, + "NewSource": true, + "NewZipf": true, +} + +// nondeterministicUUIDFunctions are google/uuid constructors that draw on +// randomness or the host clock. Name-based constructors such as NewSHA1 and +// NewMD5, and parsing helpers, are deterministic and are not listed. +var nondeterministicUUIDFunctions = map[string]bool{ + "New": true, + "NewRandom": true, + "NewRandomFromReader": true, + "NewString": true, + "NewUUID": true, +} + +// syncOnceFunctions are sync package functions that return memoized closures. +var syncOnceFunctions = map[string]bool{ + "OnceFunc": true, + "OnceValue": true, + "OnceValues": true, +} + +// externalOSFunctions are os package entry points that touch the filesystem, +// the process environment, or process lifetime. +var externalOSFunctions = map[string]bool{ + "Chdir": true, "Chmod": true, "Chown": true, "Chtimes": true, + "Clearenv": true, "Create": true, "CreateTemp": true, + "Environ": true, "Executable": true, "Exit": true, "ExpandEnv": true, + "FindProcess": true, "Getegid": true, "Getenv": true, "Geteuid": true, + "Getgid": true, "Getpid": true, "Getppid": true, "Getuid": true, + "Getwd": true, "Hostname": true, "Lchown": true, "Link": true, + "LookupEnv": true, "Lstat": true, "Mkdir": true, "MkdirAll": true, + "MkdirTemp": true, "NewFile": true, "Open": true, "OpenFile": true, + "OpenRoot": true, "Pipe": true, "ReadDir": true, "ReadFile": true, + "Readlink": true, "Remove": true, "RemoveAll": true, "Rename": true, + "Setenv": true, "StartProcess": true, "Stat": true, "Symlink": true, + "TempDir": true, "Truncate": true, "Unsetenv": true, + "UserCacheDir": true, "UserConfigDir": true, "UserHomeDir": true, + "WriteFile": true, +} + +// externalHTTPFunctions are net/http package entry points that open sockets. +var externalHTTPFunctions = map[string]bool{ + "Get": true, "Head": true, "ListenAndServe": true, "ListenAndServeTLS": true, + "Post": true, "PostForm": true, "Serve": true, "ServeFile": true, + "ServeTLS": true, +} + +var externalHTTPClientMethods = map[string]bool{ + "Do": true, "Get": true, "Head": true, "Post": true, "PostForm": true, +} + +var externalCmdMethods = map[string]bool{ + "CombinedOutput": true, "Output": true, "Run": true, "Start": true, "Wait": true, +} + +// replayUnsafeLogFunctions are log package entry points that emit output. +var replayUnsafeLogFunctions = map[string]bool{ + "Fatal": true, "Fatalf": true, "Fatalln": true, + "Output": true, "Panic": true, "Panicf": true, "Panicln": true, + "Print": true, "Printf": true, "Println": true, + "SetFlags": true, "SetOutput": true, "SetPrefix": true, +} + +// replayUnsafeSlogFunctions are package-level log/slog entry points that emit +// output. Methods on *slog.Logger are deliberately absent because +// (*task.OrchestrationContext).Logger returns a replay-safe *slog.Logger. +var replayUnsafeSlogFunctions = map[string]bool{ + "Debug": true, "DebugContext": true, + "Error": true, "ErrorContext": true, "Info": true, "InfoContext": true, + "Log": true, "LogAttrs": true, + "Warn": true, "WarnContext": true, +} + +// processLoggerFunctions are log/slog entry points that read or replace the +// process-wide logger rather than emitting a record themselves. They are still +// reported, because the logger they hand back or install writes on every replay, +// but they warrant their own wording. +var processLoggerFunctions = map[string]bool{ + "Default": true, "SetDefault": true, +} + +// pureLoopPackages are standard library packages whose functions cannot block, +// perform I/O, or advance orchestration state. Calls into them do not prevent +// the unbounded-loop check from proving that a loop makes no durable progress. +var pureLoopPackages = map[string]bool{ + "bytes": true, "cmp": true, "encoding/json": true, "errors": true, + "fmt": true, "maps": true, "math": true, "math/bits": true, + "slices": true, "sort": true, "strconv": true, "strings": true, + "unicode": true, "unicode/utf8": true, +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/callgraph/callgraph.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/callgraph/callgraph.go new file mode 100644 index 00000000..a1ac513b --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/callgraph/callgraph.go @@ -0,0 +1,126 @@ +// Package callgraph exercises whole-package call-graph following: named helpers, +// methods, function variables, nested literals, recursion, and unreachable code. +package callgraph + +import ( + "os" + "time" + + "github.com/microsoft/durabletask-go/task" +) + +// nestedHelpers reaches a hazard two named calls away. +func nestedHelpers(ctx *task.OrchestrationContext) (any, error) { + return firstLevel(), nil +} + +func firstLevel() time.Time { + return secondLevel() +} + +func secondLevel() time.Time { + return time.Now() // want `time\.Now is not deterministic in an orchestrator` +} + +// methodReceiver reaches a hazard through a method on a package type. +type loader struct{ path string } + +func (l loader) read() string { + return os.Getenv(l.path) // want `os\.Getenv performs external I/O` +} + +func (l *loader) refresh() { + l.path = os.TempDir() // want `os\.TempDir performs external I/O` +} + +func methodReceiver(ctx *task.OrchestrationContext) (any, error) { + instance := &loader{path: "REGION"} + instance.refresh() + return instance.read(), nil +} + +// packageFunctionVariable is a package-level variable holding a helper. +var packageFunctionVariable = func() string { + return os.Getenv("TIER") // want `os\.Getenv performs external I/O` +} + +func functionVariable(ctx *task.OrchestrationContext) (any, error) { + local := packageFunctionVariable + return local(), nil +} + +// nestedLiteral hides a hazard inside a closure that is only ever invoked +// through the orchestration scheduler. +func nestedLiteral(ctx *task.OrchestrationContext) (any, error) { + ctx.Go(func(child *task.OrchestrationContext) { + inner := func() { + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + } + inner() + }) + return nil, nil +} + +// mutualRecursion proves the walk terminates and still reports once. +func mutualRecursion(ctx *task.OrchestrationContext) (any, error) { + return recurseA(4), nil +} + +func recurseA(depth int) time.Time { + if depth <= 0 { + return time.Now() // want `time\.Now is not deterministic in an orchestrator` + } + return recurseB(depth - 1) +} + +func recurseB(depth int) time.Time { + return recurseA(depth - 1) +} + +// sharedHelper is reached from two orchestrators but reported only once. +func sharedHelper() string { + return os.Getenv("SHARED") // want `os\.Getenv performs external I/O` +} + +func firstSharer(ctx *task.OrchestrationContext) (any, error) { + return sharedHelper(), nil +} + +func secondSharer(ctx *task.OrchestrationContext) (any, error) { + return sharedHelper(), nil +} + +// unreachableHelper is never reached from a registered orchestrator. +func unreachableHelper() string { + go func() {}() + return os.Getenv("UNREACHABLE") +} + +// unregisteredOrchestrator has an orchestrator signature but is never registered. +func unregisteredOrchestrator(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil +} + +// reassignedHelper is assigned twice, so the analyzer cannot prove which body +// runs and follows neither. +var reassignedHelper = func() string { return os.Getenv("FIRST") } + +func reassign() { + reassignedHelper = func() string { return os.Getenv("SECOND") } +} + +func ambiguousVariable(ctx *task.OrchestrationContext) (any, error) { + return reassignedHelper(), nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(nestedHelpers) + _ = registry.AddOrchestrator(methodReceiver) + _ = registry.AddOrchestrator(functionVariable) + _ = registry.AddOrchestrator(nestedLiteral) + _ = registry.AddOrchestrator(mutualRecursion) + _ = registry.AddOrchestrator(firstSharer) + _ = registry.AddOrchestrator(secondSharer) + _ = registry.AddOrchestrator(ambiguousVariable) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/concurrency/concurrency.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/concurrency/concurrency.go new file mode 100644 index 00000000..5b931ac6 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/concurrency/concurrency.go @@ -0,0 +1,122 @@ +// Package concurrency exercises the raw goroutine, synchronization, and channel +// checks, plus the durable primitives that are allowed to replace them. +package concurrency + +import ( + "sync" + "sync/atomic" + + "github.com/microsoft/durabletask-go/task" +) + +func rawGoroutines(ctx *task.OrchestrationContext) (any, error) { + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + go helper() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func helper() {} + +func syncPrimitives(ctx *task.OrchestrationContext) (any, error) { + var mutex sync.Mutex + mutex.Lock() // want `\(sync\.Mutex\)\.Lock is not replay safe in an orchestrator` + mutex.Unlock() // want `\(sync\.Mutex\)\.Unlock is not replay safe in an orchestrator` + + var group sync.WaitGroup + group.Add(1) // want `\(sync\.WaitGroup\)\.Add is not replay safe in an orchestrator` + group.Done() // want `\(sync\.WaitGroup\)\.Done is not replay safe in an orchestrator` + group.Wait() // want `\(sync\.WaitGroup\)\.Wait is not replay safe in an orchestrator` + + var once sync.Once + once.Do(func() {}) // want `\(sync\.Once\)\.Do is not replay safe in an orchestrator` + + memo := sync.OnceValue(func() int { return 1 }) // want `sync\.OnceValue is not replay safe in an orchestrator` + _ = memo() + + var counter atomic.Int64 + counter.Add(1) // want `\(atomic\.Int64\)\.Add is not replay safe in an orchestrator` + var plain int64 + atomic.AddInt64(&plain, 1) // want `sync/atomic\.AddInt64 is not replay safe in an orchestrator` + return nil, nil +} + +// embeddedMutex proves the check keys on the declaring type, not the syntax. +type embeddedMutex struct { + sync.Mutex + value int +} + +func embeddedSync(ctx *task.OrchestrationContext) (any, error) { + state := &embeddedMutex{} + state.Lock() // want `\(sync\.Mutex\)\.Lock is not replay safe in an orchestrator` + state.value++ + state.Unlock() // want `\(sync\.Mutex\)\.Unlock is not replay safe in an orchestrator` + return state.value, nil +} + +func channelOperations(ctx *task.OrchestrationContext) (any, error) { + results := make(chan int, 1) // want `creating a channel is not deterministic in an orchestrator` + results <- 1 // want `channel send is not deterministic in an orchestrator` + value := <-results // want `channel receive is not deterministic in an orchestrator` + close(results) // want `closing a channel is not deterministic in an orchestrator` + + stream := make(chan int) // want `creating a channel is not deterministic in an orchestrator` + for item := range stream { // want `ranging over a channel is not deterministic in an orchestrator` + value += item + } + select { // want `select statement is not deterministic in an orchestrator` + case received := <-stream: // want `channel receive is not deterministic in an orchestrator` + value += received + default: + } + return value, nil +} + +// durableConcurrency uses only the orchestration scheduler primitives. +func durableConcurrency(ctx *task.OrchestrationContext) (any, error) { + group := ctx.NewWaitGroup() + group.Add(1) + ctx.Go(func(child *task.OrchestrationContext) { + defer group.Done() + _ = child.CallActivity("work") + }) + group.Wait(ctx) + + events := task.NewEventChannel[int](ctx, "tick") + value, err := events.ReceiveErr(ctx) + if err != nil { + return nil, err + } + pending := ctx.CallActivity("work") + ctx.Select( + task.OnTask(pending, func(task.Task) {}), + task.OnEvent(events, func(int) {}), + ) + return value, nil +} + +// goInsideCoroutine proves a raw go statement is still reported when it is +// written directly inside an orchestration coroutine. +func goInsideCoroutine(ctx *task.OrchestrationContext) (any, error) { + ctx.Go(func(*task.OrchestrationContext) { + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + }) + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(rawGoroutines) + _ = registry.AddOrchestrator(syncPrimitives) + _ = registry.AddOrchestrator(embeddedSync) + _ = registry.AddOrchestrator(channelOperations) + _ = registry.AddOrchestrator(durableConcurrency) + _ = registry.AddOrchestrator(goInsideCoroutine) + + // An orchestrator registered as an inline literal under an explicit name is + // a root just like a named declaration, so its body is analyzed. + _ = registry.AddOrchestratorN("inline", func(*task.OrchestrationContext) (any, error) { + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil + }) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/crosspackage/crosspackage.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/crosspackage/crosspackage.go new file mode 100644 index 00000000..070192a0 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/crosspackage/crosspackage.go @@ -0,0 +1,17 @@ +// Package crosspackage proves the unresolved-name check stays silent when a +// package registers orchestrators but no activities, because the activities are +// registered somewhere the analyzer cannot see. +package crosspackage + +import "github.com/microsoft/durabletask-go/task" + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("registeredElsewhere") + _ = ctx.CallActivity("alsoElsewhere") + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/externalio/externalio.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/externalio/externalio.go new file mode 100644 index 00000000..0893d1a2 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/externalio/externalio.go @@ -0,0 +1,108 @@ +// Package externalio exercises the filesystem, network, process, and +// environment checks. +package externalio + +import ( + "net" + "net/http" + "os" + "os/exec" + "syscall" + + "github.com/microsoft/durabletask-go/task" +) + +func filesystem(ctx *task.OrchestrationContext) (any, error) { + data, err := os.ReadFile("config.json") // want `os\.ReadFile performs external I/O` + if err != nil { + return nil, err + } + if err := os.WriteFile("out.json", data, 0o600); err != nil { // want `os\.WriteFile performs external I/O` + return nil, err + } + handle, err := os.Open("config.json") // want `os\.Open performs external I/O` + if err != nil { + return nil, err + } + defer handle.Close() + if err := os.MkdirAll("cache", 0o700); err != nil { // want `os\.MkdirAll performs external I/O` + return nil, err + } + return os.Remove("out.json"), nil // want `os\.Remove performs external I/O` +} + +func environment(ctx *task.OrchestrationContext) (any, error) { + region := os.Getenv("REGION") // want `os\.Getenv performs external I/O` + if _, ok := os.LookupEnv("TIER"); ok { // want `os\.LookupEnv performs external I/O` + _ = os.Setenv("TIER", "gold") // want `os\.Setenv performs external I/O` + } + _ = os.Environ() // want `os\.Environ performs external I/O` + return region, nil +} + +func network(ctx *task.OrchestrationContext) (any, error) { + response, err := http.Get("https://example.invalid") // want `http\.Get performs external I/O` + if err != nil { + return nil, err + } + defer response.Body.Close() + + client := &http.Client{} + request, err := http.NewRequest(http.MethodGet, "https://example.invalid", nil) + if err != nil { + return nil, err + } + direct, err := client.Do(request) // want `\(\*http\.Client\)\.Do performs external I/O` + if err != nil { + return nil, err + } + defer direct.Body.Close() + + connection, err := net.Dial("tcp", "example.invalid:80") // want `net\.Dial performs external I/O` + if err != nil { + return nil, err + } + defer connection.Close() + + addresses, err := net.LookupHost("example.invalid") // want `net\.LookupHost performs external I/O` + if err != nil { + return nil, err + } + listener, err := net.Listen("tcp", "127.0.0.1:0") // want `net\.Listen performs external I/O` + if err != nil { + return nil, err + } + defer listener.Close() + return addresses, nil +} + +func process(ctx *task.OrchestrationContext) (any, error) { + command := exec.Command("echo", "hello") // want `exec\.Command performs external I/O` + output, err := command.Output() // want `\(\*exec\.Cmd\)\.Output performs external I/O` + if err != nil { + return nil, err + } + _ = syscall.Getpid() // want `syscall\.Getpid performs external I/O` + return string(output), nil +} + +// durableIO routes every side effect through an activity. +func durableIO(ctx *task.OrchestrationContext) (any, error) { + var result string + if err := ctx.CallActivity("readConfig").Await(&result); err != nil { + return nil, err + } + if err := ctx.CallActivity("callService", task.WithActivityInput(result)).Await(nil); err != nil { + return nil, err + } + return result, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(filesystem) + _ = registry.AddOrchestrator(environment) + _ = registry.AddOrchestrator(network) + _ = registry.AddOrchestrator(process) + _ = registry.AddOrchestrator(durableIO) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go new file mode 100644 index 00000000..28bc4850 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go @@ -0,0 +1,47 @@ +package fixbindings + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func safe(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func shadowedContext(ctx *task.OrchestrationContext) (any, error) { + { + ctx := 1 + go func() { _ = ctx }() // want `raw go statement is not deterministic in an orchestrator` + } + return nil, nil +} + +func shadowedImport(ctx *task.OrchestrationContext) (any, error) { + task := 1 + go func() { _ = task }() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func sameTypeShadow(ctx *task.OrchestrationContext) (any, error) { + { + ctx := (*task.OrchestrationContext)(nil) + _ = ctx + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` + } +} + +func reassignedContext(ctx *task.OrchestrationContext) (any, error) { + ctx = nil + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(safe) + _ = registry.AddOrchestrator(shadowedContext) + _ = registry.AddOrchestrator(shadowedImport) + _ = registry.AddOrchestrator(sameTypeShadow) + _ = registry.AddOrchestrator(reassignedContext) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go.golden new file mode 100644 index 00000000..0df38cb7 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings.go.golden @@ -0,0 +1,47 @@ +package fixbindings + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func safe(ctx *task.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func shadowedContext(ctx *task.OrchestrationContext) (any, error) { + { + ctx := 1 + go func() { _ = ctx }() // want `raw go statement is not deterministic in an orchestrator` + } + return nil, nil +} + +func shadowedImport(ctx *task.OrchestrationContext) (any, error) { + task := 1 + go func() { _ = task }() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func sameTypeShadow(ctx *task.OrchestrationContext) (any, error) { + { + ctx := (*task.OrchestrationContext)(nil) + _ = ctx + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` + } +} + +func reassignedContext(ctx *task.OrchestrationContext) (any, error) { + ctx = nil + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(safe) + _ = registry.AddOrchestrator(shadowedContext) + _ = registry.AddOrchestrator(shadowedImport) + _ = registry.AddOrchestrator(sameTypeShadow) + _ = registry.AddOrchestrator(reassignedContext) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings_test.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings_test.go new file mode 100644 index 00000000..afde1694 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixbindings/fixbindings_test.go @@ -0,0 +1,25 @@ +package fixbindings + +import ( + "testing" + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func TestContextBindingsRemainSafe(t *testing.T) { + for name, orchestrator := range map[string]task.Orchestrator{ + "same type shadow": sameTypeShadow, + "reassigned": reassignedContext, + } { + t.Run(name, func(t *testing.T) { + value, err := orchestrator(&task.OrchestrationContext{}) + if err != nil { + t.Fatal(err) + } + if clock, ok := value.(time.Time); !ok || clock.IsZero() { + t.Fatalf("unfixable clock read changed: %v", value) + } + }) + } +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go new file mode 100644 index 00000000..1b2b2643 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go @@ -0,0 +1,60 @@ +// Package fixes exercises the suggested fixes the analyzer offers. Fixes are +// only produced where the rewrite is unambiguous. +package fixes + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clockRead(ctx *task.OrchestrationContext) (any, error) { + now := time.Now() // want `time\.Now is not deterministic in an orchestrator` + return now.Add(time.Hour), nil +} + +func coroutine(ctx *task.OrchestrationContext) (any, error) { + go func() { // want `raw go statement is not deterministic in an orchestrator` + _ = ctx.CallActivity("work") + }() + return nil, nil +} + +// commentedCoroutine proves the rewrite keeps everything written inside the +// literal, including comments, which are lost when a body is reprinted. +func commentedCoroutine(ctx *task.OrchestrationContext) (any, error) { + go func() { // want `raw go statement is not deterministic in an orchestrator` + // Kick off the background work. + _ = ctx.CallActivity("work") // fire and forget + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + + /* A block comment survives too. */ + if ctx.IsReplaying { + // Nothing to do while replaying. + return + } + }() + return nil, nil +} + +// noContextName has a blank context parameter, so no rewrite target exists. +func noContextName(_ *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +// namedGoroutine calls a named function, which has no one-to-one coroutine form. +func namedGoroutine(ctx *task.OrchestrationContext) (any, error) { + go background() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func background() {} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clockRead) + _ = registry.AddOrchestrator(coroutine) + _ = registry.AddOrchestrator(commentedCoroutine) + _ = registry.AddOrchestrator(noContextName) + _ = registry.AddOrchestrator(namedGoroutine) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go.golden new file mode 100644 index 00000000..4fc43b2d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixes/fixes.go.golden @@ -0,0 +1,60 @@ +// Package fixes exercises the suggested fixes the analyzer offers. Fixes are +// only produced where the rewrite is unambiguous. +package fixes + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clockRead(ctx *task.OrchestrationContext) (any, error) { + now := ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + return now.Add(time.Hour), nil +} + +func coroutine(ctx *task.OrchestrationContext) (any, error) { + ctx.Go(func(*task.OrchestrationContext) { // want `raw go statement is not deterministic in an orchestrator` + _ = ctx.CallActivity("work") + }) + return nil, nil +} + +// commentedCoroutine proves the rewrite keeps everything written inside the +// literal, including comments, which are lost when a body is reprinted. +func commentedCoroutine(ctx *task.OrchestrationContext) (any, error) { + ctx.Go(func(*task.OrchestrationContext) { // want `raw go statement is not deterministic in an orchestrator` + // Kick off the background work. + _ = ctx.CallActivity("work") // fire and forget + _ = ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + + /* A block comment survives too. */ + if ctx.IsReplaying { + // Nothing to do while replaying. + return + } + }) + return nil, nil +} + +// noContextName has a blank context parameter, so no rewrite target exists. +func noContextName(_ *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +// namedGoroutine calls a named function, which has no one-to-one coroutine form. +func namedGoroutine(ctx *task.OrchestrationContext) (any, error) { + go background() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func background() {} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clockRead) + _ = registry.AddOrchestrator(coroutine) + _ = registry.AddOrchestrator(commentedCoroutine) + _ = registry.AddOrchestrator(noContextName) + _ = registry.AddOrchestrator(namedGoroutine) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go new file mode 100644 index 00000000..a59bd4a9 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go @@ -0,0 +1,18 @@ +package fixesbatch + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clocks(ctx *task.OrchestrationContext) (any, error) { + // Both reads must be replaced together with their otherwise unused import. + first := time.Now() // want `time\.Now is not deterministic in an orchestrator` + return first.Equal(time.Now()), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go.golden new file mode 100644 index 00000000..668c6e20 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatch/fixesbatch.go.golden @@ -0,0 +1,16 @@ +package fixesbatch + +import ( + "github.com/microsoft/durabletask-go/task" +) + +func clocks(ctx *task.OrchestrationContext) (any, error) { + // Both reads must be replaced together with their otherwise unused import. + first := ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + return first.Equal(ctx.CurrentTimeUtc), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go new file mode 100644 index 00000000..9f47df4a --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go @@ -0,0 +1,22 @@ +package fixesbatchalias + +import ( + clock "time" + + durabletask "github.com/microsoft/durabletask-go/task" +) + +func first(ctx *durabletask.OrchestrationContext) (any, error) { + _ = clock.Now() // want `time\.Now is not deterministic in an orchestrator` + return helper(ctx), nil +} + +func helper(other *durabletask.OrchestrationContext) any { + // A reached helper uses its own context, not the caller's parameter name. + return clock.Now() // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := durabletask.NewTaskRegistry() + _ = registry.AddOrchestrator(first) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go.golden new file mode 100644 index 00000000..3d3df813 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchalias/fixesbatchalias.go.golden @@ -0,0 +1,20 @@ +package fixesbatchalias + +import ( + durabletask "github.com/microsoft/durabletask-go/task" +) + +func first(ctx *durabletask.OrchestrationContext) (any, error) { + _ = ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + return helper(ctx), nil +} + +func helper(other *durabletask.OrchestrationContext) any { + // A reached helper uses its own context, not the caller's parameter name. + return other.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := durabletask.NewTaskRegistry() + _ = registry.AddOrchestrator(first) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go new file mode 100644 index 00000000..e755be6d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go @@ -0,0 +1,18 @@ +package fixesbatchstandalone + +import "time" // Preserve this comment when removing the import. +import "github.com/microsoft/durabletask-go/task" + +func first(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func second(other *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(first) + _ = registry.AddOrchestrator(second) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go.golden new file mode 100644 index 00000000..79338e41 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesbatchstandalone/fixesbatchstandalone.go.golden @@ -0,0 +1,18 @@ +package fixesbatchstandalone + +// Preserve this comment when removing the import. +import "github.com/microsoft/durabletask-go/task" + +func first(ctx *task.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func second(other *task.OrchestrationContext) (any, error) { + return other.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(first) + _ = registry.AddOrchestrator(second) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go new file mode 100644 index 00000000..c65ed847 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go @@ -0,0 +1,20 @@ +// Package fixesimport proves the time.Now rewrite also removes the import it +// leaves behind, so the fixed file still compiles. +package fixesimport + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +// soleUse is the only reference to the time package in this file, so rewriting +// it must delete the import as well. +func soleUse(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(soleUse) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go.golden new file mode 100644 index 00000000..b1f2c00b --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesimport/fixesimport.go.golden @@ -0,0 +1,18 @@ +// Package fixesimport proves the time.Now rewrite also removes the import it +// leaves behind, so the fixed file still compiles. +package fixesimport + +import ( + "github.com/microsoft/durabletask-go/task" +) + +// soleUse is the only reference to the time package in this file, so rewriting +// it must delete the import as well. +func soleUse(ctx *task.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(soleUse) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go new file mode 100644 index 00000000..82c63a84 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go @@ -0,0 +1,15 @@ +// Inline imports intentionally exercise edit boundaries before formatting. +package fixesinline + +import ("time" /* Keep this inline comment. */; "github.com/microsoft/durabletask-go/task") +import (clock "time") // Keep the grouped import comment. + +func clocks(ctx *task.OrchestrationContext) (any, error) { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + return clock.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go.golden new file mode 100644 index 00000000..c781fe41 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesinline/fixesinline.go.golden @@ -0,0 +1,18 @@ +// Inline imports intentionally exercise edit boundaries before formatting. +package fixesinline + +import ( /* Keep this inline comment. */ + "github.com/microsoft/durabletask-go/task" +) + +// Keep the grouped import comment. + +func clocks(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go new file mode 100644 index 00000000..d8d5659e --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go @@ -0,0 +1,34 @@ +package fixespartial + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clocks(ctx *task.OrchestrationContext) (any, error) { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + time.Now() // want `time\.Now is not deterministic in an orchestrator` + defer time.Now() // want `time\.Now is not deterministic in an orchestrator` + go time.Now() // want `raw go statement is not deterministic in an orchestrator` `time\.Now is not deterministic in an orchestrator` + { + ctx := 0 + _ = ctx + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + } + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func noContext(_ *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func unreachable() any { + return time.Now() +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) + _ = registry.AddOrchestrator(noContext) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go.golden new file mode 100644 index 00000000..2901bf52 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixespartial/fixespartial.go.golden @@ -0,0 +1,34 @@ +package fixespartial + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clocks(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CurrentTimeUtc // want `time\.Now is not deterministic in an orchestrator` + time.Now() // want `time\.Now is not deterministic in an orchestrator` + defer time.Now() // want `time\.Now is not deterministic in an orchestrator` + go time.Now() // want `raw go statement is not deterministic in an orchestrator` `time\.Now is not deterministic in an orchestrator` + { + ctx := 0 + _ = ctx + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + } + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func noContext(_ *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func unreachable() any { + return time.Now() +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clocks) + _ = registry.AddOrchestrator(noContext) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go new file mode 100644 index 00000000..402959d3 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go @@ -0,0 +1,13 @@ +package fixesstandalone + +import "time" +import "github.com/microsoft/durabletask-go/task" + +func currentTime(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(currentTime) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go.golden new file mode 100644 index 00000000..a1ba571e --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalone/fixesstandalone.go.golden @@ -0,0 +1,12 @@ +package fixesstandalone + +import "github.com/microsoft/durabletask-go/task" + +func currentTime(ctx *task.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(currentTime) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go new file mode 100644 index 00000000..695af04e --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go @@ -0,0 +1,13 @@ +package fixesstandalonealias + +import clock "time" +import durabletask "github.com/microsoft/durabletask-go/task" + +func currentTime(ctx *durabletask.OrchestrationContext) (any, error) { + return clock.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := durabletask.NewTaskRegistry() + _ = registry.AddOrchestrator(currentTime) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go.golden new file mode 100644 index 00000000..58b31325 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesstandalonealias/fixesstandalonealias.go.golden @@ -0,0 +1,12 @@ +package fixesstandalonealias + +import durabletask "github.com/microsoft/durabletask-go/task" + +func currentTime(ctx *durabletask.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +func register() { + registry := durabletask.NewTaskRegistry() + _ = registry.AddOrchestrator(currentTime) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/example/registrationhelpers/registrationhelpers.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/example/registrationhelpers/registrationhelpers.go new file mode 100644 index 00000000..2d45387f --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/example/registrationhelpers/registrationhelpers.go @@ -0,0 +1,19 @@ +package registrationhelpers + +import "github.com/microsoft/durabletask-go/task" + +// The helper's name does not limit which namespaces it can register. +func RegisterActivities(registry *task.TaskRegistry) { + _ = registry.AddActivityN("externalWork", work) + _ = registry.AddOrchestratorN("externalChild", child) +} + +func NewRegistry() (*task.TaskRegistry, error) { + registry := task.NewTaskRegistry() + RegisterActivities(registry) + return registry, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func child(*task.OrchestrationContext) (any, error) { return nil, nil } diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/google/uuid/uuid.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/google/uuid/uuid.go new file mode 100644 index 00000000..284158f5 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/google/uuid/uuid.go @@ -0,0 +1,31 @@ +// Package uuid is a minimal stand-in for github.com/google/uuid. It exposes just +// enough surface for the orchestratorgo analysis test fixtures to type-check. +package uuid + +import "io" + +type UUID [16]byte + +func (u UUID) String() string { return "" } + +func New() UUID { return UUID{} } + +func NewString() string { return "" } + +func NewRandom() (UUID, error) { return UUID{}, nil } + +func NewRandomFromReader(r io.Reader) (UUID, error) { return UUID{}, nil } + +func NewUUID() (UUID, error) { return UUID{}, nil } + +func NewSHA1(space UUID, data []byte) UUID { return UUID{} } + +func NewMD5(space UUID, data []byte) UUID { return UUID{} } + +func Parse(s string) (UUID, error) { return UUID{}, nil } + +func MustParse(s string) UUID { return UUID{} } + +func Must(u UUID, err error) UUID { return u } + +var NameSpaceDNS = UUID{} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/microsoft/durabletask-go/task/task.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/microsoft/durabletask-go/task/task.go new file mode 100644 index 00000000..26bb99b2 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/github.com/microsoft/durabletask-go/task/task.go @@ -0,0 +1,140 @@ +// Package task is a minimal stand-in for the durable task API. It exposes just +// enough surface for the orchestratorgo analysis test fixtures to type-check. +package task + +import ( + "log/slog" + "time" +) + +type InstanceID string + +type EntityID struct { + Name string + Key string +} + +type OrchestrationContext struct { + ID InstanceID + Name string + Version string + IsReplaying bool + CurrentTimeUtc time.Time +} + +type Task interface { + Await(v any) error +} + +type WaitGroup interface { + Add(delta int) + Done() + Wait(ctx *OrchestrationContext) +} + +type SelectCase interface{ isSelectCase() } + +type EventChannel[T any] struct{} + +func NewEventChannel[T any](ctx *OrchestrationContext, name string) *EventChannel[T] { + return &EventChannel[T]{} +} + +func (*EventChannel[T]) Receive(ctx *OrchestrationContext) T { + var zero T + return zero +} + +func (*EventChannel[T]) ReceiveErr(ctx *OrchestrationContext) (T, error) { + var zero T + return zero, nil +} + +func OnTask(t Task, handler func(Task)) SelectCase { return nil } + +func OnEvent[T any](channel *EventChannel[T], handler func(T)) SelectCase { return nil } + +func (*OrchestrationContext) Select(cases ...SelectCase) {} + +func (*OrchestrationContext) WhenAny(tasks ...Task) Task { return nil } + +func (*OrchestrationContext) WhenAll(tasks ...Task) error { return nil } + +type CallActivityOption func() +type SubOrchestratorOption func() +type ContinueAsNewOption func() + +func WithActivityInput(any) CallActivityOption { return nil } +func WithSubOrchestrationInput(any) SubOrchestratorOption { return nil } + +func (*OrchestrationContext) Go(func(*OrchestrationContext)) {} + +func (*OrchestrationContext) NewWaitGroup() WaitGroup { return nil } + +func (*OrchestrationContext) NewGuid() string { return "" } + +func (*OrchestrationContext) Logger() *slog.Logger { return slog.Default() } + +func (*OrchestrationContext) GetInput(v any) error { return nil } + +func (*OrchestrationContext) CallActivity(activity any, opts ...CallActivityOption) Task { return nil } + +func (*OrchestrationContext) CallSubOrchestrator(o any, opts ...SubOrchestratorOption) Task { + return nil +} + +func (*OrchestrationContext) CallEntity(id EntityID, operation string) Task { return nil } + +func (*OrchestrationContext) SignalEntity(id EntityID, operation string, payload any) error { + return nil +} + +func (*OrchestrationContext) LockEntities(ids ...EntityID) (func(), error) { return nil, nil } + +func (*OrchestrationContext) CreateTimer(delay time.Duration) Task { return nil } + +func (*OrchestrationContext) WaitForSingleEvent(name string, timeout time.Duration) Task { + return nil +} + +func (*OrchestrationContext) ContinueAsNew(newInput any, options ...ContinueAsNewOption) {} + +func (*OrchestrationContext) SetCustomStatus(status string) {} + +type Orchestrator func(*OrchestrationContext) (any, error) + +type ActivityContext interface { + GetInput(v any) error +} + +type Activity func(ActivityContext) (any, error) + +type EntityContext interface{ Key() string } + +type Entity func(EntityContext) error + +type TaskRegistry struct{} + +func NewTaskRegistry() *TaskRegistry { + return &TaskRegistry{} +} + +func (*TaskRegistry) AddOrchestrator(Orchestrator) error { return nil } + +func (*TaskRegistry) AddOrchestratorN(string, Orchestrator) error { return nil } + +func (*TaskRegistry) AddOrchestratorVersion(string, Orchestrator) error { return nil } + +func (*TaskRegistry) AddOrchestratorNVersion(string, string, Orchestrator) error { return nil } + +func (*TaskRegistry) AddActivity(Activity) error { return nil } + +func (*TaskRegistry) AddActivityN(string, Activity) error { return nil } + +func (*TaskRegistry) AddActivityVersion(string, Activity) error { return nil } + +func (*TaskRegistry) AddActivityNVersion(string, string, Activity) error { return nil } + +func (*TaskRegistry) AddEntity(Entity) error { return nil } + +func (*TaskRegistry) AddEntityN(string, Entity) error { return nil } diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/logging/logging.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/logging/logging.go new file mode 100644 index 00000000..c899897e --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/logging/logging.go @@ -0,0 +1,58 @@ +// Package logging exercises the replay-unsafe logging checks and the +// replay-safe orchestration logger that replaces them. +package logging + +import ( + "fmt" + "log" + "log/slog" + "os" + + "github.com/microsoft/durabletask-go/task" +) + +func standardLog(ctx *task.OrchestrationContext) (any, error) { + log.Print("starting") // want `log\.Print writes on every replay` + log.Printf("step %d", 1) // want `log\.Printf writes on every replay` + log.Println("done") // want `log\.Println writes on every replay` + custom := log.New(os.Stderr, "orchestration ", 0) + custom.Printf("step %d", 2) // want `\(\*log\.Logger\)\.Printf writes on every replay` + return nil, nil +} + +func globalSlog(ctx *task.OrchestrationContext) (any, error) { + slog.Info("starting") // want `slog\.Info writes on every replay` + slog.Warn("slow") // want `slog\.Warn writes on every replay` + slog.Error("failed", "attempt", 1) // want `slog\.Error writes on every replay` + slog.Default().Info("also global") // want `slog\.Default reaches the process-wide logger, which writes on every replay` + replacement := slog.New(slog.NewTextHandler(os.Stderr, nil)) + slog.SetDefault(replacement) // want `slog\.SetDefault reaches the process-wide logger, which writes on every replay` + return nil, nil +} + +func standardStreams(ctx *task.OrchestrationContext) (any, error) { + fmt.Println("starting") // want `fmt\.Println writes on every replay` + fmt.Printf("step %d\n", 1) // want `fmt\.Printf writes on every replay` + fmt.Fprintf(os.Stdout, "step %d\n", 2) // want `fmt\.Fprintf writes on every replay` + fmt.Fprintln(os.Stderr, "warning") // want `fmt\.Fprintln writes on every replay` + return nil, nil +} + +// replaySafeLogging uses the orchestration logger and pure formatting. +func replaySafeLogging(ctx *task.OrchestrationContext) (any, error) { + logger := ctx.Logger() + logger.Info("starting", "instance", ctx.ID) + logger.With("stage", "two").Debug("progress") + ctx.Logger().Error("failed", "attempt", 1) + message := fmt.Sprintf("instance %s", ctx.ID) + ctx.SetCustomStatus(message) + return message, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(standardLog) + _ = registry.AddOrchestrator(globalSlog) + _ = registry.AddOrchestrator(standardStreams) + _ = registry.AddOrchestrator(replaySafeLogging) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/loops/loops.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/loops/loops.go new file mode 100644 index 00000000..bcc85d72 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/loops/loops.go @@ -0,0 +1,134 @@ +// Package loops exercises the unbounded orchestration loop check. A loop is only +// reported when the whole-package loop and call graph proves it can neither +// leave the loop nor make durable progress. +package loops + +import ( + "path/filepath" + "strings" + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func spinForever(ctx *task.OrchestrationContext) (any, error) { + counter := 0 + for { // want `unbounded orchestrator loop never awaits a durable task` + counter++ + } +} + +func spinOnConstantCondition(ctx *task.OrchestrationContext) (any, error) { + counter := 0 + for true { // want `unbounded orchestrator loop never awaits a durable task` + counter += 2 + } + return counter, nil +} + +func spinThroughHelpers(ctx *task.OrchestrationContext) (any, error) { + for { // want `unbounded orchestrator loop never awaits a durable task` + pureStep() + } +} + +func pureStep() { + _ = strings.Repeat("a", 2) + pureNested() +} + +func pureNested() { + _ = strings.ToUpper("b") +} + +// spinThroughRecursion proves recursion terminates the walk without a stack +// overflow and still yields a report. +func spinThroughRecursion(ctx *task.OrchestrationContext) (any, error) { + for { // want `unbounded orchestrator loop never awaits a durable task` + recursiveA(3) + } +} + +func recursiveA(depth int) { + if depth > 0 { + recursiveB(depth - 1) + } +} + +func recursiveB(depth int) { + if depth > 0 { + recursiveA(depth - 1) + } +} + +// eternalTimerLoop makes durable progress on every iteration. +func eternalTimerLoop(ctx *task.OrchestrationContext) (any, error) { + for { + if err := ctx.CreateTimer(time.Hour).Await(nil); err != nil { + return nil, err + } + } +} + +// eternalActivityLoop awaits an activity through a same-package helper. +func eternalActivityLoop(ctx *task.OrchestrationContext) (any, error) { + for { + durableStep(ctx) + } +} + +func durableStep(ctx *task.OrchestrationContext) { + _ = ctx.CallActivity("work").Await(nil) +} + +// continueAsNewLoop restarts the orchestration instead of spinning. +func continueAsNewLoop(ctx *task.OrchestrationContext) (any, error) { + for { + ctx.ContinueAsNew(nil) + return nil, nil + } +} + +// boundedLoops all have an exit path or a real condition. +func boundedLoops(ctx *task.OrchestrationContext) (any, error) { + total := 0 + for i := 0; i < 10; i++ { + total += i + } + for total < 100 { + total *= 2 + } + for { + total++ + if total > 200 { + break + } + } + for { + if total > 300 { + return total, nil + } + total += 3 + } +} + +// opaqueLoop calls a package the analyzer does not model, so nothing is proven. +func opaqueLoop(ctx *task.OrchestrationContext) (any, error) { + joined := "" + for { + joined = filepath.Join(joined, "segment") + } +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(spinForever) + _ = registry.AddOrchestrator(spinOnConstantCondition) + _ = registry.AddOrchestrator(spinThroughHelpers) + _ = registry.AddOrchestrator(spinThroughRecursion) + _ = registry.AddOrchestrator(eternalTimerLoop) + _ = registry.AddOrchestrator(eternalActivityLoop) + _ = registry.AddOrchestrator(continueAsNewLoop) + _ = registry.AddOrchestrator(boundedLoops) + _ = registry.AddOrchestrator(opaqueLoop) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/negative/negative.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/negative/negative.go new file mode 100644 index 00000000..403a64ae --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/negative/negative.go @@ -0,0 +1,128 @@ +// Package negative collects orchestrators that must never produce a diagnostic. +// Every construct here is either deterministic, routed through a durable task +// API, or outside the reachability of a registered orchestrator. +package negative + +import ( + "encoding/json" + "errors" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/microsoft/durabletask-go/task" +) + +// durableSequence uses activities, timers, events, and entities. +func durableSequence(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var greeting string + if err := ctx.CallActivity("sayHello", task.WithActivityInput(input)).Await(&greeting); err != nil { + return nil, err + } + if err := ctx.CreateTimer(30 * time.Second).Await(nil); err != nil { + return nil, err + } + if err := ctx.WaitForSingleEvent("approval", time.Hour).Await(nil); err != nil { + return nil, err + } + unlock, err := ctx.LockEntities(task.EntityID{Name: "counter", Key: input}) + if err != nil { + return nil, err + } + defer unlock() + if err := ctx.CallEntity(task.EntityID{Name: "counter", Key: input}, "add").Await(nil); err != nil { + return nil, err + } + return greeting, nil +} + +// deterministicComputation only uses pure standard library helpers. +func deterministicComputation(ctx *task.OrchestrationContext) (any, error) { + values := []int{3, 1, 2} + sort.Ints(values) + sort.Slice(values, func(i, j int) bool { return values[i] < values[j] }) + + names := strings.Split("c,a,b", ",") + sort.Strings(names) + joined := strings.Join(names, "-") + + parsed, err := strconv.Atoi("42") + if err != nil { + return nil, fmt.Errorf("parse: %w", err) + } + encoded, err := json.Marshal(map[string]int{"value": parsed}) + if err != nil { + return nil, err + } + if errors.Is(err, strconv.ErrSyntax) { + return nil, err + } + return joined + string(encoded), nil +} + +// durableFanOut uses the orchestration scheduler instead of goroutines. +func durableFanOut(ctx *task.OrchestrationContext) (any, error) { + group := ctx.NewWaitGroup() + pending := make([]task.Task, 0, 3) + for i := 0; i < 3; i++ { + pending = append(pending, ctx.CallActivity("sayHello", task.WithActivityInput(i))) + } + for _, item := range pending { + group.Add(1) + current := item + ctx.Go(func(child *task.OrchestrationContext) { + defer group.Done() + _ = current.Await(nil) + }) + } + group.Wait(ctx) + return len(pending), nil +} + +// durableSelection waits on durable cases rather than a Go select. +func durableSelection(ctx *task.OrchestrationContext) (any, error) { + work := ctx.CallActivity("sayHello") + timeout := ctx.CreateTimer(time.Minute) + winner := "" + ctx.Select( + task.OnTask(work, func(task.Task) { winner = "work" }), + task.OnTask(timeout, func(task.Task) { winner = "timeout" }), + ) + events := task.NewEventChannel[string](ctx, "signal") + ctx.Select(task.OnEvent(events, func(string) { winner = "event" })) + value, err := events.ReceiveErr(ctx) + if err != nil { + return nil, err + } + return winner + value, nil +} + +// replaySafeLogging uses the orchestration logger. +func replaySafeLogging(ctx *task.OrchestrationContext) (any, error) { + ctx.Logger().Info("progress", "instance", ctx.ID, "replaying", ctx.IsReplaying) + ctx.SetCustomStatus(fmt.Sprintf("instance %s at %s", ctx.ID, ctx.CurrentTimeUtc)) + return nil, nil +} + +// activityBody holds hazards that are never analyzed because activities are +// not replayed. Activities are delivered at least once and must be idempotent. +func activityBody(ctx task.ActivityContext) (any, error) { + go func() {}() + return time.Now(), nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(durableSequence) + _ = registry.AddOrchestrator(deterministicComputation) + _ = registry.AddOrchestrator(durableFanOut) + _ = registry.AddOrchestrator(durableSelection) + _ = registry.AddOrchestrator(replaySafeLogging) + _ = registry.AddActivityN("sayHello", activityBody) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/random/random.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/random/random.go new file mode 100644 index 00000000..2e201a29 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/random/random.go @@ -0,0 +1,129 @@ +// Package random exercises the nondeterministic identifier and random checks. +package random + +import ( + cryptorand "crypto/rand" + "math/big" + mathrand "math/rand" + randv2 "math/rand/v2" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/task" +) + +func randomIdentifiers(ctx *task.OrchestrationContext) (any, error) { + _ = uuid.New() // want `uuid\.New is not deterministic in an orchestrator` + _ = uuid.NewString() // want `uuid\.NewString is not deterministic in an orchestrator` + if id, err := uuid.NewRandom(); err == nil { // want `uuid\.NewRandom is not deterministic in an orchestrator` + _ = id + } + if id, err := uuid.NewUUID(); err == nil { // want `uuid\.NewUUID is not deterministic in an orchestrator` + _ = id + } + return nil, nil +} + +func cryptoRandom(ctx *task.OrchestrationContext) (any, error) { + buffer := make([]byte, 16) + if _, err := cryptorand.Read(buffer); err != nil { // want `crypto/rand\.Read is not deterministic in an orchestrator` + return nil, err + } + value, err := cryptorand.Int(cryptorand.Reader, big.NewInt(10)) // want `crypto/rand\.Int is not deterministic in an orchestrator` + if err != nil { + return nil, err + } + return value.String(), nil +} + +func globalMathRandom(ctx *task.OrchestrationContext) (any, error) { + _ = mathrand.Intn(10) // want `math/rand\.Intn uses the global random source` + _ = mathrand.Float64() // want `math/rand\.Float64 uses the global random source` + _ = mathrand.Perm(3) // want `math/rand\.Perm uses the global random source` + _ = randv2.IntN(10) // want `math/rand/v2\.IntN uses the global random source` + _ = randv2.Float64() // want `math/rand/v2\.Float64 uses the global random source` + return nil, nil +} + +// seededFromWallClock draws its seed from the host clock, which differs on every +// replay, so the generator is provably nondeterministic. +func seededFromWallClock(ctx *task.OrchestrationContext) (any, error) { + derived := mathrand.New(mathrand.NewSource(time.Now().UnixNano())) // want `time\.Now is not deterministic in an orchestrator` + return derived.Intn(10), nil // want `\(\*rand\.Rand\)\.Intn is seeded from a nondeterministic source` +} + +// seededThroughLocal hoists the same host-clock seed into a local, which the +// analyzer follows through its single assignment. +func seededThroughLocal(ctx *task.OrchestrationContext) (any, error) { + seed := time.Now().UnixNano() // want `time\.Now is not deterministic in an orchestrator` + generator := mathrand.New(mathrand.NewSource(seed)) + return generator.Int63(), nil // want `\(\*rand\.Rand\)\.Int63 is seeded from a nondeterministic source` +} + +// seededFromInput replays identically: orchestration input is restored from +// history on every turn, so a generator seeded from it produces the same values. +func seededFromInput(ctx *task.OrchestrationContext) (any, error) { + var seed int64 + if err := ctx.GetInput(&seed); err != nil { + return nil, err + } + derived := mathrand.New(mathrand.NewSource(seed)) + return derived.Intn(10), nil +} + +// deterministicSeed uses a compile-time constant, which replays identically. +func deterministicSeed(ctx *task.OrchestrationContext) (any, error) { + const seed = 42 + generator := mathrand.New(mathrand.NewSource(seed)) + pinned := randv2.New(randv2.NewPCG(1, 2)) + return generator.Intn(10) + pinned.IntN(10), nil +} + +// seedHeldInIdentifier stores a constant-seeded source in its own variable +// before constructing the generator, which is still fully deterministic. +func seedHeldInIdentifier(ctx *task.OrchestrationContext) (any, error) { + source := mathrand.NewSource(7) + generator := mathrand.New(source) + + const pinnedSeed = 99 + pinned := mathrand.NewSource(pinnedSeed) + fromConstant := mathrand.New(pinned) + return generator.Intn(10) + fromConstant.Intn(10), nil +} + +// seedFromParameter cannot be traced to a source either way, so the generator is +// left alone rather than guessed at. +func seedFromParameter(ctx *task.OrchestrationContext) (any, error) { + return generatorFor(ctx.NewGuid()), nil +} + +func generatorFor(token string) int { + generator := mathrand.New(mathrand.NewSource(int64(len(token)))) + return generator.Intn(10) +} + +// deterministicIdentifiers only uses name-based and durable sources. +func deterministicIdentifiers(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.NewGuid() + _ = uuid.NewSHA1(uuid.NameSpaceDNS, []byte("stable")) + _ = uuid.NewMD5(uuid.NameSpaceDNS, []byte("stable")) + parsed, err := uuid.Parse("00000000-0000-0000-0000-000000000000") + if err != nil { + return nil, err + } + return parsed.String(), nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(randomIdentifiers) + _ = registry.AddOrchestrator(cryptoRandom) + _ = registry.AddOrchestrator(globalMathRandom) + _ = registry.AddOrchestrator(seededFromWallClock) + _ = registry.AddOrchestrator(seededThroughLocal) + _ = registry.AddOrchestrator(seededFromInput) + _ = registry.AddOrchestrator(deterministicSeed) + _ = registry.AddOrchestrator(seedHeldInIdentifier) + _ = registry.AddOrchestrator(seedFromParameter) + _ = registry.AddOrchestrator(deterministicIdentifiers) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registration/registration.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registration/registration.go new file mode 100644 index 00000000..fb23590f --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registration/registration.go @@ -0,0 +1,100 @@ +// Package registration exercises the registration-form checks. +package registration + +import "github.com/microsoft/durabletask-go/task" + +func alpha(ctx *task.OrchestrationContext) (any, error) { return nil, nil } + +func beta(ctx *task.OrchestrationContext) (any, error) { return nil, nil } + +func work(ctx task.ActivityContext) (any, error) { return nil, nil } + +type workflow struct{} + +func (workflow) Run(ctx *task.OrchestrationContext) (any, error) { return nil, nil } + +func registerInvalid() { + registry := task.NewTaskRegistry() + + _ = registry.AddOrchestrator(nil) // want `task\.TaskRegistry registration with a nil orchestrator always returns an error` + _ = registry.AddActivity(nil) // want `task\.TaskRegistry registration with a nil activity always returns an error` + _ = registry.AddOrchestratorN("", alpha) // want `task\.TaskRegistry rejects an empty orchestrator name` + _ = registry.AddOrchestratorN(" ", alpha) // want `task\.TaskRegistry registers this orchestrator under a name that is only whitespace` + _ = registry.AddActivityN("", work) // want `task\.TaskRegistry rejects an empty activity name` + _ = registry.AddOrchestratorVersion(" ", alpha) // want `task\.TaskRegistry rejects an orchestrator version that is only whitespace` + _ = registry.AddActivityNVersion("work", "\t", work) // want `task\.TaskRegistry rejects an activity version that is only whitespace` + + _ = registry.AddOrchestratorN("duplicate", alpha) + _ = registry.AddOrchestratorN("duplicate", beta) // want `orchestrator "duplicate" is registered more than once` + _ = registry.AddOrchestratorN("Duplicate", beta) // want `orchestrator "Duplicate" is registered more than once` + + _ = registry.AddOrchestrator(func(ctx *task.OrchestrationContext) (any, error) { // want `task\.TaskRegistry derives the orchestrator name from a function literal` + return nil, nil + }) + assigned := func(ctx *task.OrchestrationContext) (any, error) { return nil, nil } + _ = registry.AddOrchestrator(assigned) // want `task\.TaskRegistry derives the orchestrator name from a function literal` + _ = registry.AddActivity(func(ctx task.ActivityContext) (any, error) { // want `task\.TaskRegistry derives the activity name from a function literal` + return nil, nil + }) + + instance := workflow{} + _ = registry.AddOrchestrator(instance.Run) // want `task\.TaskRegistry derives the orchestrator name from a method value` +} + +// registerValid uses every well-formed registration shape. +func registerValid() { + registry := task.NewTaskRegistry() + instance := workflow{} + _ = registry.AddOrchestrator(alpha) + _ = registry.AddOrchestratorN("named", beta) + _ = registry.AddOrchestratorVersion("2.0", beta) + _ = registry.AddOrchestratorNVersion("named", "2.0", beta) + _ = registry.AddOrchestratorN("method", instance.Run) + _ = registry.AddActivity(work) + _ = registry.AddActivityN("work-alias", work) +} + +// versionedNames prove the duplicate key matches the registry's own +// normalization, which lowercases the version and trims nothing. +func versionedNames() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestratorNVersion("versioned", "1.0", beta) + _ = registry.AddOrchestratorNVersion("versioned", "2.0", beta) + _ = registry.AddOrchestratorNVersion("versioned", "V3", beta) + _ = registry.AddOrchestratorNVersion("versioned", "v3", beta) // want `orchestrator "versioned" is registered more than once` + + // A version padded with whitespace is a distinct registry key, because the + // registry lowercases the version without trimming it. + _ = registry.AddOrchestratorNVersion("padded", "4.0", beta) + _ = registry.AddOrchestratorNVersion("padded", " 4.0", beta) +} + +// computedVersions register the same name under versions that are only known at +// runtime, so no duplicate can be proven. +func computedVersions(suffix string) { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestratorNVersion("rolling", "1."+suffix, beta) + _ = registry.AddOrchestratorNVersion("rolling", "2."+suffix, beta) + _ = registry.AddActivityNVersion("rollingWork", suffix, work) + _ = registry.AddActivityNVersion("rollingWork", suffix, work) +} + +// separateRegistry proves duplicate detection is scoped to one registry value. +func separateRegistry() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestratorN("duplicate", alpha) +} + +// directInvocation calls a registered orchestrator as a plain Go function. +func directInvocation(ctx *task.OrchestrationContext) (any, error) { + if _, err := alpha(ctx); err != nil { // want `orchestrator "alpha" is invoked directly` + return nil, err + } + return ctx.CallSubOrchestrator(alpha), nil +} + +func registerDirect() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(directInvocation) + _ = registry.AddOrchestrator(alpha) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registrationflow/registrationflow.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registrationflow/registrationflow.go new file mode 100644 index 00000000..469f7b16 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/registrationflow/registrationflow.go @@ -0,0 +1,186 @@ +package registrationflow + +import "github.com/microsoft/durabletask-go/task" + +func root(*task.OrchestrationContext) (any, error) { return nil, nil } + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func branches(flag bool) { + registry := task.NewTaskRegistry() + if flag { + _ = registry.AddOrchestratorN("branch", root) + } else { + _ = registry.AddOrchestratorN("branch", root) + } +} + +func switchClauses(value int) { + registry := task.NewTaskRegistry() + switch value { + case 1: + _ = registry.AddActivityN("clause", work) + case 2: + _ = registry.AddActivityN("clause", work) + default: + _ = registry.AddActivityN("clause", work) + } +} + +func typeSwitchClauses(value any) { + registry := task.NewTaskRegistry() + switch value.(type) { + case int: + _ = registry.AddActivityN("clause", work) + case string: + _ = registry.AddActivityN("clause", work) + } +} + +func selectClauses(first, second <-chan struct{}) { + registry := task.NewTaskRegistry() + select { + case <-first: + _ = registry.AddActivityN("clause", work) + case <-second: + _ = registry.AddActivityN("clause", work) + } +} + +func shortCircuit(flag bool) { + registry := task.NewTaskRegistry() + _ = flag && registry.AddActivityN("conditional", work) == nil + _ = !flag && registry.AddActivityN("conditional", work) == nil +} + +func reassigned() { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + registry = task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) +} + +func registryResult() (*task.TaskRegistry, error) { return task.NewTaskRegistry(), nil } + +func tupleAssignment() { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + registry, _ = registryResult() + _ = registry.AddActivityN("fresh", work) +} + +func unknownInitializer() { + var registry, _ = registryResult() + _ = registry.AddActivityN("unknown", work) + _ = registry.AddActivityN("unknown", work) +} + +func rangeAssignment(registries []*task.TaskRegistry) { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + for _, (registry) = range registries { + } + _ = registry.AddActivityN("fresh", work) +} + +func closures(flag bool) { + registry := task.NewTaskRegistry() + first := func() { _ = registry.AddActivityN("closure", work) } + second := func() { _ = registry.AddActivityN("closure", work) } + if flag { + first() + } else { + second() + } +} + +var shared = task.NewTaskRegistry() + +func firstFunction() { _ = shared.AddActivityN("shared", work) } +func secondFunction() { _ = shared.AddActivityN("shared", work) } + +func indirectWrite() { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + *registry = *task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) +} + +func aliasWrite() { + registry := task.NewTaskRegistry() + alias := registry + _ = registry.AddActivityN("fresh", work) + *alias = *task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) +} + +func writeThroughOriginal() { + registry := task.NewTaskRegistry() + alias := registry + _ = alias.AddActivityN("fresh", work) + *registry = *task.NewTaskRegistry() + _ = alias.AddActivityN("fresh", work) +} + +func sharedRegistry() *task.TaskRegistry { return shared } + +func resetShared() { *shared = *task.NewTaskRegistry() } + +func unknownSource() { + registry := sharedRegistry() + _ = registry.AddActivityN("fresh", work) + resetShared() + _ = registry.AddActivityN("fresh", work) +} + +func reset(registry *task.TaskRegistry) { *registry = *task.NewTaskRegistry() } + +func helperWrite() { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + reset(registry) + _ = registry.AddActivityN("fresh", work) +} + +func straightLine() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestratorN("root", root) + _ = registry.AddOrchestratorN("ROOT", root) // want `orchestrator "ROOT" is registered more than once` + registry.AddActivityNVersion("work", "V1", work) + registry.AddActivityNVersion("WORK", "v1", work) // want `activity "WORK" is registered more than once` +} + +func sameClause(value int) { + registry := task.NewTaskRegistry() + switch value { + case 1: + _ = registry.AddActivityN("work", work) + _ = registry.AddActivityN("work", work) // want `activity "work" is registered more than once` + case 2: + _ = registry.AddActivityN("work", work) + } +} + +func exclusiveJumps(flag bool) { + registry := task.NewTaskRegistry() + if flag { + goto second + } + _ = registry.AddActivityN("once", work) + goto done +second: + _ = flag + _ = registry.AddActivityN("once", work) +done: + _ = flag +} + +func mapKeyAliasWrite() { + registry := task.NewTaskRegistry() + _ = registry.AddActivityN("fresh", work) + registries := map[*task.TaskRegistry]bool{registry: true} + for alias := range registries { + *alias = *task.NewTaskRegistry() + } + _ = registry.AddActivityN("fresh", work) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknames/tasknames.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknames/tasknames.go new file mode 100644 index 00000000..2a198115 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknames/tasknames.go @@ -0,0 +1,49 @@ +// Package tasknames exercises the unresolved activity and sub-orchestration +// name check against a complete, fully literal registration set. +package tasknames + +import "github.com/microsoft/durabletask-go/task" + +func parent(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("sendGreeting") + _ = ctx.CallActivity(sendGreeting) + _ = ctx.CallActivity("format") + _ = ctx.CallActivity("missingActivity") // want `activity "missingActivity" is not registered with task.TaskRegistry in this package` + + _ = ctx.CallSubOrchestrator("child") + _ = ctx.CallSubOrchestrator(child) + _ = ctx.CallSubOrchestrator("missingChild") // want `sub-orchestration "missingChild" is not registered with task.TaskRegistry in this package` + return nil, nil +} + +// dynamicNames are computed at runtime, so the analyzer cannot prove anything. +func dynamicNames(ctx *task.OrchestrationContext) (any, error) { + var name string + if err := ctx.GetInput(&name); err != nil { + return nil, err + } + _ = ctx.CallActivity(name) + _ = ctx.CallSubOrchestrator(name) + return nil, nil +} + +func child(ctx *task.OrchestrationContext) (any, error) { + return nil, nil +} + +func sendGreeting(ctx task.ActivityContext) (any, error) { + return nil, nil +} + +func format(ctx task.ActivityContext) (any, error) { + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(parent) + _ = registry.AddOrchestrator(dynamicNames) + _ = registry.AddOrchestrator(child) + _ = registry.AddActivity(sendGreeting) + _ = registry.AddActivityN("format", format) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesalias/tasknamesalias.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesalias/tasknamesalias.go new file mode 100644 index 00000000..aaf5822e --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesalias/tasknamesalias.go @@ -0,0 +1,25 @@ +package tasknamesalias + +import ( + external "github.com/example/registrationhelpers" + "github.com/microsoft/durabletask-go/task" +) + +type registryAlias = task.TaskRegistry + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func register() { + var registry *registryAlias = task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + alias := registry + install := external.RegisterActivities + install(alias) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesdynamic/tasknamesdynamic.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesdynamic/tasknamesdynamic.go new file mode 100644 index 00000000..1407da2d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesdynamic/tasknamesdynamic.go @@ -0,0 +1,43 @@ +// Package tasknamesdynamic proves the unresolved-name check stays silent when +// the package's registration set cannot be proven complete. +package tasknamesdynamic + +import ( + "strings" + + "github.com/microsoft/durabletask-go/task" +) + +// dynamicActivityName calls a name that is absent from the literal registrations, +// but one activity is registered under a computed name, so absence proves nothing. +func dynamicActivityName(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("missingActivity") + return nil, nil +} + +// wildcardSubOrchestration is covered by a "*" orchestrator registration. +func wildcardSubOrchestration(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallSubOrchestrator("missingChild") + return nil, nil +} + +func known(ctx task.ActivityContext) (any, error) { + return nil, nil +} + +func computed(ctx task.ActivityContext) (any, error) { + return nil, nil +} + +func fallback(ctx *task.OrchestrationContext) (any, error) { + return nil, nil +} + +func register(suffix string) { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(dynamicActivityName) + _ = registry.AddOrchestrator(wildcardSubOrchestration) + _ = registry.AddOrchestratorN("*", fallback) + _ = registry.AddActivity(known) + _ = registry.AddActivityN(strings.ToLower("computed"+suffix), computed) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesexternal/tasknamesexternal.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesexternal/tasknamesexternal.go new file mode 100644 index 00000000..f153a3f7 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesexternal/tasknamesexternal.go @@ -0,0 +1,23 @@ +package tasknamesexternal + +import ( + external "github.com/example/registrationhelpers" + "github.com/microsoft/durabletask-go/task" +) + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + external.RegisterActivities(registry) + _ = registry.AddActivity(nil) // want `task\.TaskRegistry registration with a nil activity always returns an error` +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesglobal/tasknamesglobal.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesglobal/tasknamesglobal.go new file mode 100644 index 00000000..ad243e44 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesglobal/tasknamesglobal.go @@ -0,0 +1,20 @@ +package tasknamesglobal + +import "github.com/microsoft/durabletask-go/task" + +var Shared *task.TaskRegistry + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func Register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + Shared = registry +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesinterface/tasknamesinterface.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesinterface/tasknamesinterface.go new file mode 100644 index 00000000..ba0c9c23 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesinterface/tasknamesinterface.go @@ -0,0 +1,19 @@ +package tasknamesinterface + +import "github.com/microsoft/durabletask-go/task" + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func register(install func(any)) { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + var erased any = registry + install(erased) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesmethodvalue/tasknamesmethodvalue.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesmethodvalue/tasknamesmethodvalue.go new file mode 100644 index 00000000..badf1a25 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesmethodvalue/tasknamesmethodvalue.go @@ -0,0 +1,23 @@ +package tasknamesmethodvalue + +import "github.com/microsoft/durabletask-go/task" + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("indirectWork") + _ = ctx.CallSubOrchestrator("indirectChild") + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func child(*task.OrchestrationContext) (any, error) { return nil, nil } + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + addActivity := registry.AddActivityN + _ = addActivity("indirectWork", work) + addOrchestrator := registry.AddOrchestratorN + _ = addOrchestrator("indirectChild", child) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesparameter/tasknamesparameter.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesparameter/tasknamesparameter.go new file mode 100644 index 00000000..cc35d41a --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesparameter/tasknamesparameter.go @@ -0,0 +1,18 @@ +package tasknamesparameter + +import "github.com/microsoft/durabletask-go/task" + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func Install(registry *task.TaskRegistry) { + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivityN("local", work) + _ = registry.AddActivity(nil) // want `task\.TaskRegistry registration with a nil activity always returns an error` +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamestuplefactory/tasknamestuplefactory.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamestuplefactory/tasknamestuplefactory.go new file mode 100644 index 00000000..fb2de51a --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamestuplefactory/tasknamestuplefactory.go @@ -0,0 +1,25 @@ +package tasknamestuplefactory + +import ( + external "github.com/example/registrationhelpers" + "github.com/microsoft/durabletask-go/task" +) + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func register() error { + registry, err := external.NewRegistry() + if err != nil { + return err + } + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivityN("local", work) + return nil +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesunresolved/tasknamesunresolved.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesunresolved/tasknamesunresolved.go new file mode 100644 index 00000000..5bf6588d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/tasknamesunresolved/tasknamesunresolved.go @@ -0,0 +1,18 @@ +package tasknamesunresolved + +import "github.com/microsoft/durabletask-go/task" + +func caller(ctx *task.OrchestrationContext) (any, error) { + _ = ctx.CallActivity("externalWork") + _ = ctx.CallSubOrchestrator("externalChild") + return nil, nil +} + +func work(task.ActivityContext) (any, error) { return nil, nil } + +func register(install func(*task.TaskRegistry)) { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(caller) + _ = registry.AddActivity(work) + install(registry) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/wallclock/wallclock.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/wallclock/wallclock.go new file mode 100644 index 00000000..e1b0f559 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/wallclock/wallclock.go @@ -0,0 +1,52 @@ +// Package wallclock exercises the wall-clock checks. +package wallclock + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func clockReads(ctx *task.OrchestrationContext) (any, error) { + start := time.Now() // want `time\.Now is not deterministic in an orchestrator` + _ = time.Since(start) // want `time\.Since is not deterministic in an orchestrator` + _ = time.Until(start) // want `time\.Until is not deterministic in an orchestrator` + return start, nil +} + +func hostTimers(ctx *task.OrchestrationContext) (any, error) { + time.Sleep(time.Second) // want `time\.Sleep is not deterministic in an orchestrator` + _ = time.After(time.Second) // want `time\.After is not deterministic in an orchestrator` + _ = time.Tick(time.Second) // want `time\.Tick is not deterministic in an orchestrator` + timer := time.NewTimer(time.Second) // want `time\.NewTimer is not deterministic in an orchestrator` + ticker := time.NewTicker(time.Second) // want `time\.NewTicker is not deterministic in an orchestrator` + timer.Stop() + ticker.Stop() + time.AfterFunc(time.Second, func() {}) // want `time\.AfterFunc is not deterministic in an orchestrator` + return nil, nil +} + +// durableClock only uses the orchestration clock and durable timers. +func durableClock(ctx *task.OrchestrationContext) (any, error) { + now := ctx.CurrentTimeUtc + deadline := now.Add(24 * time.Hour) + if deadline.After(now) && !deadline.IsZero() { + _ = ctx.CreateTimer(time.Minute) + } + _ = deadline.Sub(now) + _ = time.Duration(5) * time.Second + _ = time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC) + return deadline.Format(time.RFC3339), nil +} + +// unreachableClock is never registered, so it is never analyzed. +func unreachableClock() time.Time { + return time.Now() +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(clockReads) + _ = registry.AddOrchestrator(hostTimers) + _ = registry.AddOrchestrator(durableClock) +} diff --git a/cmd/orchestratorvet/go.mod b/cmd/orchestratorvet/go.mod new file mode 100644 index 00000000..982d6189 --- /dev/null +++ b/cmd/orchestratorvet/go.mod @@ -0,0 +1,10 @@ +module github.com/microsoft/durabletask-go/cmd/orchestratorvet + +go 1.25.0 + +require golang.org/x/tools v0.49.0 + +require ( + golang.org/x/mod v0.40.0 // indirect + golang.org/x/sync v0.22.0 // indirect +) diff --git a/cmd/orchestratorvet/go.sum b/cmd/orchestratorvet/go.sum new file mode 100644 index 00000000..8721addb --- /dev/null +++ b/cmd/orchestratorvet/go.sum @@ -0,0 +1,8 @@ +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +golang.org/x/mod v0.40.0 h1:hUv+3cXcdRHz08UmSiOob7sadHig73uo5bkXxQ/tvUs= +golang.org/x/mod v0.40.0/go.mod h1:0/weTWkPWGBikyTWAX3dkjVztMmBA5hM0DH6BElSupE= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/tools v0.49.0 h1:3NI7VXzL9+1WZD52Dx2ttoPwD5DWrFGpl9mFZDlmisI= +golang.org/x/tools v0.49.0/go.mod h1:SJNXV9DBKT0UbdttsQjbfJlAE/q+y36++zo3uL3N0Oo= diff --git a/cmd/orchestratorvet/main.go b/cmd/orchestratorvet/main.go new file mode 100644 index 00000000..e868c9e2 --- /dev/null +++ b/cmd/orchestratorvet/main.go @@ -0,0 +1,10 @@ +package main + +import ( + "github.com/microsoft/durabletask-go/cmd/orchestratorvet/analysis/orchestratorgo" + "golang.org/x/tools/go/analysis/unitchecker" +) + +func main() { + unitchecker.Main(orchestratorgo.Analyzer) +} diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md new file mode 100644 index 00000000..1d0b4702 --- /dev/null +++ b/durabletaskscheduler/README.md @@ -0,0 +1,365 @@ +# Durable Task Scheduler SDK + +The `durabletaskscheduler` package is the primary SDK surface for connecting Go +management clients and workers to Durable Task Scheduler (DTS). + +## Configuration + +Use a connection string from the environment: + +```bash +export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' +go run ./samples/durabletaskscheduler +``` + +For the local DTS emulator: + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/durabletaskscheduler +``` + +Connection strings support the Azure Identity modes that have Go equivalents: +`DefaultAzure`, `ManagedIdentity`, `WorkloadIdentity`, `Environment`, +`AzureCLI`, `AzurePowerShell`, `InteractiveBrowser`, and `None`. Keys and +`Authentication` values are case-insensitive, surrounding whitespace is trimmed, +empty segments are skipped, and a repeated key uses its last value. Only the +first `=` in a segment separates the key from the value, so values may contain +`=`. Unknown keys, malformed segments, and a missing or blank `Endpoint`, +`TaskHub`, or `Authentication` are rejected. `VisualStudio` and +`VisualStudioCode` have no Go equivalent, and `TokenCredential` is programmatic +only: both are rejected with guidance to use `NewOptionsWithCredential` with any +`azcore.TokenCredential`. Plaintext `http://` endpoints are accepted only with +`Authentication=None`, which also requires `AllowInsecureConnection`. + +Authentication-specific keys are `ClientID`, `TenantID`, `TokenFilePath`, and +the comma-separated `AdditionallyAllowedTenants`. Each mode consumes only the +fields Azure Identity for Go supports for it: + +| Authentication | ClientID | TenantID | TokenFilePath | AdditionallyAllowedTenants | +| --- | --- | --- | --- | --- | +| `DefaultAzure` | ignored | used | ignored | used | +| `ManagedIdentity` | used | ignored | ignored | ignored | +| `WorkloadIdentity` | used | used | used | used | +| `Environment` | ignored | ignored | ignored | ignored | +| `AzureCLI` | ignored | used | ignored | used | +| `AzurePowerShell` | ignored | used | ignored | used | +| `InteractiveBrowser` | used | used | ignored | used | +| `None` | rejected | rejected | rejected | rejected | +| `TokenCredential` | rejected | rejected | rejected | rejected | + +`Environment` is configured entirely by the `AZURE_*` environment variables, and +`WorkloadIdentity` falls back to them for any field left unset. Fields are +trimmed before use and blank `AdditionallyAllowedTenants` entries are dropped. +`None` and `TokenCredential` construct no Azure Identity credential, so +supplying identity fields for them fails validation instead of being silently +ignored. + +Access tokens are requested for `Options.ResourceID` (default +`https://durabletask.io`) with trailing slashes removed and `/.default` +appended. Token acquisition failures are surfaced as retriable `Unavailable` +gRPC errors. Tokens and immutable authorization metadata are cached until the +credential's `RefreshOn` time (or five minutes before expiry), and concurrent +refreshes are coalesced. Every RPC carries `taskhub` and `x-user-agent` metadata; worker +connections add `workerid`. `Options.UserAgent` and `Options.WorkerID` override +the generated values and are rejected if they contain leading/trailing +whitespace or newlines. Unset worker IDs default to +`,,`. + +`Options.HelloTimeout` (default 30 seconds) bounds the fail-fast `Hello` +handshake for both `NewClient` and the worker connection factory; the caller's +context still applies when it is shorter. Client channels use a default gRPC +service config that retries `UNAVAILABLE` up to five attempts with a 50 ms +initial backoff, 250 ms cap, and multiplier 2. Worker channels do not, because +the worker owns its own reconnect loop. + +Individual gRPC messages are bounded to 64 MiB by default through +`Options.MaxReceiveMessageSize` and `Options.MaxSendMessageSize`. The worker uses a +3.9 MiB safety bound below the service ceiling. When +`Options.MaxSendMessageSize` is lower, that configured send limit becomes the +bound instead. The worker validates the final response after large-payload +externalization and fails an orchestration once +with `OrchestratorResponseTooLarge` if it still exceeds that limit. The legacy +`isPartial`/`chunkIndex` response fields are deprecated; the Go worker does not +use them because validation against the current DTS emulator did not provide a +portable chunking path. Configure large-payload storage or reduce per-turn +fan-out instead. Active streams +use a two-minute keepalive with a 20-second acknowledgement timeout; set +`Options.KeepaliveTime` to zero to disable it. The timeout is ignored while +keepalive is disabled. Values below 30 seconds are rejected to avoid +aggressive-ping disconnects. + +`NewClient` creates and owns a management connection; call `Close` when done. +Options created with `NewOptions`, `NewOptionsFromConnectionString`, or +`NewOptionsWithCredential` recreate the channel after five consecutive +`Unavailable` responses (or unexpected `DeadlineExceeded` responses), with a +30-second minimum interval. Successful and application-level responses reset +the counter, while caller cancellation/deadlines and expected deadlines from +instance start/completion long polls do not count. Configure the thresholds with +`Options.ChannelRecreateFailureThreshold` and +`Options.ChannelRecreateMinInterval`. Authentication, interceptors, large +payload settings, and data conversion are preserved across replacements. +A hand-built `Options` value with a zero failure threshold disables recreation. + +`client.NewTaskHubGrpcClient`, `client.NewTaskHubGrpcWorker`, and +`TaskHubGrpcClient.StartWorkItemListener` are lower-level APIs that borrow the +caller's `grpc.ClientConnInterface`; they never replace or close it. +`NewWorker` owns its channels, recreates them after transient disconnects, and +closes retired channels after their in-flight completions drain. +`client.NewTaskHubGrpcWorkerWithConnectionFactory` provides the same lifecycle +when its factory returns a non-nil closer, which transfers ownership to the +worker. Use an owning configuration to recover from a permanently wedged +channel. After a worker has started, `Unauthenticated` and `PermissionDenied` +stream or reconnect-handshake responses are retried with backoff so token +refresh and RBAC propagation can recover without restarting the worker. See +`client.NewTaskHubGrpcWorker` for the full ownership contract. + +Reconnect and RPC retry delays are deterministic and always stay within +`[baseDelay, maxDelay]`. A stream that delivers at least one message before it +ends is treated as a drain and restarts the schedule at the base delay; a stream +that stays silent past `WithWorkerSilentDisconnectTimeout` before its first +message is treated as poisoned and keeps escalating. + +`Options.MaximumTimerInterval` defaults to three days. Longer durable timers +are split into deterministic sequential timer actions that retain the original +deadline. Histories written before splitting remain compatible once the original +logical timer deadline fires. Changing the interval between two splitting +configurations is replay-breaking for affected in-flight orchestrations. + +### Advanced management + +`TaskHubGrpcClient` exposes bounded `QueryInstances` and `ListInstanceIDs` +operations with opaque continuation tokens, plus `RestartInstance`, +`RewindInstance`, batch/filter `PurgeInstances`, +`SkipGracefulOrchestrationTerminations`, and task-hub lifecycle RPCs. Queries +can filter locally by exact tag key/value pairs when the current wire contract +does not carry tag filters. + +`GetOrchestrationHistory` returns API-owned history records with validated event +and approximate byte caps. `StreamOrchestrationHistory` invokes a callback in +service order without retaining the history in memory. Both preserve execution identity, +timestamps, failures, tags, context fields, rewind markers, and native entity +events. Serialized payloads remain raw until a `ReadInput`, `ReadResult`, or +`ReadData` helper applies the configured data converter. + +The current DTS emulator supports query, restart, and batch purge, but has known +service limitations: `SkipGracefulOrchestrationTerminations` +is unimplemented, rewind can return success without transitioning the failed +instance, filtered purge can complete without deleting matches, and +`ListInstanceIds` can omit matching IDs. The emulator integration tests record +these limitations explicitly. + +### Worker routing and capabilities + +Use `client.WithTaskVersioning` for `None`, `Strict`, or `CurrentOrOlder` +acceptance and `Reject` or `Fail` mismatch handling. `VersioningOptions.DefaultVersion` +is applied to sub-orchestrations. When `Options.Versioning` is set, its default +also applies to top-level starts made by the DTS client. + +Registrations use `AddOrchestratorNVersion` and `AddActivityNVersion`. Exact +name/version matches win. Unversioned fallback is allowed only for a logical +name with no versioned registrations. Activities inherit their parent +orchestration version unless explicitly overridden, including an explicit +unversioned `""` override. ContinueAsNew can migrate with +`task.WithContinueAsNewVersion`, including to +`task.UnversionedTaskVersion`. + +The current DTS service accepts numeric versions in +`Major[.Minor[.Patch]]` form. The Go registry also supports case-insensitive +opaque version strings, but DTS applications should use numeric versions such as +`"1.0"` and `"2.0"`. + +Use `client.WithAutoWorkItemFilters()` to derive filters from the registry, or +`client.WithWorkItemFilters` for an explicit override. Local enforcement is a +fallback for services that ignore the filter request. `CurrentOrOlder` ranges +cannot be represented by the protocol filter and are therefore enforced by the +worker. Service-side +filters can leave a task pending indefinitely when no worker advertises it, +whereas unfiltered delivery produces a deterministic task-not-found failure. +Auto-generated filters reject task kinds with no registrations and validate +that strict worker versions can resolve every advertised logical name. +Capability advertisement is explicit: history streaming is enabled by default, +while scheduled tasks use `durabletaskscheduler.WithScheduledTasks()`. Streamed +history accumulation is bounded to 100,000 events and 64 MiB by default; use +`client.WithMaxStreamedHistoryEvents` and +`client.WithMaxStreamedHistoryBytes` to choose different bounded limits. These +are per-work-item limits, so size worker concurrency with the aggregate memory +budget in mind. Exceeding either limit delays and abandons the work item until +the worker is reconfigured with a larger bounded limit. + +### Recurring scheduled tasks + +> **Mixed-SDK constraint:** this surface uses Go SDK-owned system entity and +> orchestrator names and state. Use it only in Go-only task hubs; interoperability +> with .NET workers sharing those system names and state is not defined. + +Scheduled tasks use recurring UTC intervals, not cron expressions. Register the +system entity and orchestrators before creating the worker: + +```go +registry := task.NewTaskRegistry() +registry.AddOrchestratorNVersion("Backup", "1.0", backup) +durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion(registry, "1.0") + +worker, _ := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskscheduler.WithScheduledTasks(), + client.WithAutoWorkItemFilters(), +) +``` + +Create and manage schedules through the DTS client: + +```go +schedules := schedulerClient.ScheduledTasks() +handle, _ := schedules.Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ + ScheduleID: "nightly-backup", + OrchestrationName: "Backup", + TypedOrchestrationInput: backupRequest, + Interval: 24 * time.Hour, + StartAt: firstRun, + StartImmediatelyIfLate: true, + Tags: map[string]string{"team": "storage"}, + ContextFields: api.ContextFields{"tenant": "north"}, +}) + +description, _ := handle.Describe(ctx) +_ = handle.Pause(ctx) +_ = handle.Update(ctx, durabletaskscheduler.ScheduleUpdateOptions{Interval: &newInterval}) +_ = handle.Resume(ctx) +_ = handle.Delete(ctx) +``` + +`Get` returns `nil, nil` for a missing schedule. `List` is continuation-based; +status and creation-time filters are applied after each entity page, so pages +can be underfilled. Schedule operations return typed errors compatible with +`errors.Is` and `errors.As`. A fixed target instance ID prevents overlapping +runs. Fixed target instance IDs cannot be combined with a retry policy because +each retry requires a distinct durable instance. Tags, context fields, and +retry policies use an internal launch +orchestrator because the entity-start wire action cannot carry them directly. +Explicit work-item filters must include the `Schedule` entity and both system +orchestrators; auto filters include them from the registry. + +### Data converters + +Set `Options.DataConverter` to configure one `api.DataConverter` for the DTS +client and worker. The default is `api.JSONDataConverter`. Conversion happens +before large-payload externalization and after hydration. Converter errors are +returned; the SDK never retries a payload with JSON. Raw input/output APIs and +serialized metadata fields bypass conversion. Converter identity is not stored +by the protocol, so deployments must retain backward decoding compatibility. +The legacy skip-graceful termination `reason` remains a plain protocol string +for cross-version service compatibility. + +### Large payloads + +Large payload support uses an opaque, integrity-checked reference encoded in +the existing string payload fields. Production DTS workloads should configure +the same Azure Blob store for every management client and worker: + +```go +store, err := payload.NewAzureBlobStore(payload.AzureBlobStoreOptions{ + ConnectionString: os.Getenv("AzureWebJobsStorage"), + Container: "durabletask-payloads", +}) +if err != nil { + return err +} +options.LargePayloads = &api.LargePayloadOptions{ + Store: store, + Resolver: store, +} +``` + +`payload.NewAzureBlobStore` emits the same self-describing +`blob:v2:` token as the .NET SDK and can read legacy .NET +`blob:v1` tokens. `payload.NewMemoryStore` is only for tests and single-process +experiments because references are lost on restart. `payload.NewFileStore` +requires every client and worker to share the same durable filesystem path; +container-local or other ephemeral storage is not safe. + +Identity authentication uses `AccountURL` plus an `azcore.TokenCredential`. +Cross-account identity reads require an explicit `AllowedHosts` entry. Azure +Blob defaults match .NET: 256 KiB inclusive threshold, 10 MiB maximum, gzip +enabled, eight exponential retries, and the `durabletask-payloads` container. +Tokens are treated as untrusted: userinfo, SAS/query strings, fragments, +unapproved hosts, malformed paths, oversized downloads, and invalid integrity +metadata are rejected. Go-written blobs include size, SHA-256, and content-MD5 +integrity data; .NET blobs without that metadata remain readable. + +Workers advertise `LARGE_PAYLOADS` only when `LargePayloads` is configured. +Resolver implementations must treat reference locations as untrusted and enforce +their own scheme/account/path allow lists. + +## Worker lifecycle + +Use `Start` for background execution or `Run` for blocking execution. `Shutdown` +stops intake, allows in-flight execution and completion RPCs to drain, and +cancels them only if the shutdown context expires. + +## Feature matrix + +| Feature | Status | +| --- | --- | +| Schedule, bounded query/list, and wait for orchestrations | Supported | +| Tags on schedule, metadata, query, sub-orchestration, continue-as-new, restart, and rewind | Supported; distinct from immutable context fields | +| Restart and batch/filter purge | Supported; see emulator limitations above | +| Rewind | Client and wire support are complete; current emulator does not transition instances | +| Skip-graceful termination | Client and wire support are complete; current emulator returns `Unimplemented` | +| Task-hub create/delete | Client and wire support are complete; remote-service behavior is provider-specific | +| Raise events, suspend/resume, terminate, and single-instance purge | Supported | +| Orchestration and activity execution | Supported | +| Bounded orchestration/activity/entity concurrency | Supported | +| Work-item filters for orchestrations, activities, and entities | Supported | +| Completion tokens and abandon RPCs | Supported | +| Oversized orchestration responses | Blob-externalized before send; residual responses above the effective limit fail once with non-retriable guidance; the Go worker does not use deprecated legacy chunking | +| Health pings, silent-disconnect detection, auth/RBAC recovery, and channel recreation | Supported | +| Public orchestration history | Supported through buffered and callback-streaming API-owned records | +| Version-aware registry dispatch and controlled unversioned fallback | Supported | +| Default versions, activity inheritance, and ContinueAsNew migration | Supported; DTS must honor `newVersion` | +| Name/version work-item filters | Supported, auto-generated or explicit, advertised, and locally enforced | +| Pluggable application data conversion | Supported with shared client/worker configuration; default is JSON | +| Recurring interval schedules | Supported in Go-only task hubs; mixed-SDK interoperability for the SDK-owned system entity state is not defined | +| Scheduled-task capability | Supported; register system tasks and opt in with `durabletaskscheduler.WithScheduledTasks()` | +| Azure Blob `blob:v2` payloads | Supported with connection-string or identity authentication and .NET-compatible gzip/token semantics | +| Large-payload capability | Supported and advertised only when a store/resolver is configured | +| Durable entities | Supported: legacy and V2 work items, scheduled signals, calls, queries, and critical sections | +| Status-based instance-ID deduplication and replacement | Supported through `api.OrchestrationIDReusePolicy.DedupeStatuses` | +| History export jobs (preview) | Buffers each complete history in worker memory and writes `api.HistoryEvent` JSON/JSONL; the schema version defaults to preview value `1.0` and is caller-configurable, so assess memory and schema compatibility before enabling | +| Sandbox worker profiles | Not implemented | + +The current V2 protobuf cannot carry per-operation trace context or request time +to an entity worker, and it has no properties map for legacy extended-session +state elision. DTS therefore sends entity state on every V2 work item; causal +trace metadata on entity-emitted actions is best-effort. + +## Emulator tests + +On Apple silicon with Apple Container, the current MCR emulator image runs +under Rosetta: + +```bash +container image pull mcr.microsoft.com/dts/dts-emulator:latest +container run --detach --name dts-emulator \ + --arch amd64 --rosetta \ + --publish 8080:8080 --publish 8082:8082 \ + --env DTS_TASK_HUB_NAMES=default \ + mcr.microsoft.com/dts/dts-emulator:latest +``` + +The integration suite is environment-gated: + +```bash +DTS_EMULATOR_ENDPOINT=http://127.0.0.1:8080 \ +DTS_TASK_HUB=default \ +go test ./tests/durabletaskscheduler -count=1 +``` + +Azurite-backed blob tests additionally use: + +```bash +export AZURITE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;' +``` diff --git a/durabletaskscheduler/authentication_test.go b/durabletaskscheduler/authentication_test.go new file mode 100644 index 00000000..b0adf69a --- /dev/null +++ b/durabletaskscheduler/authentication_test.go @@ -0,0 +1,842 @@ +package durabletaskscheduler + +import ( + "context" + "encoding/json" + "errors" + "os" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/stretchr/testify/require" +) + +// stubCredential is a placeholder azcore.TokenCredential. It never contacts +// Azure and is only used to assert credential plumbing. +type stubCredential struct{ name string } + +func (stubCredential) GetToken( + context.Context, + policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "stub", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +// recordingFactory wraps a credential factory so the specs it receives can be +// asserted. It is passed explicitly instead of replacing process-global state, +// so concurrent tests never observe each other's factory. +type recordingFactory struct { + observed []credentialSpec + factory credentialFactory +} + +func newRecordingFactory(factory credentialFactory) *recordingFactory { + return &recordingFactory{factory: factory} +} + +func (r *recordingFactory) build(spec credentialSpec) (azcore.TokenCredential, error) { + r.observed = append(r.observed, spec) + return r.factory(spec) +} + +// allAuthenticationTypes is the complete supported set. Tests iterate it so a +// newly added mode fails the exhaustiveness assertions until it is covered. +var allAuthenticationTypes = []AuthenticationType{ + AuthenticationNone, + AuthenticationDefaultAzure, + AuthenticationManagedIdentity, + AuthenticationWorkloadIdentity, + AuthenticationEnvironment, + AuthenticationAzureCLI, + AuthenticationAzurePowerShell, + AuthenticationInteractiveBrowser, + AuthenticationTokenCredential, +} + +// TestCredentialSpecForEveryAuthenticationMode pins the exact Azure Identity +// inputs each mode consumes when every identity field is populated. +func TestCredentialSpecForEveryAuthenticationMode(t *testing.T) { + tests := []struct { + authentication AuthenticationType + want credentialSpec + }{ + { + authentication: AuthenticationNone, + want: credentialSpec{authentication: AuthenticationNone}, + }, + { + authentication: AuthenticationDefaultAzure, + want: credentialSpec{ + authentication: AuthenticationDefaultAzure, + tenantID: "tenant", + additionallyAllowedTenants: []string{"one", "two"}, + }, + }, + { + authentication: AuthenticationManagedIdentity, + want: credentialSpec{ + authentication: AuthenticationManagedIdentity, + clientID: "client", + }, + }, + { + authentication: AuthenticationWorkloadIdentity, + want: credentialSpec{ + authentication: AuthenticationWorkloadIdentity, + clientID: "client", + tenantID: "tenant", + tokenFilePath: "/token", + additionallyAllowedTenants: []string{"one", "two"}, + }, + }, + { + authentication: AuthenticationEnvironment, + want: credentialSpec{authentication: AuthenticationEnvironment}, + }, + { + authentication: AuthenticationAzureCLI, + want: credentialSpec{ + authentication: AuthenticationAzureCLI, + tenantID: "tenant", + additionallyAllowedTenants: []string{"one", "two"}, + }, + }, + { + authentication: AuthenticationAzurePowerShell, + want: credentialSpec{ + authentication: AuthenticationAzurePowerShell, + tenantID: "tenant", + additionallyAllowedTenants: []string{"one", "two"}, + }, + }, + { + authentication: AuthenticationInteractiveBrowser, + want: credentialSpec{ + authentication: AuthenticationInteractiveBrowser, + clientID: "client", + tenantID: "tenant", + additionallyAllowedTenants: []string{"one", "two"}, + }, + }, + { + authentication: AuthenticationTokenCredential, + want: credentialSpec{authentication: AuthenticationTokenCredential}, + }, + } + + covered := make(map[AuthenticationType]bool, len(tests)) + for _, tt := range tests { + t.Run(string(tt.authentication), func(t *testing.T) { + covered[tt.authentication] = true + // Untrimmed values prove the spec normalizes before construction. + options := &Options{ + Authentication: tt.authentication, + ClientID: " client ", + TenantID: "\ttenant\t", + TokenFilePath: " /token ", + AdditionallyAllowedTenants: []string{" one ", "", " ", "two"}, + } + spec, err := newCredentialSpec(options) + require.NoError(t, err) + require.Equal(t, tt.want, spec) + }) + } + for _, authentication := range allAuthenticationTypes { + require.Truef(t, covered[authentication], "authentication %q is not covered", authentication) + } +} + +func TestCredentialSpecOmitsUnsetFields(t *testing.T) { + for _, authentication := range allAuthenticationTypes { + t.Run(string(authentication), func(t *testing.T) { + spec, err := newCredentialSpec(&Options{Authentication: authentication}) + require.NoError(t, err) + require.Equal(t, credentialSpec{authentication: authentication}, spec) + require.Nil(t, spec.additionallyAllowedTenants) + }) + } +} + +func TestCredentialSpecRejectsUnknownAuthentication(t *testing.T) { + for _, value := range []AuthenticationType{"", "VisualStudio", "defaultazure"} { + t.Run(string(value), func(t *testing.T) { + _, err := newCredentialSpec(&Options{Authentication: value}) + require.ErrorContains(t, err, "unsupported DTS authentication type") + }) + } +} + +func TestNormalizeAdditionallyAllowedTenants(t *testing.T) { + require.Nil(t, normalizeAdditionallyAllowedTenants(nil)) + require.Nil(t, normalizeAdditionallyAllowedTenants([]string{"", " ", "\t"})) + require.Equal( + t, + []string{"one", "two", "*"}, + normalizeAdditionallyAllowedTenants([]string{" one ", "", "two", " * "}), + ) +} + +// TestResolveCredentialDispatchesEveryAzureIdentityMode asserts each mode that +// builds an Azure Identity credential reaches the factory with its own spec. +func TestResolveCredentialDispatchesEveryAzureIdentityMode(t *testing.T) { + azureIdentityModes := []AuthenticationType{ + AuthenticationDefaultAzure, + AuthenticationManagedIdentity, + AuthenticationWorkloadIdentity, + AuthenticationEnvironment, + AuthenticationAzureCLI, + AuthenticationAzurePowerShell, + AuthenticationInteractiveBrowser, + } + for _, authentication := range azureIdentityModes { + t.Run(string(authentication), func(t *testing.T) { + built := stubCredential{name: string(authentication)} + factory := newRecordingFactory( + func(credentialSpec) (azcore.TokenCredential, error) { return built, nil }, + ) + options := &Options{ + Authentication: authentication, + ClientID: "client", + TenantID: "tenant", + TokenFilePath: "/token", + AdditionallyAllowedTenants: []string{"one"}, + } + credential, err := resolveCredential(options, factory.build) + require.NoError(t, err) + require.Equal(t, built, credential) + + want, err := newCredentialSpec(options) + require.NoError(t, err) + require.Equal(t, []credentialSpec{want}, factory.observed) + }) + } +} + +func TestResolveCredentialNoneReturnsNilWithoutFactory(t *testing.T) { + factory := newRecordingFactory(func(credentialSpec) (azcore.TokenCredential, error) { + t.Fatal("Authentication None must not construct a credential") + return nil, nil + }) + credential, err := resolveCredential(&Options{Authentication: AuthenticationNone}, factory.build) + require.NoError(t, err) + require.Nil(t, credential) + require.Empty(t, factory.observed) +} + +func TestResolveCredentialTokenCredentialReturnsConfiguredCredential(t *testing.T) { + factory := newRecordingFactory(func(credentialSpec) (azcore.TokenCredential, error) { + t.Fatal("Authentication TokenCredential must not construct a credential") + return nil, nil + }) + configured := stubCredential{name: "explicit"} + credential, err := resolveCredential(&Options{ + Authentication: AuthenticationTokenCredential, + Credential: configured, + }, factory.build) + require.NoError(t, err) + require.Equal(t, configured, credential) + require.Empty(t, factory.observed) +} + +func TestResolveCredentialPropagatesFactoryError(t *testing.T) { + failure := errors.New("credential construction failed") + factory := newRecordingFactory(func(credentialSpec) (azcore.TokenCredential, error) { + return nil, failure + }) + _, err := resolveCredential(&Options{Authentication: AuthenticationAzureCLI}, factory.build) + require.ErrorIs(t, err, failure) +} + +func TestResolveCredentialRejectsUnsupportedAuthentication(t *testing.T) { + _, err := resolveCredential(&Options{Authentication: "Password"}, newAzureIdentityCredential) + require.ErrorContains(t, err, `unsupported DTS authentication type "Password"`) +} + +// TestNewAzureIdentityCredentialRejectsNonIdentityModes covers the production +// factory branches that cannot construct a credential. The remaining branches +// call Azure Identity constructors and are exercised through the seam instead. +func TestNewAzureIdentityCredentialRejectsNonIdentityModes(t *testing.T) { + for _, authentication := range []AuthenticationType{ + AuthenticationNone, + AuthenticationTokenCredential, + "Password", + } { + t.Run(string(authentication), func(t *testing.T) { + _, err := newAzureIdentityCredential(credentialSpec{authentication: authentication}) + require.ErrorContains(t, err, "unsupported DTS authentication type") + }) + } +} + +// TestPrepareOptionsNormalizesResolvedCredential asserts every Azure Identity +// mode collapses to TokenCredential once its credential has been constructed. +func TestPrepareOptionsNormalizesResolvedCredential(t *testing.T) { + for _, authentication := range allAuthenticationTypes { + t.Run(string(authentication), func(t *testing.T) { + built := stubCredential{name: string(authentication)} + options := NewOptions("scheduler.example.com", "hub") + options.Authentication = authentication + switch authentication { + case AuthenticationNone: + options.EndpointAddress = "http://127.0.0.1:8080" + options.AllowInsecureConnection = true + case AuthenticationTokenCredential: + options.Credential = built + } + + prepared, err := prepareOptionsWith( + options, + func(credentialSpec) (azcore.TokenCredential, error) { return built, nil }, + ) + require.NoError(t, err) + if authentication == AuthenticationNone { + require.Equal(t, AuthenticationNone, prepared.Authentication) + require.Nil(t, prepared.Credential) + return + } + require.Equal(t, AuthenticationTokenCredential, prepared.Authentication) + require.Equal(t, built, prepared.Credential) + }) + } +} + +func TestPrepareOptionsPropagatesCredentialFailure(t *testing.T) { + failure := errors.New("no managed identity endpoint") + options := NewOptions("scheduler.example.com", "hub") + options.Authentication = AuthenticationManagedIdentity + _, err := prepareOptionsWith(options, func(credentialSpec) (azcore.TokenCredential, error) { + return nil, failure + }) + require.ErrorIs(t, err, failure) +} + +// TestPrepareOptionsValidatesBeforeConstructingCredential keeps invalid +// configuration from reaching Azure Identity. +func TestPrepareOptionsValidatesBeforeConstructingCredential(t *testing.T) { + options := NewOptions("scheduler.example.com", "") + options.Authentication = AuthenticationAzureCLI + _, err := prepareOptionsWith(options, func(credentialSpec) (azcore.TokenCredential, error) { + t.Fatal("invalid options must not construct a credential") + return nil, nil + }) + require.ErrorContains(t, err, "task hub name is required") +} + +func TestTokenScope(t *testing.T) { + tests := []struct { + resourceID string + want string + }{ + {resourceID: DefaultResourceID, want: "https://durabletask.io/.default"}, + {resourceID: "https://durabletask.io/", want: "https://durabletask.io/.default"}, + {resourceID: "https://durabletask.io///", want: "https://durabletask.io/.default"}, + {resourceID: " https://durabletask.io ", want: "https://durabletask.io/.default"}, + {resourceID: "https://custom.example.com", want: "https://custom.example.com/.default"}, + } + for _, tt := range tests { + t.Run(tt.resourceID, func(t *testing.T) { + require.Equal(t, tt.want, tokenScope(tt.resourceID)) + }) + } +} + +func TestOptionsValidateRejectsBlankOrInjectedResourceID(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.ResourceID = " " + require.ErrorContains(t, options.Validate(), "resource ID cannot be blank") + + options.ResourceID = "https://durabletask.io\r\nx" + require.ErrorContains(t, options.Validate(), "resource ID cannot contain newlines") +} + +// TestOptionsValidateCredentialPresencePerMode covers the credential guard for +// every supported authentication mode plus the unset default. +func TestOptionsValidateCredentialPresencePerMode(t *testing.T) { + tests := []struct { + name string + authentication AuthenticationType + credential azcore.TokenCredential + wantErr string + }{ + {name: "unset defaults to DefaultAzure", authentication: ""}, + { + name: "unset rejects credential", + authentication: "", + credential: stubCredential{}, + wantErr: "use Authentication TokenCredential", + }, + {name: "DefaultAzure", authentication: AuthenticationDefaultAzure}, + { + name: "DefaultAzure with credential", + authentication: AuthenticationDefaultAzure, + credential: stubCredential{}, + wantErr: "use Authentication TokenCredential", + }, + {name: "ManagedIdentity", authentication: AuthenticationManagedIdentity}, + { + name: "ManagedIdentity with credential", + authentication: AuthenticationManagedIdentity, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is ManagedIdentity", + }, + {name: "WorkloadIdentity", authentication: AuthenticationWorkloadIdentity}, + { + name: "WorkloadIdentity with credential", + authentication: AuthenticationWorkloadIdentity, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is WorkloadIdentity", + }, + {name: "Environment", authentication: AuthenticationEnvironment}, + { + name: "Environment with credential", + authentication: AuthenticationEnvironment, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is Environment", + }, + {name: "AzureCLI", authentication: AuthenticationAzureCLI}, + { + name: "AzureCLI with credential", + authentication: AuthenticationAzureCLI, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is AzureCLI", + }, + {name: "AzurePowerShell", authentication: AuthenticationAzurePowerShell}, + { + name: "AzurePowerShell with credential", + authentication: AuthenticationAzurePowerShell, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is AzurePowerShell", + }, + {name: "InteractiveBrowser", authentication: AuthenticationInteractiveBrowser}, + { + name: "InteractiveBrowser with credential", + authentication: AuthenticationInteractiveBrowser, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is InteractiveBrowser", + }, + { + name: "TokenCredential", + authentication: AuthenticationTokenCredential, + credential: stubCredential{}, + }, + { + name: "TokenCredential without credential", + authentication: AuthenticationTokenCredential, + wantErr: "TokenCredential authentication requires a credential", + }, + { + name: "None rejects credential", + authentication: AuthenticationNone, + credential: stubCredential{}, + wantErr: "credential must be nil when Authentication is None", + }, + { + name: "unknown authentication", + authentication: "Password", + wantErr: `unsupported DTS authentication type "Password"`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.Authentication = tt.authentication + options.Credential = tt.credential + err := options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// TestOptionsValidateIdentityFieldCombinations documents which modes accept +// tenant, client, token-file, and additional-tenant configuration. +func TestOptionsValidateIdentityFieldCombinations(t *testing.T) { + type identityFields struct { + clientID string + tenantID string + tokenFilePath string + tenants []string + } + all := identityFields{ + clientID: "client", + tenantID: "tenant", + tokenFilePath: "/token", + tenants: []string{"one"}, + } + + tests := []struct { + name string + authentication AuthenticationType + fields identityFields + wantErr string + }{ + {name: "DefaultAzure accepts all", authentication: AuthenticationDefaultAzure, fields: all}, + {name: "ManagedIdentity accepts all", authentication: AuthenticationManagedIdentity, fields: all}, + {name: "WorkloadIdentity accepts all", authentication: AuthenticationWorkloadIdentity, fields: all}, + {name: "Environment accepts all", authentication: AuthenticationEnvironment, fields: all}, + {name: "AzureCLI accepts all", authentication: AuthenticationAzureCLI, fields: all}, + {name: "AzurePowerShell accepts all", authentication: AuthenticationAzurePowerShell, fields: all}, + {name: "InteractiveBrowser accepts all", authentication: AuthenticationInteractiveBrowser, fields: all}, + { + name: "None rejects ClientID", + authentication: AuthenticationNone, + fields: identityFields{clientID: "client"}, + wantErr: "Authentication None does not use ClientID", + }, + { + name: "None rejects TenantID", + authentication: AuthenticationNone, + fields: identityFields{tenantID: "tenant"}, + wantErr: "Authentication None does not use TenantID", + }, + { + name: "None rejects TokenFilePath", + authentication: AuthenticationNone, + fields: identityFields{tokenFilePath: "/token"}, + wantErr: "Authentication None does not use TokenFilePath", + }, + { + name: "None rejects AdditionallyAllowedTenants", + authentication: AuthenticationNone, + fields: identityFields{tenants: []string{"one"}}, + wantErr: "Authentication None does not use AdditionallyAllowedTenants", + }, + { + name: "None reports every unusable field", + authentication: AuthenticationNone, + fields: all, + wantErr: "ClientID, TenantID, TokenFilePath, AdditionallyAllowedTenants", + }, + { + name: "None ignores blank fields", + authentication: AuthenticationNone, + fields: identityFields{clientID: " ", tenantID: "\t", tenants: []string{"", " "}}, + }, + { + name: "TokenCredential rejects identity fields", + authentication: AuthenticationTokenCredential, + fields: all, + wantErr: "Authentication TokenCredential does not use ClientID, TenantID", + }, + { + name: "TokenCredential ignores blank fields", + authentication: AuthenticationTokenCredential, + fields: identityFields{tenants: []string{""}}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.Authentication = tt.authentication + options.ClientID = tt.fields.clientID + options.TenantID = tt.fields.tenantID + options.TokenFilePath = tt.fields.tokenFilePath + options.AdditionallyAllowedTenants = tt.fields.tenants + switch tt.authentication { + case AuthenticationNone: + options.EndpointAddress = "http://127.0.0.1:8080" + options.AllowInsecureConnection = true + case AuthenticationTokenCredential: + options.Credential = stubCredential{} + } + + err := options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// TestOptionsValidatePlaintextAndCredentialGuards covers every scheme, +// insecure-opt-in, and authentication combination. +func TestOptionsValidatePlaintextAndCredentialGuards(t *testing.T) { + tests := []struct { + name string + endpoint string + allowInsecure bool + authentication AuthenticationType + wantErr string + }{ + { + name: "https with credentials", + endpoint: "https://scheduler.example.com", + authentication: AuthenticationDefaultAzure, + }, + { + name: "implicit https with credentials", + endpoint: "scheduler.example.com", + authentication: AuthenticationDefaultAzure, + }, + { + name: "https with None", + endpoint: "https://scheduler.example.com", + authentication: AuthenticationNone, + }, + { + name: "https allows the insecure opt-in", + endpoint: "https://scheduler.example.com", + allowInsecure: true, + authentication: AuthenticationDefaultAzure, + }, + { + name: "http without opt-in", + endpoint: "http://127.0.0.1:8080", + authentication: AuthenticationNone, + wantErr: "requires AllowInsecureConnection", + }, + { + name: "http with opt-in and None", + endpoint: "http://127.0.0.1:8080", + allowInsecure: true, + authentication: AuthenticationNone, + }, + { + name: "http with opt-in and credentials", + endpoint: "http://127.0.0.1:8080", + allowInsecure: true, + authentication: AuthenticationDefaultAzure, + wantErr: "cannot be used with credentials", + }, + { + name: "http with opt-in and AzureCLI", + endpoint: "http://127.0.0.1:8080", + allowInsecure: true, + authentication: AuthenticationAzureCLI, + wantErr: "cannot be used with credentials", + }, + { + name: "http with opt-in and default authentication", + endpoint: "http://127.0.0.1:8080", + allowInsecure: true, + wantErr: "cannot be used with credentials", + }, + { + name: "http without opt-in reports the opt-in first", + endpoint: "http://127.0.0.1:8080", + authentication: AuthenticationDefaultAzure, + wantErr: "requires AllowInsecureConnection", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := NewOptions(tt.endpoint, "hub") + options.Authentication = tt.authentication + options.AllowInsecureConnection = tt.allowInsecure + err := options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + return + } + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// TestConnectRejectsCredentialsOnPlaintextTransport is the transport-level +// backstop for the Validate guard: gRPC refuses per-RPC credentials that +// require transport security on an insecure channel. +func TestConnectRejectsCredentialsOnPlaintextTransport(t *testing.T) { + options := NewOptions("http://127.0.0.1:8080", "hub") + options.Authentication = AuthenticationTokenCredential + options.Credential = stubCredential{} + options.AllowInsecureConnection = true + + connection, err := connect(options, clientRole, "") + require.Error(t, err) + require.Nil(t, connection) +} + +func TestNewOptionsWithCredentialUsesTokenCredentialMode(t *testing.T) { + credential := stubCredential{name: "explicit"} + options := NewOptionsWithCredential("scheduler.example.com", "hub", credential) + require.Equal(t, AuthenticationTokenCredential, options.Authentication) + require.Equal(t, credential, options.Credential) + require.Equal(t, DefaultResourceID, options.ResourceID) + require.Equal(t, 30*time.Second, options.HelloTimeout) + require.NoError(t, options.Validate()) + + nilCredential := NewOptionsWithCredential("scheduler.example.com", "hub", nil) + require.ErrorContains(t, nilCredential.Validate(), "requires a credential") +} + +// TestRetryServiceConfigDefaults pins the client channel retry defaults. +func TestRetryServiceConfigDefaults(t *testing.T) { + var parsed struct { + MethodConfig []struct { + Name []map[string]any `json:"name"` + RetryPolicy struct { + MaxAttempts int `json:"maxAttempts"` + InitialBackoff string `json:"initialBackoff"` + MaxBackoff string `json:"maxBackoff"` + BackoffMultiplier float64 `json:"backoffMultiplier"` + RetryableStatusCodes []string `json:"retryableStatusCodes"` + } `json:"retryPolicy"` + } `json:"methodConfig"` + } + require.NoError(t, json.Unmarshal([]byte(retryServiceConfig), &parsed)) + require.Len(t, parsed.MethodConfig, 1) + method := parsed.MethodConfig[0] + require.Equal(t, []map[string]any{{}}, method.Name) + require.Equal(t, 5, method.RetryPolicy.MaxAttempts) + require.Equal(t, "0.050s", method.RetryPolicy.InitialBackoff) + require.Equal(t, "0.250s", method.RetryPolicy.MaxBackoff) + require.InDelta(t, 2.0, method.RetryPolicy.BackoffMultiplier, 0) + require.Equal(t, []string{"UNAVAILABLE"}, method.RetryPolicy.RetryableStatusCodes) +} + +// azureIdentityEnvironmentVariables are neutralized so credential-construction +// tests do not depend on ambient developer or CI configuration. +var azureIdentityEnvironmentVariables = []string{ + "AZURE_TENANT_ID", + "AZURE_CLIENT_ID", + "AZURE_CLIENT_SECRET", + "AZURE_CLIENT_CERTIFICATE_PATH", + "AZURE_CLIENT_CERTIFICATE_PASSWORD", + "AZURE_USERNAME", + "AZURE_PASSWORD", + "AZURE_FEDERATED_TOKEN_FILE", + "AZURE_ADDITIONALLY_ALLOWED_TENANTS", + "AZURE_AUTHORITY_HOST", + "AZURE_TOKEN_CREDENTIALS", +} + +func clearAzureIdentityEnvironment(t *testing.T) { + t.Helper() + for _, name := range azureIdentityEnvironmentVariables { + t.Setenv(name, "") + require.NoError(t, os.Unsetenv(name)) + } +} + +// TestNewAzureIdentityCredentialConstructsOffline exercises the production +// factory for the modes whose Azure Identity constructors do no network I/O. +// ManagedIdentity and DefaultAzure are covered separately because their +// construction depends on the hosting environment. +func TestNewAzureIdentityCredentialConstructsOffline(t *testing.T) { + tests := []struct { + name string + spec credentialSpec + wantErr string + }{ + { + name: "WorkloadIdentity with explicit configuration", + spec: credentialSpec{ + authentication: AuthenticationWorkloadIdentity, + clientID: "00000000-0000-0000-0000-000000000001", + tenantID: "00000000-0000-0000-0000-000000000002", + tokenFilePath: "/var/run/secrets/token", + additionallyAllowedTenants: []string{"*"}, + }, + }, + { + name: "WorkloadIdentity without configuration", + spec: credentialSpec{authentication: AuthenticationWorkloadIdentity}, + wantErr: "WorkloadIdentityCredential", + }, + { + name: "AzureCLI", + spec: credentialSpec{ + authentication: AuthenticationAzureCLI, + tenantID: "00000000-0000-0000-0000-000000000002", + additionallyAllowedTenants: []string{"*"}, + }, + }, + { + name: "AzurePowerShell", + spec: credentialSpec{ + authentication: AuthenticationAzurePowerShell, + tenantID: "00000000-0000-0000-0000-000000000002", + additionallyAllowedTenants: []string{"*"}, + }, + }, + { + name: "InteractiveBrowser", + spec: credentialSpec{ + authentication: AuthenticationInteractiveBrowser, + clientID: "00000000-0000-0000-0000-000000000001", + tenantID: "00000000-0000-0000-0000-000000000002", + additionallyAllowedTenants: []string{"*"}, + }, + }, + { + name: "Environment without environment variables", + spec: credentialSpec{authentication: AuthenticationEnvironment}, + wantErr: "EnvironmentCredential", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clearAzureIdentityEnvironment(t) + credential, err := newAzureIdentityCredential(tt.spec) + if tt.wantErr != "" { + require.ErrorContains(t, err, tt.wantErr) + require.Nil(t, credential) + return + } + require.NoError(t, err) + require.NotNil(t, credential) + }) + } +} + +// TestNewAzureIdentityCredentialEnvironmentModeIgnoresSpecFields confirms +// EnvironmentCredential is configured only by environment variables. +func TestNewAzureIdentityCredentialEnvironmentModeIgnoresSpecFields(t *testing.T) { + clearAzureIdentityEnvironment(t) + _, err := newAzureIdentityCredential(credentialSpec{ + authentication: AuthenticationEnvironment, + clientID: "00000000-0000-0000-0000-000000000001", + tenantID: "00000000-0000-0000-0000-000000000002", + }) + require.ErrorContains(t, err, "EnvironmentCredential") + + t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000002") + t.Setenv("AZURE_CLIENT_ID", "00000000-0000-0000-0000-000000000001") + t.Setenv("AZURE_CLIENT_SECRET", "not-a-real-secret") + credential, err := newAzureIdentityCredential( + credentialSpec{authentication: AuthenticationEnvironment}, + ) + require.NoError(t, err) + require.NotNil(t, credential) +} + +// TestNewAzureIdentityCredentialWorkloadIdentityUsesEnvironmentFallback matches +// Azure Identity behavior where unset fields fall back to the webhook-injected +// environment variables. +func TestNewAzureIdentityCredentialWorkloadIdentityUsesEnvironmentFallback(t *testing.T) { + clearAzureIdentityEnvironment(t) + t.Setenv("AZURE_TENANT_ID", "00000000-0000-0000-0000-000000000002") + t.Setenv("AZURE_CLIENT_ID", "00000000-0000-0000-0000-000000000001") + t.Setenv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/secrets/token") + + credential, err := newAzureIdentityCredential( + credentialSpec{authentication: AuthenticationWorkloadIdentity}, + ) + require.NoError(t, err) + require.NotNil(t, credential) +} + +// TestNewAzureIdentityCredentialDefaultAzureAcceptsTenantScoping covers the +// DefaultAzure branch, including the tenant fields that were previously dropped. +func TestNewAzureIdentityCredentialDefaultAzureAcceptsTenantScoping(t *testing.T) { + clearAzureIdentityEnvironment(t) + credential, err := newAzureIdentityCredential(credentialSpec{ + authentication: AuthenticationDefaultAzure, + tenantID: "00000000-0000-0000-0000-000000000002", + additionallyAllowedTenants: []string{"*"}, + }) + require.NoError(t, err) + require.NotNil(t, credential) +} diff --git a/durabletaskscheduler/client.go b/durabletaskscheduler/client.go new file mode 100644 index 00000000..c8bc7520 --- /dev/null +++ b/durabletaskscheduler/client.go @@ -0,0 +1,87 @@ +package durabletaskscheduler + +import ( + "context" + "errors" + "fmt" + "sync" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/emptypb" +) + +// Client owns the management connection it creates. +type Client struct { + *durabletaskclient.TaskHubGrpcClient + + connection *recreatingClientConn + closeOnce sync.Once + closeErr error + converter api.DataConverter +} + +// NewClient creates an independently owned management connection and validates +// it with a deadline-bound Hello call. +// A nil logger uses [api.DefaultLogger]. +func NewClient(ctx context.Context, options *Options, logger api.Logger) (*Client, error) { + if logger == nil { + logger = api.DefaultLogger() + } + prepared, err := prepareOptions(options) + if err != nil { + return nil, err + } + factory := func(ctx context.Context, _ *clientTransport) (*clientTransport, error) { + connection, err := connect(&prepared, clientRole, "") + if err != nil { + return nil, err + } + helloCtx, cancel := context.WithTimeout(ctx, prepared.HelloTimeout) + _, err = protos.NewTaskHubSidecarServiceClient(connection).Hello(helloCtx, &emptypb.Empty{}) + cancel() + if err != nil { + _ = connection.Close() + return nil, fmt.Errorf("DTS client Hello failed: %w", err) + } + return &clientTransport{connection: connection, closer: connection}, nil + } + initial, err := factory(ctx, nil) + if err != nil { + return nil, err + } + connection := newRecreatingClientConn( + initial, + factory, + prepared.ChannelRecreateFailureThreshold, + prepared.ChannelRecreateMinInterval, + logger, + ) + clientOptions := []durabletaskclient.TaskHubGrpcClientOption{ + durabletaskclient.WithLargePayloads(prepared.LargePayloads), + durabletaskclient.WithDataConverter(prepared.DataConverter), + } + if prepared.Versioning != nil && prepared.Versioning.DefaultVersion != "" { + clientOptions = append(clientOptions, durabletaskclient.WithDefaultVersion(prepared.Versioning.DefaultVersion)) + } + return &Client{ + TaskHubGrpcClient: durabletaskclient.NewTaskHubGrpcClient(connection, logger, clientOptions...), + connection: connection, + converter: api.NormalizeDataConverter(prepared.DataConverter), + }, nil +} + +func (c *Client) Close() error { + if c == nil { + return nil + } + c.closeOnce.Do(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + stopErr := c.StopWorkItemListener(shutdownCtx) + cancel() + c.closeErr = errors.Join(stopErr, c.connection.Close()) + }) + return c.closeErr +} diff --git a/durabletaskscheduler/client_connection.go b/durabletaskscheduler/client_connection.go new file mode 100644 index 00000000..1e20414f --- /dev/null +++ b/durabletaskscheduler/client_connection.go @@ -0,0 +1,301 @@ +package durabletaskscheduler + +import ( + "context" + "errors" + "io" + "sync" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const clientChannelRecreateTimeout = 30 * time.Second + +type clientTransport struct { + connection grpc.ClientConnInterface + closer io.Closer + inFlight int + retired bool + closed bool +} + +type clientTransportFactory func(context.Context, *clientTransport) (*clientTransport, error) + +type recreatingClientConn struct { + mu sync.Mutex + current *clientTransport + transports map[*clientTransport]struct{} + factory clientTransportFactory + failureThreshold int + minRecreateInterval time.Duration + consecutiveFailures int + lastRecreateAttempt time.Time + recreateInFlight bool + recreateContext context.Context + cancelRecreate context.CancelFunc + recreateWait sync.WaitGroup + logger api.Logger + closed bool + closeOnce sync.Once + closeErr error +} + +func newRecreatingClientConn( + initial *clientTransport, + factory clientTransportFactory, + failureThreshold int, + minRecreateInterval time.Duration, + logger api.Logger, +) *recreatingClientConn { + recreateContext, cancelRecreate := context.WithCancel(context.Background()) + return &recreatingClientConn{ + current: initial, + transports: map[*clientTransport]struct{}{initial: {}}, + factory: factory, + failureThreshold: failureThreshold, + minRecreateInterval: minRecreateInterval, + recreateContext: recreateContext, + cancelRecreate: cancelRecreate, + logger: logger, + } +} + +func (c *recreatingClientConn) Invoke( + ctx context.Context, + method string, + args any, + reply any, + opts ...grpc.CallOption, +) error { + transport, err := c.acquire() + if err != nil { + return err + } + err = transport.connection.Invoke(ctx, method, args, reply, opts...) + c.release(transport) + c.recordOutcome(ctx, method, transport, err) + return err +} + +func (c *recreatingClientConn) NewStream( + ctx context.Context, + desc *grpc.StreamDesc, + method string, + opts ...grpc.CallOption, +) (grpc.ClientStream, error) { + transport, err := c.acquire() + if err != nil { + return nil, err + } + var finishOnce sync.Once + finish := func(callErr error) { + finishOnce.Do(func() { + c.release(transport) + c.recordOutcome(ctx, method, transport, callErr) + }) + } + opts = append(opts, grpc.OnFinish(finish)) + stream, err := transport.connection.NewStream(ctx, desc, method, opts...) + if err != nil { + finish(err) + } + return stream, err +} + +func (c *recreatingClientConn) Close() error { + if c == nil { + return nil + } + c.closeOnce.Do(func() { + c.mu.Lock() + c.closed = true + transports := c.takeAllTransportsLocked() + c.mu.Unlock() + + c.cancelRecreate() + c.closeErr = closeClientTransports(transports) + c.recreateWait.Wait() + }) + return c.closeErr +} + +func (c *recreatingClientConn) acquire() (*clientTransport, error) { + c.mu.Lock() + defer c.mu.Unlock() + if c.closed || c.current == nil { + return nil, status.Error(codes.Unavailable, "DTS management client is closed") + } + c.current.inFlight++ + return c.current, nil +} + +func (c *recreatingClientConn) release(transport *clientTransport) { + var closer io.Closer + c.mu.Lock() + transport.inFlight-- + if transport.inFlight < 0 { + c.mu.Unlock() + panic("DTS client transport lease released more than once") + } + if transport.retired && transport.inFlight == 0 && !transport.closed { + transport.closed = true + delete(c.transports, transport) + closer = transport.closer + } + c.mu.Unlock() + if closer != nil { + if err := closer.Close(); err != nil { + c.logger.Warnf("failed to close retired DTS management channel: %v", err) + } + } +} + +func (c *recreatingClientConn) recordOutcome( + ctx context.Context, + method string, + transport *clientTransport, + err error, +) { + c.mu.Lock() + if c.closed || transport != c.current { + c.mu.Unlock() + return + } + if err == nil { + c.consecutiveFailures = 0 + c.mu.Unlock() + return + } + if isNeutralChannelOutcome(ctx, method, err) { + c.mu.Unlock() + return + } + if !countsTowardChannelRecreation(err) { + c.consecutiveFailures = 0 + c.mu.Unlock() + return + } + + c.consecutiveFailures++ + failureCount := c.consecutiveFailures + if c.failureThreshold <= 0 || + failureCount < c.failureThreshold || + c.recreateInFlight || + !c.recreateIntervalElapsedLocked() { + c.mu.Unlock() + return + } + c.recreateInFlight = true + previous := c.current + c.recreateWait.Add(1) + c.mu.Unlock() + + c.logger.Warnf( + "recreating DTS management channel after %d consecutive transport failures", + failureCount, + ) + go c.recreate(previous) +} + +func (c *recreatingClientConn) recreate(previous *clientTransport) { + defer c.recreateWait.Done() + ctx, cancel := context.WithTimeout(c.recreateContext, clientChannelRecreateTimeout) + replacement, err := c.factory(ctx, previous) + cancel() + + var closer io.Closer + c.mu.Lock() + c.recreateInFlight = false + c.lastRecreateAttempt = time.Now() + switch { + case err != nil: + c.mu.Unlock() + if c.recreateContext.Err() == nil { + c.logger.Errorf("failed to recreate DTS management channel: %v", err) + } + return + case replacement == nil || replacement.connection == nil: + c.mu.Unlock() + c.logger.Errorf("failed to recreate DTS management channel: factory returned no connection") + return + case c.closed: + if replacement != previous && !replacement.closed { + replacement.closed = true + closer = replacement.closer + } + c.mu.Unlock() + if closer != nil { + _ = closer.Close() + } + return + case replacement == previous: + c.consecutiveFailures = 0 + c.mu.Unlock() + return + default: + c.current = replacement + c.transports[replacement] = struct{}{} + c.consecutiveFailures = 0 + previous.retired = true + if previous.inFlight == 0 && !previous.closed { + previous.closed = true + delete(c.transports, previous) + closer = previous.closer + } + c.mu.Unlock() + if closer != nil { + if closeErr := closer.Close(); closeErr != nil { + c.logger.Warnf("failed to close replaced DTS management channel: %v", closeErr) + } + } + } +} + +func (c *recreatingClientConn) recreateIntervalElapsedLocked() bool { + return c.lastRecreateAttempt.IsZero() || + time.Since(c.lastRecreateAttempt) >= c.minRecreateInterval +} + +func (c *recreatingClientConn) takeAllTransportsLocked() []*clientTransport { + transports := make([]*clientTransport, 0, len(c.transports)) + for transport := range c.transports { + if transport.closed { + continue + } + transport.closed = true + transport.retired = true + transports = append(transports, transport) + } + clear(c.transports) + c.current = nil + return transports +} + +func closeClientTransports(transports []*clientTransport) error { + var errs []error + for _, transport := range transports { + if transport.closer != nil { + errs = append(errs, transport.closer.Close()) + } + } + return errors.Join(errs...) +} + +func isNeutralChannelOutcome(ctx context.Context, method string, err error) bool { + if ctx.Err() != nil { + return true + } + return status.Code(err) == codes.DeadlineExceeded && + (method == protos.TaskHubSidecarService_WaitForInstanceStart_FullMethodName || + method == protos.TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName) +} + +func countsTowardChannelRecreation(err error) bool { + code := status.Code(err) + return code == codes.Unavailable || code == codes.DeadlineExceeded +} diff --git a/durabletaskscheduler/client_connection_test.go b/durabletaskscheduler/client_connection_test.go new file mode 100644 index 00000000..a3cf383e --- /dev/null +++ b/durabletaskscheduler/client_connection_test.go @@ -0,0 +1,576 @@ +package durabletaskscheduler + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +type fakeClientConnection struct { + invoke func(context.Context, string) error + newStream func(context.Context, string, ...grpc.CallOption) (grpc.ClientStream, error) + closeCount atomic.Int32 +} + +type closeRecorder struct { + close func() error + closeCount atomic.Int32 +} + +func (c *closeRecorder) Close() error { + c.closeCount.Add(1) + return c.close() +} + +func (c *fakeClientConnection) Invoke( + ctx context.Context, + method string, + _ any, + _ any, + _ ...grpc.CallOption, +) error { + if c.invoke == nil { + return nil + } + return c.invoke(ctx, method) +} + +func (c *fakeClientConnection) NewStream( + ctx context.Context, + _ *grpc.StreamDesc, + method string, + options ...grpc.CallOption, +) (grpc.ClientStream, error) { + if c.newStream != nil { + return c.newStream(ctx, method, options...) + } + return nil, errors.New("streaming is not configured") +} + +func (c *fakeClientConnection) Close() error { + c.closeCount.Add(1) + return nil +} + +func fakeTransport(connection *fakeClientConnection) *clientTransport { + return &clientTransport{connection: connection, closer: connection} +} + +func newTestRecreatingClientConn( + t *testing.T, + initial *clientTransport, + factory clientTransportFactory, + failureThreshold int, + minRecreateInterval time.Duration, +) *recreatingClientConn { + t.Helper() + connection := newRecreatingClientConn( + initial, + factory, + failureThreshold, + minRecreateInterval, + api.DefaultLogger(), + ) + t.Cleanup(func() { + require.NoError(t, connection.Close()) + }) + return connection +} + +func TestRecreatingClientConnReplacesChannelAfterConsecutiveFailures(t *testing.T) { + initialConnection := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.Unavailable, "initial unavailable") + }, + } + replacementConnection := &fakeClientConnection{} + initial := fakeTransport(initialConnection) + replacement := fakeTransport(replacementConnection) + recreateStarted := make(chan struct{}, 1) + recreateRelease := make(chan struct{}) + releaseRecreation := sync.OnceFunc(func() { + close(recreateRelease) + }) + defer releaseRecreation() + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + initial, + func(context.Context, *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + recreateStarted <- struct{}{} + <-recreateRelease + return replacement, nil + }, + 2, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + select { + case <-recreateStarted: + case <-time.After(time.Second): + t.Fatal("channel recreation was not requested") + } + for range 3 { + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + } + require.Equal(t, int32(1), recreateCalls.Load()) + releaseRecreation() + require.Eventually(t, func() bool { + connection.mu.Lock() + defer connection.mu.Unlock() + return connection.current == replacement + }, time.Second, time.Millisecond) + require.NoError(t, connection.Invoke(context.Background(), "/test/success", nil, nil)) + require.Eventually(t, func() bool { + return initialConnection.closeCount.Load() == 1 + }, time.Second, time.Millisecond) +} + +func TestRecreatingClientConnSameChannelNoOpAndCooldown(t *testing.T) { + connectionImpl := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.Unavailable, "unavailable") + }, + } + initial := fakeTransport(connectionImpl) + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + initial, + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 1, + time.Hour, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Eventually(t, func() bool { + connection.mu.Lock() + defer connection.mu.Unlock() + return !connection.recreateInFlight && connection.consecutiveFailures == 0 + }, time.Second, time.Millisecond) + require.Same(t, initial, connection.current) + require.Zero(t, connectionImpl.closeCount.Load()) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Never(t, func() bool { + return recreateCalls.Load() > 1 + }, 50*time.Millisecond, time.Millisecond) +} + +func TestRecreatingClientConnSuccessAndApplicationErrorResetFailureCount(t *testing.T) { + var callCount atomic.Int32 + connectionImpl := &fakeClientConnection{ + invoke: func(context.Context, string) error { + switch callCount.Add(1) { + case 2: + return nil + case 4: + return status.Error(codes.InvalidArgument, "application error") + default: + return status.Error(codes.Unavailable, "unavailable") + } + }, + } + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 2, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.NoError(t, connection.Invoke(context.Background(), "/test/success", nil, nil)) + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Equal( + t, + codes.InvalidArgument, + status.Code(connection.Invoke(context.Background(), "/test/application-error", nil, nil)), + ) + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Never(t, func() bool { + return recreateCalls.Load() != 0 + }, 50*time.Millisecond, time.Millisecond) + connection.mu.Lock() + defer connection.mu.Unlock() + require.Equal(t, 1, connection.consecutiveFailures) +} + +func TestRecreatingClientConnLongPollDeadlineDoesNotTriggerRecreation(t *testing.T) { + connectionImpl := &fakeClientConnection{ + invoke: func(_ context.Context, method string) error { + if method == "/test/fail" { + return status.Error(codes.Unavailable, "unavailable") + } + return status.Error(codes.DeadlineExceeded, "long poll elapsed") + }, + } + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 2, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + for _, method := range []string{ + protos.TaskHubSidecarService_WaitForInstanceStart_FullMethodName, + protos.TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName, + } { + require.Error(t, connection.Invoke(context.Background(), method, nil, nil)) + } + require.Never(t, func() bool { + return recreateCalls.Load() != 0 + }, 50*time.Millisecond, time.Millisecond) + connection.mu.Lock() + defer connection.mu.Unlock() + require.Equal(t, 1, connection.consecutiveFailures) +} + +func TestRecreatingClientConnRegularDeadlineTriggersRecreation(t *testing.T) { + connectionImpl := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.DeadlineExceeded, "server deadline") + }, + } + recreated := make(chan struct{}, 1) + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreated <- struct{}{} + return previous, nil + }, + 1, + 0, + ) + + require.Equal(t, codes.DeadlineExceeded, status.Code(connection.Invoke( + context.Background(), + protos.TaskHubSidecarService_GetInstance_FullMethodName, + nil, + nil, + ))) + select { + case <-recreated: + case <-time.After(time.Second): + t.Fatal("regular deadline did not trigger channel recreation") + } +} + +func TestRecreatingClientConnCallerDeadlineDoesNotTriggerRecreation(t *testing.T) { + connectionImpl := &fakeClientConnection{ + invoke: func(ctx context.Context, method string) error { + if method == "/test/fail" { + return status.Error(codes.Unavailable, "unavailable") + } + <-ctx.Done() + return status.FromContextError(ctx.Err()).Err() + }, + } + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 2, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond) + defer cancel() + <-ctx.Done() + require.Equal(t, codes.DeadlineExceeded, status.Code(connection.Invoke( + ctx, + protos.TaskHubSidecarService_GetInstance_FullMethodName, + nil, + nil, + ))) + require.Never(t, func() bool { + return recreateCalls.Load() != 0 + }, 50*time.Millisecond, time.Millisecond) + connection.mu.Lock() + defer connection.mu.Unlock() + require.Equal(t, 1, connection.consecutiveFailures) +} + +func TestRecreatingClientConnStreamingOutcomesAffectFailureCount(t *testing.T) { + streamFailure := status.Error(codes.Unavailable, "stream unavailable") + connectionImpl := &fakeClientConnection{ + newStream: func( + _ context.Context, + _ string, + options ...grpc.CallOption, + ) (grpc.ClientStream, error) { + return nil, finishStreamCall(options, streamFailure) + }, + } + recreated := make(chan struct{}, 1) + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreated <- struct{}{} + return previous, nil + }, + 1, + 0, + ) + + _, err := connection.NewStream( + context.Background(), + &grpc.StreamDesc{ServerStreams: true}, + protos.TaskHubSidecarService_StreamInstanceHistory_FullMethodName, + ) + require.Equal(t, codes.Unavailable, status.Code(err)) + select { + case <-recreated: + case <-time.After(time.Second): + t.Fatal("stream failure did not trigger channel recreation") + } +} + +func TestRecreatingClientConnSuccessfulStreamResetsFailureCount(t *testing.T) { + callCount := 0 + connectionImpl := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.Unavailable, "unavailable") + }, + newStream: func( + _ context.Context, + _ string, + options ...grpc.CallOption, + ) (grpc.ClientStream, error) { + callCount++ + _ = finishStreamCall(options, nil) + return nil, nil + }, + } + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 2, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + connection.mu.Lock() + require.Equal(t, 1, connection.consecutiveFailures) + connection.mu.Unlock() + stream, err := connection.NewStream( + context.Background(), + &grpc.StreamDesc{ServerStreams: true}, + protos.TaskHubSidecarService_StreamInstanceHistory_FullMethodName, + ) + require.NoError(t, err) + require.Nil(t, stream) + require.Equal(t, 1, callCount) + connection.mu.Lock() + require.Zero(t, connection.consecutiveFailures) + connection.mu.Unlock() + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Never(t, func() bool { + return recreateCalls.Load() != 0 + }, 50*time.Millisecond, time.Millisecond) +} + +func finishStreamCall(options []grpc.CallOption, err error) error { + for _, option := range options { + if onFinish, ok := option.(grpc.OnFinishCallOption); ok { + onFinish.OnFinish(err) + } + } + return err +} + +func TestRecreatingClientConnRequestCancellationDoesNotTriggerRecreation(t *testing.T) { + connectionImpl := &fakeClientConnection{ + invoke: func(ctx context.Context, _ string) error { + <-ctx.Done() + return status.FromContextError(ctx.Err()).Err() + }, + } + var recreateCalls atomic.Int32 + connection := newTestRecreatingClientConn( + t, + fakeTransport(connectionImpl), + func(_ context.Context, previous *clientTransport) (*clientTransport, error) { + recreateCalls.Add(1) + return previous, nil + }, + 1, + 0, + ) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.Equal(t, codes.Canceled, status.Code(connection.Invoke(ctx, "/test/cancel", nil, nil))) + require.Never(t, func() bool { + return recreateCalls.Load() != 0 + }, 50*time.Millisecond, time.Millisecond) +} + +func TestRecreatingClientConnDefersRetiredChannelCloseAndIgnoresItsOutcome(t *testing.T) { + blockingStarted := make(chan struct{}) + releaseBlocking := make(chan struct{}) + var startOnce sync.Once + initialConnection := &fakeClientConnection{ + invoke: func(_ context.Context, method string) error { + if method == "/test/block" { + startOnce.Do(func() { close(blockingStarted) }) + <-releaseBlocking + return nil + } + return status.Error(codes.Unavailable, "unavailable") + }, + } + replacementConnection := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.Unavailable, "replacement unavailable") + }, + } + replacement := fakeTransport(replacementConnection) + connection := newTestRecreatingClientConn( + t, + fakeTransport(initialConnection), + func(context.Context, *clientTransport) (*clientTransport, error) { + return replacement, nil + }, + 1, + time.Hour, + ) + + blockingDone := make(chan error, 1) + go func() { + blockingDone <- connection.Invoke(context.Background(), "/test/block", nil, nil) + }() + <-blockingStarted + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + require.Eventually(t, func() bool { + connection.mu.Lock() + defer connection.mu.Unlock() + return connection.current == replacement + }, time.Second, time.Millisecond) + require.Zero(t, initialConnection.closeCount.Load()) + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + + close(releaseBlocking) + require.NoError(t, <-blockingDone) + connection.mu.Lock() + require.Equal(t, 1, connection.consecutiveFailures) + connection.mu.Unlock() + require.Eventually(t, func() bool { + return initialConnection.closeCount.Load() == 1 + }, time.Second, time.Millisecond) +} + +func TestRecreatingClientConnDefersRetiredChannelCloseUntilStreamCompletes(t *testing.T) { + server := &metadataServer{ + metadata: make(chan metadata.MD, 1), + getErr: status.Error(codes.Unavailable, "replace this channel"), + } + listener, stop := startBufconnServer(t, server) + defer stop() + grpcConnection, err := grpc.NewClient( + "passthrough:///bufconn", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(bufconnDialer(listener)), + ) + require.NoError(t, err) + initialCloser := &closeRecorder{close: grpcConnection.Close} + replacement := fakeTransport(&fakeClientConnection{}) + connection := newTestRecreatingClientConn( + t, + &clientTransport{connection: grpcConnection, closer: initialCloser}, + func(context.Context, *clientTransport) (*clientTransport, error) { + return replacement, nil + }, + 1, + 0, + ) + client := protos.NewTaskHubSidecarServiceClient(connection) + + streamCtx, cancelStream := context.WithCancel(context.Background()) + _, err = client.GetWorkItems(streamCtx, &protos.GetWorkItemsRequest{}) + require.NoError(t, err) + <-server.metadata + + _, err = client.GetInstance(context.Background(), &protos.GetInstanceRequest{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Eventually(t, func() bool { + connection.mu.Lock() + defer connection.mu.Unlock() + return connection.current == replacement + }, time.Second, time.Millisecond) + require.Zero(t, initialCloser.closeCount.Load()) + + cancelStream() + require.Eventually(t, func() bool { + return initialCloser.closeCount.Load() == 1 + }, time.Second, time.Millisecond) +} + +func TestRecreatingClientConnDisposalClosesLateReplacement(t *testing.T) { + initialConnection := &fakeClientConnection{ + invoke: func(context.Context, string) error { + return status.Error(codes.Unavailable, "unavailable") + }, + } + replacementConnection := &fakeClientConnection{} + replacement := fakeTransport(replacementConnection) + recreateStarted := make(chan struct{}) + connection := newTestRecreatingClientConn( + t, + fakeTransport(initialConnection), + func(ctx context.Context, _ *clientTransport) (*clientTransport, error) { + close(recreateStarted) + <-ctx.Done() + return replacement, nil + }, + 1, + 0, + ) + + require.Error(t, connection.Invoke(context.Background(), "/test/fail", nil, nil)) + <-recreateStarted + require.NoError(t, connection.Close()) + require.Equal(t, int32(1), initialConnection.closeCount.Load()) + require.Equal(t, int32(1), replacementConnection.closeCount.Load()) + require.NoError(t, connection.Close()) +} diff --git a/durabletaskscheduler/connection.go b/durabletaskscheduler/connection.go new file mode 100644 index 00000000..081c7e5d --- /dev/null +++ b/durabletaskscheduler/connection.go @@ -0,0 +1,573 @@ +package durabletaskscheduler + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "os" + "runtime/debug" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" +) + +const modulePath = "github.com/microsoft/durabletask-go" + +const ( + accessTokenRefreshBuffer = 5 * time.Minute + accessTokenMinimumCacheDuration = time.Second + accessTokenRefreshRetryDelay = time.Second + accessTokenRefreshTimeout = 5 * time.Minute +) + +type schedulerCredentialState struct { + metadata map[string]string + refreshAfter time.Time + validUntil time.Time +} + +type schedulerCredentialRefresh struct { + done chan struct{} + state *schedulerCredentialState + err error +} + +const retryServiceConfig = `{ + "methodConfig": [{ + "name": [{}], + "retryPolicy": { + "maxAttempts": 5, + "initialBackoff": "0.050s", + "maxBackoff": "0.250s", + "backoffMultiplier": 2, + "retryableStatusCodes": ["UNAVAILABLE"] + } + }] +}` + +type connectionRole string + +const ( + clientRole connectionRole = "DurableTaskClient" + workerRole connectionRole = "DurableTaskWorker" +) + +type schedulerPerRPCCredentials struct { + credential azcore.TokenCredential + scope string + taskHub string + userAgent string + workerID string + + metadataOnce sync.Once + // metadata is immutable after construction; gRPC copies it into each request. + metadata map[string]string + token atomic.Pointer[schedulerCredentialState] + refreshMu sync.Mutex + refresh *schedulerCredentialRefresh + refreshErr error + retryAfter time.Time +} + +func (c *schedulerPerRPCCredentials) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + if c.credential != nil { + state, err := c.getToken(ctx) + if err != nil { + return nil, status.Errorf(codes.Unavailable, "failed to acquire DTS access token: %v", err) + } + return state.metadata, nil + } + return c.baseMetadata(), nil +} + +func (c *schedulerPerRPCCredentials) baseMetadata() map[string]string { + c.metadataOnce.Do(func() { + c.metadata = map[string]string{ + "taskhub": c.taskHub, + "x-user-agent": c.userAgent, + } + if c.workerID != "" { + c.metadata["workerid"] = c.workerID + } + }) + return c.metadata +} + +// getToken keeps the common cached-token path lock-free. The mutex only +// coordinates refresh ownership and waiters when the cached token needs work. +func (c *schedulerPerRPCCredentials) getToken(ctx context.Context) (*schedulerCredentialState, error) { + now := time.Now() + cached := c.token.Load() + if tokenIsFresh(cached, now) { + return cached, nil + } + + c.refreshMu.Lock() + now = time.Now() + cached = c.token.Load() + switch { + case tokenIsFresh(cached, now): + c.refreshMu.Unlock() + return cached, nil + case c.refreshErr != nil && now.Before(c.retryAfter): + err := c.refreshErr + c.refreshMu.Unlock() + if tokenIsValid(cached, now) { + return cached, nil + } + return nil, err + case c.refresh == nil: + if err := ctx.Err(); err != nil { + c.refreshMu.Unlock() + return nil, err + } + refresh := &schedulerCredentialRefresh{done: make(chan struct{})} + c.refresh = refresh + c.refreshMu.Unlock() + go c.refreshToken(context.WithoutCancel(ctx), cached, refresh) + if tokenIsValid(cached, now) { + return cached, nil + } + return waitForCredentialRefresh(ctx, refresh) + case tokenIsValid(cached, now): + c.refreshMu.Unlock() + return cached, nil + } + + refresh := c.refresh + c.refreshMu.Unlock() + return waitForCredentialRefresh(ctx, refresh) +} + +func (c *schedulerPerRPCCredentials) refreshToken( + ctx context.Context, + cached *schedulerCredentialState, + refresh *schedulerCredentialRefresh, +) { + refreshCtx, cancel := context.WithTimeout(ctx, accessTokenRefreshTimeout) + defer cancel() + token, refreshErr := c.credential.GetToken(refreshCtx, policy.TokenRequestOptions{Scopes: []string{c.scope}}) + if refreshErr == nil { + refreshed := c.newCredentialState(token) + if tokenIsValid(refreshed, time.Now()) { + cached = refreshed + } else { + refreshErr = errors.New("DTS credential returned an expired access token") + } + } + err := refreshErr + if refreshErr != nil && tokenIsValid(cached, time.Now()) { + err = nil + } + + c.refreshMu.Lock() + if refreshErr == nil { + c.token.Store(cached) + c.refreshErr = nil + c.retryAfter = time.Time{} + } else { + c.refreshErr = refreshErr + c.retryAfter = time.Now().Add(accessTokenRefreshRetryDelay) + } + refresh.state = cached + refresh.err = err + c.refresh = nil + close(refresh.done) + c.refreshMu.Unlock() +} + +func waitForCredentialRefresh( + ctx context.Context, + refresh *schedulerCredentialRefresh, +) (*schedulerCredentialState, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-refresh.done: + return refresh.state, refresh.err + } +} + +func (c *schedulerPerRPCCredentials) newCredentialState(token azcore.AccessToken) *schedulerCredentialState { + now := time.Now() + validUntil := token.ExpiresOn + if validUntil.IsZero() { + validUntil = now.Add(accessTokenMinimumCacheDuration) + } + refreshAfter := token.RefreshOn + if refreshAfter.IsZero() { + refreshAfter = token.ExpiresOn.Add(-accessTokenRefreshBuffer) + } + minimumRefresh := now.Add(accessTokenMinimumCacheDuration) + if refreshAfter.IsZero() || refreshAfter.Before(minimumRefresh) { + refreshAfter = minimumRefresh + } + if refreshAfter.After(validUntil) { + refreshAfter = validUntil + } + + base := c.baseMetadata() + metadata := make(map[string]string, len(base)+1) + for key, value := range base { + metadata[key] = value + } + metadata["authorization"] = "Bearer " + token.Token + return &schedulerCredentialState{ + metadata: metadata, + refreshAfter: refreshAfter, + validUntil: validUntil, + } +} + +func tokenIsFresh(state *schedulerCredentialState, now time.Time) bool { + return state != nil && state.refreshAfter.After(now) +} + +func tokenIsValid(state *schedulerCredentialState, now time.Time) bool { + return state != nil && state.validUntil.After(now) +} + +func (c *schedulerPerRPCCredentials) RequireTransportSecurity() bool { + return c.credential != nil +} + +// credentialSpec is the normalized set of Azure Identity inputs a single DTS +// authentication mode can actually use. Fields the selected mode cannot use are +// left empty so credential construction is deterministic and reviewable. +type credentialSpec struct { + authentication AuthenticationType + clientID string + tenantID string + tokenFilePath string + additionallyAllowedTenants []string +} + +// credentialFactory builds the Azure Identity credential for a resolved spec. +// Production code always uses newAzureIdentityCredential; it is threaded as a +// parameter rather than a package variable so tests can exercise credential +// selection without contacting Azure and without mutating process-global state. +type credentialFactory func(credentialSpec) (azcore.TokenCredential, error) + +// newCredentialSpec maps options onto the fields the selected authentication +// mode supports in Azure Identity for Go. Values are trimmed, and fields a mode +// cannot consume are dropped rather than forwarded. +func newCredentialSpec(options *Options) (credentialSpec, error) { + clientID := strings.TrimSpace(options.ClientID) + tenantID := strings.TrimSpace(options.TenantID) + tokenFilePath := strings.TrimSpace(options.TokenFilePath) + additionalTenants := normalizeAdditionallyAllowedTenants(options.AdditionallyAllowedTenants) + + spec := credentialSpec{authentication: options.Authentication} + switch options.Authentication { + case AuthenticationNone, AuthenticationTokenCredential: + // No Azure Identity credential is constructed for these modes. + case AuthenticationDefaultAzure: + spec.tenantID = tenantID + spec.additionallyAllowedTenants = additionalTenants + case AuthenticationManagedIdentity: + spec.clientID = clientID + case AuthenticationWorkloadIdentity: + spec.clientID = clientID + spec.tenantID = tenantID + spec.tokenFilePath = tokenFilePath + spec.additionallyAllowedTenants = additionalTenants + case AuthenticationEnvironment: + // EnvironmentCredential is configured entirely by environment variables. + case AuthenticationAzureCLI, AuthenticationAzurePowerShell: + spec.tenantID = tenantID + spec.additionallyAllowedTenants = additionalTenants + case AuthenticationInteractiveBrowser: + spec.clientID = clientID + spec.tenantID = tenantID + spec.additionallyAllowedTenants = additionalTenants + default: + return credentialSpec{}, fmt.Errorf("unsupported DTS authentication type %q", options.Authentication) + } + return spec, nil +} + +func normalizeAdditionallyAllowedTenants(tenants []string) []string { + var normalized []string + for _, tenant := range tenants { + if tenant = strings.TrimSpace(tenant); tenant != "" { + normalized = append(normalized, tenant) + } + } + return normalized +} + +func newAzureIdentityCredential(spec credentialSpec) (azcore.TokenCredential, error) { + switch spec.authentication { + case AuthenticationDefaultAzure: + credential, err := azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ + TenantID: spec.tenantID, + AdditionallyAllowedTenants: spec.additionallyAllowedTenants, + }) + if err != nil { + return nil, fmt.Errorf("failed to create DefaultAzureCredential: %w", err) + } + return credential, nil + case AuthenticationManagedIdentity: + credentialOptions := &azidentity.ManagedIdentityCredentialOptions{} + if spec.clientID != "" { + credentialOptions.ID = azidentity.ClientID(spec.clientID) + } + credential, err := azidentity.NewManagedIdentityCredential(credentialOptions) + if err != nil { + return nil, fmt.Errorf("failed to create ManagedIdentityCredential: %w", err) + } + return credential, nil + case AuthenticationWorkloadIdentity: + credential, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + ClientID: spec.clientID, + TenantID: spec.tenantID, + TokenFilePath: spec.tokenFilePath, + AdditionallyAllowedTenants: spec.additionallyAllowedTenants, + }) + if err != nil { + return nil, fmt.Errorf("failed to create WorkloadIdentityCredential: %w", err) + } + return credential, nil + case AuthenticationEnvironment: + credential, err := azidentity.NewEnvironmentCredential(nil) + if err != nil { + return nil, fmt.Errorf("failed to create EnvironmentCredential: %w", err) + } + return credential, nil + case AuthenticationAzureCLI: + credential, err := azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{ + TenantID: spec.tenantID, + AdditionallyAllowedTenants: spec.additionallyAllowedTenants, + }) + if err != nil { + return nil, fmt.Errorf("failed to create AzureCLICredential: %w", err) + } + return credential, nil + case AuthenticationAzurePowerShell: + credential, err := azidentity.NewAzurePowerShellCredential(&azidentity.AzurePowerShellCredentialOptions{ + TenantID: spec.tenantID, + AdditionallyAllowedTenants: spec.additionallyAllowedTenants, + }) + if err != nil { + return nil, fmt.Errorf("failed to create AzurePowerShellCredential: %w", err) + } + return credential, nil + case AuthenticationInteractiveBrowser: + credential, err := azidentity.NewInteractiveBrowserCredential(&azidentity.InteractiveBrowserCredentialOptions{ + ClientID: spec.clientID, + TenantID: spec.tenantID, + AdditionallyAllowedTenants: spec.additionallyAllowedTenants, + }) + if err != nil { + return nil, fmt.Errorf("failed to create InteractiveBrowserCredential: %w", err) + } + return credential, nil + default: + return nil, fmt.Errorf("unsupported DTS authentication type %q", spec.authentication) + } +} + +func resolveCredential(options *Options, factory credentialFactory) (azcore.TokenCredential, error) { + switch options.Authentication { + case AuthenticationNone: + return nil, nil + case AuthenticationTokenCredential: + return options.Credential, nil + } + spec, err := newCredentialSpec(options) + if err != nil { + return nil, err + } + return factory(spec) +} + +// tokenScope builds the OAuth scope requested for a DTS resource ID. +func tokenScope(resourceID string) string { + return strings.TrimRight(strings.TrimSpace(resourceID), "/") + "/.default" +} + +func prepareOptions(options *Options) (Options, error) { + return prepareOptionsWith(options, newAzureIdentityCredential) +} + +// prepareOptionsWith normalizes, validates, and resolves the credential for a +// caller-supplied option set. The credential factory is a parameter so tests can +// cover every authentication mode offline. +func prepareOptionsWith(options *Options, factory credentialFactory) (Options, error) { + if options == nil { + return Options{}, fmt.Errorf("DTS options are required") + } + prepared := *options + prepared.AdditionallyAllowedTenants = slices.Clone(options.AdditionallyAllowedTenants) + prepared.UnaryInterceptors = slices.Clone(options.UnaryInterceptors) + prepared.StreamInterceptors = slices.Clone(options.StreamInterceptors) + if options.Versioning != nil { + versioning := *options.Versioning + prepared.Versioning = &versioning + } + if prepared.Authentication == "" { + prepared.Authentication = AuthenticationDefaultAzure + } + // Only an exactly empty resource ID defaults. A whitespace-only value is + // left intact so Validate below rejects it as documented instead of + // silently collapsing to the default resource. + if trimmed := strings.TrimSpace(prepared.ResourceID); trimmed != "" { + prepared.ResourceID = trimmed + } else if prepared.ResourceID == "" { + prepared.ResourceID = DefaultResourceID + } + if prepared.HelloTimeout == 0 { + prepared.HelloTimeout = 30 * time.Second + } + if prepared.MaxReceiveMessageSize == 0 { + prepared.MaxReceiveMessageSize = DefaultMaxReceiveMessageSize + } + if prepared.MaxSendMessageSize == 0 { + prepared.MaxSendMessageSize = DefaultMaxSendMessageSize + } + if prepared.KeepaliveTime > 0 && prepared.KeepaliveTimeout == 0 { + prepared.KeepaliveTimeout = DefaultKeepaliveTimeout + } + if prepared.MaximumTimerInterval == 0 { + prepared.MaximumTimerInterval = task.DefaultMaximumTimerInterval + } + if err := prepared.Validate(); err != nil { + return Options{}, err + } + credential, err := resolveCredential(&prepared, factory) + if err != nil { + return Options{}, err + } + if credential != nil { + prepared.Authentication = AuthenticationTokenCredential + prepared.Credential = credential + } + return prepared, nil +} + +// newPerRPCCredentials builds the DTS metadata carried on every RPC for a role. +func newPerRPCCredentials( + options *Options, + role connectionRole, + workerID string, +) *schedulerPerRPCCredentials { + userAgent := options.UserAgent + if userAgent == "" { + userAgent = fmt.Sprintf("durabletask-go/%s (%s)", sdkVersion(), role) + } + return &schedulerPerRPCCredentials{ + credential: options.Credential, + scope: tokenScope(options.ResourceID), + taskHub: options.TaskHubName, + userAgent: userAgent, + workerID: workerID, + } +} + +func connect( + options *Options, + role connectionRole, + workerID string, +) (*grpc.ClientConn, error) { + endpoint, err := normalizeEndpoint(options.EndpointAddress) + if err != nil { + return nil, err + } + + host := endpoint.Host + if endpoint.Port() == "" { + port := "443" + if endpoint.Scheme == "http" { + port = "80" + } + host = net.JoinHostPort(endpoint.Hostname(), port) + } + + var transportCredentials credentials.TransportCredentials + if endpoint.Scheme == "https" { + transportCredentials = credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + ServerName: endpoint.Hostname(), + }) + } else { + transportCredentials = insecure.NewCredentials() + } + + dialOptions := []grpc.DialOption{ + grpc.WithTransportCredentials(transportCredentials), + grpc.WithPerRPCCredentials(newPerRPCCredentials(options, role, workerID)), + grpc.WithDefaultCallOptions( + grpc.MaxCallRecvMsgSize(options.MaxReceiveMessageSize), + grpc.MaxCallSendMsgSize(options.MaxSendMessageSize), + ), + } + if options.KeepaliveTime > 0 { + dialOptions = append(dialOptions, grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: options.KeepaliveTime, + Timeout: options.KeepaliveTimeout, + PermitWithoutStream: false, + })) + } + if len(options.UnaryInterceptors) > 0 { + dialOptions = append(dialOptions, grpc.WithChainUnaryInterceptor(options.UnaryInterceptors...)) + } + if len(options.StreamInterceptors) > 0 { + dialOptions = append(dialOptions, grpc.WithChainStreamInterceptor(options.StreamInterceptors...)) + } + if role == clientRole { + dialOptions = append(dialOptions, grpc.WithDefaultServiceConfig(retryServiceConfig)) + } + if options.dialer != nil { + dialOptions = append(dialOptions, grpc.WithContextDialer(options.dialer)) + } + + target := "dns:///" + host + if options.dialer != nil { + target = "passthrough:///" + host + } + connection, err := grpc.NewClient(target, dialOptions...) + if err != nil { + return nil, fmt.Errorf("failed to create DTS gRPC connection: %w", err) + } + return connection, nil +} + +func defaultWorkerID() string { + hostname, err := os.Hostname() + if err != nil || hostname == "" { + hostname = "unknown-host" + } + return fmt.Sprintf("%s,%d,%s", hostname, os.Getpid(), strings.ReplaceAll(uuid.NewString(), "-", "")) +} + +func sdkVersion() string { + buildInfo, ok := debug.ReadBuildInfo() + if !ok { + return "dev" + } + if buildInfo.Main.Path == modulePath && buildInfo.Main.Version != "" && buildInfo.Main.Version != "(devel)" { + return strings.TrimPrefix(buildInfo.Main.Version, "v") + } + for _, dependency := range buildInfo.Deps { + if dependency.Path == modulePath && dependency.Version != "" { + return strings.TrimPrefix(dependency.Version, "v") + } + } + return "dev" +} diff --git a/durabletaskscheduler/connection_test.go b/durabletaskscheduler/connection_test.go new file mode 100644 index 00000000..2bb74e0e --- /dev/null +++ b/durabletaskscheduler/connection_test.go @@ -0,0 +1,1169 @@ +package durabletaskscheduler + +import ( + "context" + "errors" + "net" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type recordingCredential struct { + mu sync.Mutex + options []policy.TokenRequestOptions +} + +type failingCredential struct{} + +type blockingCredential struct { + mu sync.Mutex + calls int + started chan struct{} + release chan struct{} + token azcore.AccessToken + err error +} + +func newBlockingCredential() *blockingCredential { + return &blockingCredential{ + started: make(chan struct{}, 1), + release: make(chan struct{}), + token: azcore.AccessToken{Token: "token", ExpiresOn: time.Now().Add(time.Hour)}, + } +} + +func (c *blockingCredential) GetToken( + ctx context.Context, + _ policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + c.mu.Lock() + c.calls++ + c.mu.Unlock() + select { + case c.started <- struct{}{}: + default: + } + select { + case <-ctx.Done(): + return azcore.AccessToken{}, ctx.Err() + case <-c.release: + return c.token, c.err + } +} + +func (c *blockingCredential) callCount() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.calls +} + +func newTestSchedulerCredentials(credential azcore.TokenCredential) *schedulerPerRPCCredentials { + return &schedulerPerRPCCredentials{ + credential: credential, + scope: "https://durabletask.io/.default", + taskHub: "hub", + userAgent: "agent", + } +} + +func callSchedulerCredentialsConcurrently( + perRPC *schedulerPerRPCCredentials, + callers int, +) <-chan error { + start := make(chan struct{}) + errs := make(chan error, callers) + for range callers { + go func() { + <-start + _, err := perRPC.GetRequestMetadata(context.Background()) + errs <- err + }() + } + close(start) + return errs +} + +func (failingCredential) GetToken( + context.Context, + policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + return azcore.AccessToken{}, errors.New("temporary token failure") +} + +func (c *recordingCredential) GetToken( + _ context.Context, + options policy.TokenRequestOptions, +) (azcore.AccessToken, error) { + c.mu.Lock() + c.options = append(c.options, options) + c.mu.Unlock() + return azcore.AccessToken{Token: "token", ExpiresOn: time.Now().Add(time.Hour)}, nil +} + +type metadataServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + metadata chan metadata.MD + helloErr error + getErr error + state *protos.OrchestrationState +} + +func (s *metadataServer) GetInstance( + ctx context.Context, + _ *protos.GetInstanceRequest, +) (*protos.GetInstanceResponse, error) { + incoming, _ := metadata.FromIncomingContext(ctx) + s.metadata <- incoming + if s.getErr != nil { + return nil, s.getErr + } + return &protos.GetInstanceResponse{Exists: true, OrchestrationState: s.state}, nil +} + +type recreationDataConverter struct{} + +func (recreationDataConverter) Serialize(value any) (string, error) { + text, ok := value.(string) + if !ok { + return "", errors.New("recreation converter only supports strings") + } + return "recreated:" + text, nil +} + +func (recreationDataConverter) Deserialize(value string, target any) error { + text, ok := target.(*string) + if !ok { + return errors.New("recreation converter target must be *string") + } + if !strings.HasPrefix(value, "recreated:") { + return errors.New("recreation converter prefix is missing") + } + *text = strings.TrimPrefix(value, "recreated:") + return nil +} + +func (s *metadataServer) Hello(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { + if s.helloErr != nil { + return nil, s.helloErr + } + incoming, _ := metadata.FromIncomingContext(ctx) + s.metadata <- incoming + return &emptypb.Empty{}, nil +} + +func (s *metadataServer) GetWorkItems( + _ *protos.GetWorkItemsRequest, + stream protos.TaskHubSidecarService_GetWorkItemsServer, +) error { + incoming, _ := metadata.FromIncomingContext(stream.Context()) + s.metadata <- incoming + <-stream.Context().Done() + return nil +} + +func recreationMetadataInterceptor( + ctx context.Context, + method string, + request any, + reply any, + connection *grpc.ClientConn, + invoker grpc.UnaryInvoker, + options ...grpc.CallOption, +) error { + ctx = metadata.AppendToOutgoingContext(ctx, "x-recreation-interceptor", "preserved") + return invoker(ctx, method, request, reply, connection, options...) +} + +func startBufconnServer(t *testing.T, server protos.TaskHubSidecarServiceServer) (*bufconn.Listener, func()) { + t.Helper() + listener := bufconn.Listen(1024 * 1024) + grpcServer := grpc.NewServer() + protos.RegisterTaskHubSidecarServiceServer(grpcServer, server) + go func() { + _ = grpcServer.Serve(listener) + }() + return listener, func() { + grpcServer.Stop() + require.NoError(t, listener.Close()) + } +} + +func bufconnDialer(listener *bufconn.Listener) func(context.Context, string) (net.Conn, error) { + return func(context.Context, string) (net.Conn, error) { + return listener.Dial() + } +} + +func TestSchedulerCredentialsUseExpectedScope(t *testing.T) { + credential := &recordingCredential{} + perRPC := &schedulerPerRPCCredentials{ + credential: credential, + scope: "https://durabletask.io/.default", + taskHub: "hub", + userAgent: "agent", + workerID: "worker", + } + + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, "hub", values["taskhub"]) + require.Equal(t, "agent", values["x-user-agent"]) + require.Equal(t, "worker", values["workerid"]) + require.Equal(t, "Bearer token", values["authorization"]) + require.True(t, perRPC.RequireTransportSecurity()) + + credential.mu.Lock() + defer credential.mu.Unlock() + require.Equal(t, []string{"https://durabletask.io/.default"}, credential.options[0].Scopes) +} + +func TestSchedulerCredentialFailureIsTransient(t *testing.T) { + perRPC := newTestSchedulerCredentials(failingCredential{}) + _, err := perRPC.GetRequestMetadata(context.Background()) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "temporary token failure") +} + +func TestSchedulerCredentialsCacheAccessToken(t *testing.T) { + credential := &recordingCredential{} + perRPC := newTestSchedulerCredentials(credential) + + first, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + second, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, first["authorization"], second["authorization"]) + + credential.mu.Lock() + defer credential.mu.Unlock() + require.Len(t, credential.options, 1) +} + +func TestSchedulerCredentialsCoalesceConcurrentTokenRequests(t *testing.T) { + credential := newBlockingCredential() + perRPC := newTestSchedulerCredentials(credential) + + const callers = 64 + errs := callSchedulerCredentialsConcurrently(perRPC, callers) + <-credential.started + close(credential.release) + for range callers { + require.NoError(t, <-errs) + } + require.Equal(t, 1, credential.callCount()) +} + +func TestSchedulerCredentialsCoalesceShortLivedTokenRequests(t *testing.T) { + credential := newBlockingCredential() + credential.token.ExpiresOn = time.Time{} + perRPC := newTestSchedulerCredentials(credential) + + const callers = 64 + errs := callSchedulerCredentialsConcurrently(perRPC, callers) + <-credential.started + close(credential.release) + for range callers { + require.NoError(t, <-errs) + } + require.Equal(t, 1, credential.callCount()) +} + +func TestSchedulerCredentialsCoalesceRefreshFailure(t *testing.T) { + credential := newBlockingCredential() + credential.err = errors.New("token unavailable") + perRPC := newTestSchedulerCredentials(credential) + + const callers = 64 + errs := callSchedulerCredentialsConcurrently(perRPC, callers) + <-credential.started + close(credential.release) + for range callers { + require.ErrorContains(t, <-errs, "token unavailable") + } + _, err := perRPC.GetRequestMetadata(context.Background()) + require.ErrorContains(t, err, "token unavailable") + require.Equal(t, 1, credential.callCount()) +} + +func TestSchedulerCredentialsCanceledWaiterDoesNotBlockOnRefresh(t *testing.T) { + credential := newBlockingCredential() + perRPC := newTestSchedulerCredentials(credential) + + firstDone := make(chan error, 1) + go func() { + _, err := perRPC.GetRequestMetadata(context.Background()) + firstDone <- err + }() + <-credential.started + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := perRPC.GetRequestMetadata(ctx) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, context.Canceled.Error()) + + close(credential.release) + require.NoError(t, <-firstDone) +} + +func TestSchedulerCredentialsCanceledLeaderDoesNotCancelSharedRefresh(t *testing.T) { + credential := newBlockingCredential() + perRPC := newTestSchedulerCredentials(credential) + + leaderCtx, cancelLeader := context.WithCancel(context.Background()) + leaderDone := make(chan error, 1) + go func() { + _, err := perRPC.GetRequestMetadata(leaderCtx) + leaderDone <- err + }() + <-credential.started + + waiterDone := make(chan error, 1) + go func() { + _, err := perRPC.GetRequestMetadata(context.Background()) + waiterDone <- err + }() + cancelLeader() + require.ErrorContains(t, <-leaderDone, context.Canceled.Error()) + + select { + case err := <-waiterDone: + t.Fatalf("shared refresh ended with leader context: %v", err) + case <-time.After(10 * time.Millisecond): + } + close(credential.release) + require.NoError(t, <-waiterDone) + require.Equal(t, 1, credential.callCount()) +} + +func TestSchedulerCredentialsUseValidTokenWhenRefreshFails(t *testing.T) { + perRPC := newTestSchedulerCredentials(failingCredential{}) + perRPC.token.Store(perRPC.newCredentialState(azcore.AccessToken{ + Token: "cached", + ExpiresOn: time.Now().Add(time.Minute), + })) + + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, "Bearer cached", values["authorization"]) +} + +func TestSchedulerCredentialsRefreshesStaleTokenWithoutMutatingOldMetadata(t *testing.T) { + credential := &recordingCredential{} + perRPC := newTestSchedulerCredentials(credential) + oldState := perRPC.newCredentialState(azcore.AccessToken{ + Token: "old", + ExpiresOn: time.Now().Add(time.Hour), + }) + oldState.refreshAfter = time.Now().Add(-time.Second) + perRPC.token.Store(oldState) + + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, oldState.metadata["authorization"], values["authorization"]) + require.Eventually(t, func() bool { + return perRPC.token.Load() != oldState + }, time.Second, time.Millisecond) + values, err = perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.NotEqual(t, oldState.metadata["authorization"], values["authorization"]) + require.Equal(t, "Bearer old", oldState.metadata["authorization"]) + credential.mu.Lock() + defer credential.mu.Unlock() + require.Len(t, credential.options, 1) +} + +func TestSchedulerCredentialStateHonorsRefreshOnAndExpiryBuffer(t *testing.T) { + perRPC := newTestSchedulerCredentials(&recordingCredential{}) + expiresOn := time.Now().Add(time.Hour) + refreshOn := time.Now().Add(10 * time.Minute) + + withRefreshOn := perRPC.newCredentialState(azcore.AccessToken{ + Token: "token", + ExpiresOn: expiresOn, + RefreshOn: refreshOn, + }) + require.Equal(t, refreshOn, withRefreshOn.refreshAfter) + + withoutRefreshOn := perRPC.newCredentialState(azcore.AccessToken{ + Token: "token", + ExpiresOn: expiresOn, + }) + require.Equal(t, expiresOn.Add(-accessTokenRefreshBuffer), withoutRefreshOn.refreshAfter) +} + +func TestSchedulerCredentialsDoNotUseExpiredTokenWhenRefreshFails(t *testing.T) { + perRPC := newTestSchedulerCredentials(failingCredential{}) + perRPC.token.Store(perRPC.newCredentialState(azcore.AccessToken{ + Token: "expired", + ExpiresOn: time.Now().Add(-time.Minute), + })) + + _, err := perRPC.GetRequestMetadata(context.Background()) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "temporary token failure") +} + +func TestSchedulerCredentialsRejectExpiredCredentialToken(t *testing.T) { + credential := newBlockingCredential() + credential.token.ExpiresOn = time.Now().Add(-time.Minute) + close(credential.release) + perRPC := newTestSchedulerCredentials(credential) + + _, err := perRPC.GetRequestMetadata(context.Background()) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "expired access token") + _, err = perRPC.GetRequestMetadata(context.Background()) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, 1, credential.callCount()) +} + +func TestSchedulerCredentialsUseValidTokenWhileRefreshing(t *testing.T) { + credential := newBlockingCredential() + credential.token.Token = "refreshed" + perRPC := newTestSchedulerCredentials(credential) + cached := perRPC.newCredentialState(azcore.AccessToken{ + Token: "cached", + ExpiresOn: time.Now().Add(time.Hour), + }) + cached.refreshAfter = time.Now().Add(-time.Second) + perRPC.token.Store(cached) + + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, "Bearer cached", values["authorization"]) + <-credential.started + close(credential.release) + require.Eventually(t, func() bool { + return perRPC.token.Load() != cached + }, time.Second, time.Millisecond) +} + +func BenchmarkSchedulerCredentialsCachedToken(b *testing.B) { + perRPC := &schedulerPerRPCCredentials{ + credential: &recordingCredential{}, + scope: "https://durabletask.io/.default", + taskHub: "hub", + userAgent: "agent", + workerID: "worker", + } + if _, err := perRPC.GetRequestMetadata(context.Background()); err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, err := perRPC.GetRequestMetadata(context.Background()); err != nil { + b.Error(err) + } + } + }) +} + +func TestConfiguredClientAndWorkerUseSeparateMetadataAndConnections(t *testing.T) { + server := &metadataServer{metadata: make(chan metadata.MD, 4)} + listener, stop := startBufconnServer(t, server) + defer stop() + + options, err := NewOptionsFromConnectionString( + "Endpoint=http://bufconn;TaskHub=default;Authentication=None", + ) + require.NoError(t, err) + options.WorkerID = "worker-id" + options.dialer = bufconnDialer(listener) + + managementClient, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.NoError(t, err) + defer func() { + require.NoError(t, managementClient.Close()) + }() + clientMetadata := <-server.metadata + require.Equal(t, []string{"default"}, clientMetadata.Get("taskhub")) + require.Empty(t, clientMetadata.Get("workerid")) + require.Contains(t, clientMetadata.Get("x-user-agent")[0], "DurableTaskClient") + + worker, err := NewWorker( + options, + task.NewTaskRegistry(), + api.DefaultLogger(), + durabletaskclient.WithWorkerSilentDisconnectTimeout(time.Second), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + workerHelloMetadata := <-server.metadata + workerStreamMetadata := <-server.metadata + for _, incoming := range []metadata.MD{workerHelloMetadata, workerStreamMetadata} { + require.Equal(t, []string{"default"}, incoming.Get("taskhub")) + require.Equal(t, []string{"worker-id"}, incoming.Get("workerid")) + require.True(t, strings.Contains(incoming.Get("x-user-agent")[0], "DurableTaskWorker")) + } + + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) +} + +func TestClientCloseStopsCompatibilityListener(t *testing.T) { + server := &metadataServer{metadata: make(chan metadata.MD, 4)} + listener, stop := startBufconnServer(t, server) + defer stop() + + options, err := NewOptionsFromConnectionString( + "Endpoint=http://bufconn;TaskHub=default;Authentication=None", + ) + require.NoError(t, err) + options.dialer = bufconnDialer(listener) + managementClient, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.NoError(t, err) + <-server.metadata + require.NoError(t, managementClient.StartWorkItemListener(context.Background(), task.NewTaskRegistry())) + <-server.metadata + <-server.metadata + require.NoError(t, managementClient.Close()) + + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, managementClient.StopWorkItemListener(shutdownCtx)) +} + +func TestNewClientFailsFastWhenHelloIsRejected(t *testing.T) { + server := &metadataServer{ + metadata: make(chan metadata.MD, 1), + helloErr: status.Error(codes.PermissionDenied, "forbidden"), + } + listener, stop := startBufconnServer(t, server) + defer stop() + + options, err := NewOptionsFromConnectionString( + "Endpoint=http://bufconn;TaskHub=default;Authentication=None", + ) + require.NoError(t, err) + options.dialer = bufconnDialer(listener) + + managementClient, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.Nil(t, managementClient) + require.ErrorContains(t, err, "Hello") + require.ErrorContains(t, err, "PermissionDenied") +} + +func TestNewClientReceivesMessagesLargerThanGrpcDefault(t *testing.T) { + const payloadSize = 5 * 1024 * 1024 + server := &metadataServer{ + metadata: make(chan metadata.MD, 4), + state: &protos.OrchestrationState{ + InstanceId: "large-message", + Name: "orchestrator", + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + CreatedTimestamp: timestamppb.Now(), + LastUpdatedTimestamp: timestamppb.Now(), + Input: wrapperspb.String(strings.Repeat("x", payloadSize)), + }, + } + listener, stop := startBufconnServer(t, server) + defer stop() + + options := NewOptions("http://bufconn", "default") + options.Authentication = AuthenticationNone + options.AllowInsecureConnection = true + options.dialer = bufconnDialer(listener) + managementClient, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.NoError(t, err) + defer func() { + require.NoError(t, managementClient.Close()) + }() + + result, err := managementClient.FetchOrchestrationMetadata( + context.Background(), + "large-message", + api.WithFetchPayloads(true), + ) + require.NoError(t, err) + require.Len(t, result.SerializedInput, payloadSize) + + limitedOptions := *options + limitedOptions.MaxReceiveMessageSize = 1024 * 1024 + limitedClient, err := NewClient(context.Background(), &limitedOptions, api.DefaultLogger()) + require.NoError(t, err) + defer func() { + require.NoError(t, limitedClient.Close()) + }() + _, err = limitedClient.FetchOrchestrationMetadata( + context.Background(), + "large-message", + api.WithFetchPayloads(true), + ) + require.Equal(t, codes.ResourceExhausted, status.Code(err)) +} + +func TestNewClientRecreatesChannelAndPreservesConfiguration(t *testing.T) { + firstServer := &metadataServer{ + getErr: status.Error(codes.Unavailable, "replace this channel"), + metadata: make(chan metadata.MD, 16), + } + firstListener, stopFirst := startBufconnServer(t, firstServer) + defer stopFirst() + + converter := recreationDataConverter{} + store := payload.NewMemoryStore() + largePayloads := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + serializedOutput, err := converter.Serialize("preserved") + require.NoError(t, err) + externalizedOutput, err := largepayload.Externalize( + context.Background(), + largePayloads, + wrapperspb.String(serializedOutput), + ) + require.NoError(t, err) + secondServer := &metadataServer{ + state: &protos.OrchestrationState{ + InstanceId: "recreated-instance", + Name: "orchestrator", + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + CreatedTimestamp: timestamppb.Now(), + LastUpdatedTimestamp: timestamppb.Now(), + Output: externalizedOutput, + }, + metadata: make(chan metadata.MD, 16), + } + secondListener, stopSecond := startBufconnServer(t, secondServer) + defer stopSecond() + + var listenerMu sync.RWMutex + activeListener := firstListener + options, err := NewOptionsFromConnectionString( + "Endpoint=http://bufconn;TaskHub=default;Authentication=None", + ) + require.NoError(t, err) + options.ChannelRecreateFailureThreshold = 1 + options.ChannelRecreateMinInterval = 0 + options.DataConverter = converter + options.LargePayloads = largePayloads + options.UnaryInterceptors = []grpc.UnaryClientInterceptor{recreationMetadataInterceptor} + options.dialer = func(context.Context, string) (net.Conn, error) { + listenerMu.RLock() + listener := activeListener + listenerMu.RUnlock() + return listener.Dial() + } + + client, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.NoError(t, err) + defer func() { + require.NoError(t, client.Close()) + }() + <-firstServer.metadata + + listenerMu.Lock() + activeListener = secondListener + listenerMu.Unlock() + + _, err = client.FetchOrchestrationMetadata( + context.Background(), + "recreated-instance", + api.WithFetchPayloads(true), + ) + require.Error(t, err) + + var orchestration *api.OrchestrationMetadata + require.Eventually(t, func() bool { + orchestration, err = client.FetchOrchestrationMetadata( + context.Background(), + "recreated-instance", + api.WithFetchPayloads(true), + ) + return err == nil + }, 5*time.Second, 10*time.Millisecond) + var output string + require.NoError(t, orchestration.ReadOutput(&output)) + require.Equal(t, "preserved", output) + + replacementMetadata := <-secondServer.metadata + require.Equal(t, []string{"preserved"}, replacementMetadata.Get("x-recreation-interceptor")) + require.Equal(t, []string{"default"}, replacementMetadata.Get("taskhub")) + require.Contains(t, replacementMetadata.Get("x-user-agent")[0], "DurableTaskClient") +} + +func TestNewClientNilLoggerSupportsRecoveryAndClose(t *testing.T) { + server := &metadataServer{ + metadata: make(chan metadata.MD, 8), + state: &protos.OrchestrationState{InstanceId: "nil-logger"}, + } + listener, stop := startBufconnServer(t, server) + t.Cleanup(stop) + options := insecureBufconnOptions(t, listener) + options.ChannelRecreateFailureThreshold = 1 + options.ChannelRecreateMinInterval = 0 + var failFirst atomic.Bool + failFirst.Store(true) + options.UnaryInterceptors = []grpc.UnaryClientInterceptor{ + func(ctx context.Context, method string, request, response any, conn *grpc.ClientConn, + invoker grpc.UnaryInvoker, callOptions ...grpc.CallOption) error { + if method == protos.TaskHubSidecarService_GetInstance_FullMethodName && failFirst.Swap(false) { + return status.Error(codes.Unavailable, "replace this channel") + } + return invoker(ctx, method, request, response, conn, callOptions...) + }, + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := NewClient(ctx, options, nil) + require.NoError(t, err) + t.Cleanup(func() { + closed := make(chan error, 1) + go func() { closed <- client.Close() }() + select { + case err := <-closed: + require.NoError(t, err) + case <-time.After(time.Second): + t.Error("management client Close did not finish") + } + }) + initial := client.connection.current + require.NotPanics(t, func() { + _, err = client.FetchOrchestrationMetadata(ctx, "nil-logger") + }) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Eventually(t, func() bool { + client.connection.mu.Lock() + defer client.connection.mu.Unlock() + return client.connection.current != initial + }, time.Second, time.Millisecond) + result, err := client.FetchOrchestrationMetadata(ctx, "nil-logger") + require.NoError(t, err) + require.Equal(t, api.InstanceID("nil-logger"), result.InstanceID) +} + +// delayedHelloServer blocks Hello so client and worker deadlines can be observed. +type delayedHelloServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + delay time.Duration +} + +func (s *delayedHelloServer) Hello(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { + select { + case <-time.After(s.delay): + return &emptypb.Empty{}, nil + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } +} + +func (s *delayedHelloServer) GetWorkItems( + _ *protos.GetWorkItemsRequest, + stream protos.TaskHubSidecarService_GetWorkItemsServer, +) error { + <-stream.Context().Done() + return nil +} + +// flakyHelloServer fails Hello with UNAVAILABLE a fixed number of times so the +// client channel retry policy can be observed. +type flakyHelloServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + mu sync.Mutex + failures int + attempts int + lastMeta metadata.MD +} + +func (s *flakyHelloServer) Hello(ctx context.Context, _ *emptypb.Empty) (*emptypb.Empty, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.attempts++ + s.lastMeta, _ = metadata.FromIncomingContext(ctx) + if s.attempts <= s.failures { + return nil, status.Error(codes.Unavailable, "try again") + } + return &emptypb.Empty{}, nil +} + +func (s *flakyHelloServer) attemptCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.attempts +} + +func insecureBufconnOptions(t *testing.T, listener *bufconn.Listener) *Options { + t.Helper() + options, err := NewOptionsFromConnectionString( + "Endpoint=http://bufconn;TaskHub=default;Authentication=None", + ) + require.NoError(t, err) + options.dialer = bufconnDialer(listener) + return options +} + +// TestNewPerRPCCredentialsPreservesRoleMetadata pins the metadata each role +// sends, including the user-agent override and worker identity. +func TestNewPerRPCCredentialsPreservesRoleMetadata(t *testing.T) { + tests := []struct { + name string + role connectionRole + workerID string + userAgent string + resourceID string + credential azcore.TokenCredential + wantUserAgent string + wantScope string + }{ + { + name: "client default user agent", + role: clientRole, + wantUserAgent: "durabletask-go/" + sdkVersion() + " (DurableTaskClient)", + wantScope: "https://durabletask.io/.default", + }, + { + name: "worker default user agent", + role: workerRole, + workerID: "worker-1", + wantUserAgent: "durabletask-go/" + sdkVersion() + " (DurableTaskWorker)", + wantScope: "https://durabletask.io/.default", + }, + { + name: "client user agent override", + role: clientRole, + userAgent: "contoso-app/2.1", + wantUserAgent: "contoso-app/2.1", + wantScope: "https://durabletask.io/.default", + }, + { + name: "worker user agent override", + role: workerRole, + workerID: "worker-2", + userAgent: "contoso-app/2.1", + wantUserAgent: "contoso-app/2.1", + wantScope: "https://durabletask.io/.default", + }, + { + name: "custom resource ID with credential", + role: clientRole, + credential: &recordingCredential{}, + resourceID: "https://custom.example.com/", + wantUserAgent: "durabletask-go/" + sdkVersion() + " (DurableTaskClient)", + wantScope: "https://custom.example.com/.default", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.UserAgent = tt.userAgent + options.Credential = tt.credential + if tt.resourceID != "" { + options.ResourceID = tt.resourceID + } + + perRPC := newPerRPCCredentials(options, tt.role, tt.workerID) + require.Equal(t, "hub", perRPC.taskHub) + require.Equal(t, tt.workerID, perRPC.workerID) + require.Equal(t, tt.wantUserAgent, perRPC.userAgent) + require.Equal(t, tt.wantScope, perRPC.scope) + require.Equal(t, tt.credential, perRPC.credential) + require.Equal(t, tt.credential != nil, perRPC.RequireTransportSecurity()) + + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, "hub", values["taskhub"]) + require.Equal(t, tt.wantUserAgent, values["x-user-agent"]) + if tt.workerID == "" { + require.NotContains(t, values, "workerid") + } else { + require.Equal(t, tt.workerID, values["workerid"]) + } + if tt.credential == nil { + require.NotContains(t, values, "authorization") + } else { + require.Contains(t, values["authorization"], "Bearer ") + } + }) + } +} + +func TestSchedulerCredentialsOmitAuthorizationWithoutCredential(t *testing.T) { + perRPC := &schedulerPerRPCCredentials{taskHub: "hub", userAgent: "agent"} + values, err := perRPC.GetRequestMetadata(context.Background()) + require.NoError(t, err) + require.Equal(t, map[string]string{"taskhub": "hub", "x-user-agent": "agent"}, values) + require.False(t, perRPC.RequireTransportSecurity()) +} + +// TestClientChannelRetriesUnavailable exercises the client-only retry service +// config: a transparent retry recovers a single Hello call. +func TestClientChannelRetriesUnavailable(t *testing.T) { + server := &flakyHelloServer{failures: 3} + listener, stop := startBufconnServer(t, server) + defer stop() + + options := insecureBufconnOptions(t, listener) + connection, err := connect(options, clientRole, "") + require.NoError(t, err) + defer func() { + require.NoError(t, connection.Close()) + }() + + _, err = protos.NewTaskHubSidecarServiceClient(connection).Hello(context.Background(), &emptypb.Empty{}) + require.NoError(t, err) + require.Equal(t, 4, server.attemptCount()) + require.Equal(t, []string{"default"}, server.lastMeta.Get("taskhub")) +} + +// TestClientChannelRetriesStopAtMaxAttempts pins maxAttempts at five. +func TestClientChannelRetriesStopAtMaxAttempts(t *testing.T) { + server := &flakyHelloServer{failures: 10} + listener, stop := startBufconnServer(t, server) + defer stop() + + options := insecureBufconnOptions(t, listener) + connection, err := connect(options, clientRole, "") + require.NoError(t, err) + defer func() { + require.NoError(t, connection.Close()) + }() + + _, err = protos.NewTaskHubSidecarServiceClient(connection).Hello(context.Background(), &emptypb.Empty{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, 5, server.attemptCount()) +} + +// TestWorkerChannelDoesNotUseClientRetryConfig confirms the retry service config +// is applied to the client role only; worker resilience is the worker's own +// reconnect loop. +func TestWorkerChannelDoesNotUseClientRetryConfig(t *testing.T) { + server := &flakyHelloServer{failures: 3} + listener, stop := startBufconnServer(t, server) + defer stop() + + options := insecureBufconnOptions(t, listener) + connection, err := connect(options, workerRole, "worker-1") + require.NoError(t, err) + defer func() { + require.NoError(t, connection.Close()) + }() + + _, err = protos.NewTaskHubSidecarServiceClient(connection).Hello(context.Background(), &emptypb.Empty{}) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.Equal(t, 1, server.attemptCount()) + require.Equal(t, []string{"worker-1"}, server.lastMeta.Get("workerid")) +} + +// TestNewClientHelloUsesConfiguredTimeout proves Options.HelloTimeout bounds the +// client fail-fast handshake. +func TestNewClientHelloUsesConfiguredTimeout(t *testing.T) { + listener, stop := startBufconnServer(t, &delayedHelloServer{delay: 5 * time.Second}) + defer stop() + + options := insecureBufconnOptions(t, listener) + options.HelloTimeout = 100 * time.Millisecond + + start := time.Now() + client, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.Nil(t, client) + require.ErrorContains(t, err, "DTS client Hello failed") + require.Equal(t, codes.DeadlineExceeded, status.Code(errors.Unwrap(err))) + require.Less(t, time.Since(start), 3*time.Second) +} + +// TestNewClientHelloSucceedsWithinTimeout is the positive counterpart: the same +// delayed server is accepted when the deadline allows it. +func TestNewClientHelloSucceedsWithinTimeout(t *testing.T) { + listener, stop := startBufconnServer(t, &delayedHelloServer{delay: 50 * time.Millisecond}) + defer stop() + + options := insecureBufconnOptions(t, listener) + options.HelloTimeout = 10 * time.Second + + client, err := NewClient(context.Background(), options, api.DefaultLogger()) + require.NoError(t, err) + require.NoError(t, client.Close()) +} + +// TestNewClientHonorsCallerContextDeadline confirms the caller context still +// bounds the handshake when it is shorter than HelloTimeout. +func TestNewClientHonorsCallerContextDeadline(t *testing.T) { + listener, stop := startBufconnServer(t, &delayedHelloServer{delay: 5 * time.Second}) + defer stop() + + options := insecureBufconnOptions(t, listener) + options.HelloTimeout = 30 * time.Second + + ctx, cancel := context.WithTimeout(context.Background(), 100*time.Millisecond) + defer cancel() + client, err := NewClient(ctx, options, api.DefaultLogger()) + require.Nil(t, client) + require.ErrorContains(t, err, "DTS client Hello failed") +} + +// TestNewWorkerHelloUsesConfiguredTimeout proves Options.HelloTimeout reaches the +// worker connection factory. +func TestNewWorkerHelloUsesConfiguredTimeout(t *testing.T) { + listener, stop := startBufconnServer(t, &delayedHelloServer{delay: 5 * time.Second}) + defer stop() + + options := insecureBufconnOptions(t, listener) + options.HelloTimeout = 100 * time.Millisecond + + worker, err := NewWorker(options, task.NewTaskRegistry(), api.DefaultLogger()) + require.NoError(t, err) + + start := time.Now() + err = worker.Start(context.Background()) + require.ErrorContains(t, err, "Hello failed") + require.Equal(t, codes.DeadlineExceeded, status.Code(errors.Unwrap(err))) + require.Less(t, time.Since(start), 3*time.Second) + require.False(t, worker.Running()) +} + +// TestNewWorkerHelloTimeoutOptionOverridesOptions confirms caller-supplied worker +// options are applied after the DTS-derived defaults. +func TestNewWorkerHelloTimeoutOptionOverridesOptions(t *testing.T) { + listener, stop := startBufconnServer(t, &delayedHelloServer{delay: 5 * time.Second}) + defer stop() + + options := insecureBufconnOptions(t, listener) + options.HelloTimeout = 30 * time.Second + + worker, err := NewWorker( + options, + task.NewTaskRegistry(), + api.DefaultLogger(), + durabletaskclient.WithWorkerHelloTimeout(100*time.Millisecond), + ) + require.NoError(t, err) + + start := time.Now() + require.ErrorContains(t, worker.Start(context.Background()), "Hello failed") + require.Less(t, time.Since(start), 3*time.Second) +} + +// TestNewClientAndNewWorkerRejectInvalidOptions confirms both entry points share +// the same validation before any connection is created. +func TestNewClientAndNewWorkerRejectInvalidOptions(t *testing.T) { + tests := []struct { + name string + options *Options + wantErr string + }{ + {name: "nil options", options: nil, wantErr: "options are required"}, + { + name: "missing task hub", + options: &Options{EndpointAddress: "scheduler.example.com", Authentication: AuthenticationNone}, + wantErr: "task hub name is required", + }, + { + name: "plaintext without opt-in", + options: &Options{ + EndpointAddress: "http://127.0.0.1:8080", + TaskHubName: "hub", + Authentication: AuthenticationNone, + }, + wantErr: "AllowInsecureConnection", + }, + { + name: "identity fields with None", + options: &Options{ + EndpointAddress: "http://127.0.0.1:8080", + TaskHubName: "hub", + Authentication: AuthenticationNone, + AllowInsecureConnection: true, + TenantID: "tenant", + }, + wantErr: "Authentication None does not use TenantID", + }, + { + name: "worker ID injection", + options: &Options{ + EndpointAddress: "http://127.0.0.1:8080", + TaskHubName: "hub", + Authentication: AuthenticationNone, + AllowInsecureConnection: true, + WorkerID: "worker\r\nid", + }, + wantErr: "worker ID", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + client, err := NewClient(context.Background(), tt.options, api.DefaultLogger()) + require.Nil(t, client) + require.ErrorContains(t, err, tt.wantErr) + + worker, err := NewWorker(tt.options, task.NewTaskRegistry(), api.DefaultLogger()) + require.Nil(t, worker) + require.ErrorContains(t, err, tt.wantErr) + }) + } +} + +// TestNewWorkerGeneratesStableWorkerIDWhenUnset covers the default worker +// identity shape and its reuse across reconnects. +func TestNewWorkerGeneratesStableWorkerIDWhenUnset(t *testing.T) { + server := &metadataServer{metadata: make(chan metadata.MD, 4)} + listener, stop := startBufconnServer(t, server) + defer stop() + + options := insecureBufconnOptions(t, listener) + worker, err := NewWorker( + options, + task.NewTaskRegistry(), + api.DefaultLogger(), + durabletaskclient.WithWorkerSilentDisconnectTimeout(time.Second), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + + helloMetadata := <-server.metadata + streamMetadata := <-server.metadata + workerIDs := helloMetadata.Get("workerid") + require.Len(t, workerIDs, 1) + require.Equal(t, workerIDs, streamMetadata.Get("workerid")) + workerIDParts := strings.Split(workerIDs[0], ",") + require.Len(t, workerIDParts, 3) + require.NotEmpty(t, workerIDParts[0]) + + shutdownCtx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) +} + +func TestDefaultWorkerIDIsUniquePerCall(t *testing.T) { + first := defaultWorkerID() + second := defaultWorkerID() + require.NotEqual(t, first, second) + firstParts := strings.Split(first, ",") + secondParts := strings.Split(second, ",") + require.Len(t, firstParts, 3) + require.Equal(t, secondParts[0], firstParts[0]) + require.Equal(t, secondParts[1], firstParts[1]) + require.Len(t, firstParts[2], 32) +} diff --git a/durabletaskscheduler/options.go b/durabletaskscheduler/options.go new file mode 100644 index 00000000..79aa388c --- /dev/null +++ b/durabletaskscheduler/options.go @@ -0,0 +1,435 @@ +// Package durabletaskscheduler configures Durable Task Scheduler (DTS) +// management clients and workers. +package durabletaskscheduler + +import ( + "context" + "fmt" + "net" + "net/url" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" +) + +const ( + // DefaultResourceID is the Azure resource used to request DTS access tokens. + DefaultResourceID = "https://durabletask.io" + + // DefaultMaxReceiveMessageSize and DefaultMaxSendMessageSize are the default + // per-message gRPC bounds. + DefaultMaxReceiveMessageSize = 64 * 1024 * 1024 + DefaultMaxSendMessageSize = 64 * 1024 * 1024 + // DefaultKeepaliveTime and DefaultKeepaliveTimeout configure pings while a + // gRPC stream is active. + DefaultKeepaliveTime = 2 * time.Minute + DefaultKeepaliveTimeout = 20 * time.Second + minimumKeepaliveTime = 30 * time.Second + minimumGRPCMessageSize = 64 * 1024 +) + +type AuthenticationType string + +const ( + AuthenticationNone AuthenticationType = "None" + AuthenticationDefaultAzure AuthenticationType = "DefaultAzure" + AuthenticationManagedIdentity AuthenticationType = "ManagedIdentity" + AuthenticationWorkloadIdentity AuthenticationType = "WorkloadIdentity" + AuthenticationEnvironment AuthenticationType = "Environment" + AuthenticationAzureCLI AuthenticationType = "AzureCLI" + AuthenticationAzurePowerShell AuthenticationType = "AzurePowerShell" + AuthenticationInteractiveBrowser AuthenticationType = "InteractiveBrowser" + AuthenticationTokenCredential AuthenticationType = "TokenCredential" +) + +// Options configures connections to a Durable Task Scheduler endpoint. +type Options struct { + EndpointAddress string + TaskHubName string + Authentication AuthenticationType + + // Credential is required by, and valid only for, AuthenticationTokenCredential. + Credential azcore.TokenCredential + + // ResourceID is the DTS resource the access token is requested for. The + // requested scope is the resource ID without surrounding whitespace or + // trailing slashes plus "/.default". Exactly empty uses DefaultResourceID; + // a whitespace-only value is rejected. + ResourceID string + + // ClientID is used by ManagedIdentity, WorkloadIdentity, and + // InteractiveBrowser. Other modes ignore it, except None and TokenCredential, + // which reject it. + ClientID string + + // TenantID is used by DefaultAzure, WorkloadIdentity, AzureCLI, + // AzurePowerShell, and InteractiveBrowser. Environment and ManagedIdentity + // ignore it; None and TokenCredential reject it. + TenantID string + + // TokenFilePath is used only by WorkloadIdentity. Other modes ignore it, + // except None and TokenCredential, which reject it. + TokenFilePath string + + // AdditionallyAllowedTenants is used by DefaultAzure, WorkloadIdentity, + // AzureCLI, AzurePowerShell, and InteractiveBrowser. Environment reads + // AZURE_ADDITIONALLY_ALLOWED_TENANTS instead and ManagedIdentity ignores it; + // None and TokenCredential reject it. Empty entries are dropped. + AdditionallyAllowedTenants []string + + // AllowInsecureConnection must be true to use an http:// endpoint. Plaintext + // connections are supported only when Authentication is None. + AllowInsecureConnection bool + + // WorkerID overrides the generated worker identity. It is ignored by clients. + WorkerID string + + // UserAgent overrides the x-user-agent metadata value. + UserAgent string + + // HelloTimeout controls fail-fast connectivity checks. + HelloTimeout time.Duration + + // MaxReceiveMessageSize and MaxSendMessageSize bound individual gRPC + // messages. Zero uses the 64 MiB SDK default. + MaxReceiveMessageSize int + MaxSendMessageSize int + + // KeepaliveTime controls client pings while a gRPC stream is active. Zero + // disables keepalive, in which case KeepaliveTimeout is ignored. + // KeepaliveTimeout bounds each ping acknowledgement. + KeepaliveTime time.Duration + KeepaliveTimeout time.Duration + + // MaximumTimerInterval limits one physical durable timer action. Longer + // timers are split deterministically. Zero uses the three-day default. + MaximumTimerInterval time.Duration + + // ChannelRecreateFailureThreshold controls how many consecutive transport + // failures cause the owned management client to create a new gRPC channel. + // A non-positive value disables recreation. NewOptions defaults to 5. + ChannelRecreateFailureThreshold int + + // ChannelRecreateMinInterval limits how often the owned management client + // attempts channel recreation. NewOptions defaults to 30 seconds. + ChannelRecreateMinInterval time.Duration + + // UnaryInterceptors and StreamInterceptors are applied in order to every + // gRPC channel created for clients and workers, including replacements. + UnaryInterceptors []grpc.UnaryClientInterceptor + StreamInterceptors []grpc.StreamClientInterceptor + + // LargePayloads enables external payload references for clients and workers. + LargePayloads *api.LargePayloadOptions + + // Versioning configures worker version acceptance and the default version + // applied by both the client and worker. + Versioning *task.VersioningOptions + + // DataConverter configures application payload serialization for both clients and workers. + DataConverter api.DataConverter + + dialer func(context.Context, string) (net.Conn, error) +} + +func NewOptions(endpointAddress, taskHubName string) *Options { + return &Options{ + EndpointAddress: endpointAddress, + TaskHubName: taskHubName, + Authentication: AuthenticationDefaultAzure, + ResourceID: DefaultResourceID, + HelloTimeout: 30 * time.Second, + MaxReceiveMessageSize: DefaultMaxReceiveMessageSize, + MaxSendMessageSize: DefaultMaxSendMessageSize, + KeepaliveTime: DefaultKeepaliveTime, + KeepaliveTimeout: DefaultKeepaliveTimeout, + MaximumTimerInterval: task.DefaultMaximumTimerInterval, + ChannelRecreateFailureThreshold: 5, + ChannelRecreateMinInterval: 30 * time.Second, + } +} + +func NewOptionsWithCredential(endpointAddress, taskHubName string, credential azcore.TokenCredential) *Options { + options := NewOptions(endpointAddress, taskHubName) + options.Authentication = AuthenticationTokenCredential + options.Credential = credential + return options +} + +// NewOptionsFromConnectionString parses the DTS connection-string form: +// +// Endpoint=;TaskHub=;Authentication= +// +// Keys and Authentication values are case-insensitive, surrounding whitespace is +// trimmed, empty segments are skipped, and a repeated key uses its last value. +// Supported optional keys are ClientID, TenantID, TokenFilePath, and the +// comma-separated AdditionallyAllowedTenants. +func NewOptionsFromConnectionString(connectionString string) (*Options, error) { + values := make(map[string]string) + for _, segment := range strings.Split(connectionString, ";") { + segment = strings.TrimSpace(segment) + if segment == "" { + continue + } + keyValue := strings.SplitN(segment, "=", 2) + if len(keyValue) != 2 { + return nil, fmt.Errorf("invalid connection string segment %q", segment) + } + key := strings.ToLower(strings.TrimSpace(keyValue[0])) + value := strings.TrimSpace(keyValue[1]) + switch key { + case "endpoint", "taskhub", "authentication", "clientid", "tenantid", "tokenfilepath", "additionallyallowedtenants": + values[key] = value + default: + return nil, fmt.Errorf("unsupported connection string key %q", keyValue[0]) + } + } + + endpoint, ok := values["endpoint"] + if !ok || endpoint == "" { + return nil, fmt.Errorf("connection string is missing required Endpoint") + } + taskHub, ok := values["taskhub"] + if !ok || taskHub == "" { + return nil, fmt.Errorf("connection string is missing required TaskHub") + } + authentication, ok := values["authentication"] + if !ok || authentication == "" { + return nil, fmt.Errorf("connection string is missing required Authentication") + } + + options := NewOptions(endpoint, taskHub) + parsedAuthentication, err := parseAuthenticationType(authentication) + if err != nil { + return nil, err + } + options.Authentication = parsedAuthentication + options.ClientID = values["clientid"] + options.TenantID = values["tenantid"] + options.TokenFilePath = values["tokenfilepath"] + options.AdditionallyAllowedTenants = normalizeAdditionallyAllowedTenants( + strings.Split(values["additionallyallowedtenants"], ","), + ) + if options.Authentication == AuthenticationNone { + options.AllowInsecureConnection = true + } + if err := options.Validate(); err != nil { + return nil, err + } + return options, nil +} + +func parseAuthenticationType(value string) (AuthenticationType, error) { + for _, authentication := range []AuthenticationType{ + AuthenticationDefaultAzure, + AuthenticationManagedIdentity, + AuthenticationWorkloadIdentity, + AuthenticationEnvironment, + AuthenticationAzureCLI, + AuthenticationAzurePowerShell, + AuthenticationInteractiveBrowser, + AuthenticationNone, + } { + if strings.EqualFold(value, string(authentication)) { + return authentication, nil + } + } + switch { + case strings.EqualFold(value, string(AuthenticationTokenCredential)): + return "", fmt.Errorf( + "authentication %q cannot be set from a connection string; use NewOptionsWithCredential", + value, + ) + case strings.EqualFold(value, "VisualStudio"), strings.EqualFold(value, "VisualStudioCode"): + return "", fmt.Errorf( + "authentication %q has no Azure Identity for Go equivalent; provide an explicit TokenCredential", + value, + ) + default: + return "", fmt.Errorf("unsupported Authentication value %q", value) + } +} + +func (o *Options) Validate() error { + if o == nil { + return fmt.Errorf("DTS options are required") + } + if strings.TrimSpace(o.TaskHubName) == "" { + return fmt.Errorf("DTS task hub name is required") + } + if o.TaskHubName != strings.TrimSpace(o.TaskHubName) { + return fmt.Errorf("DTS task hub name cannot have leading or trailing whitespace") + } + if strings.ContainsAny(o.TaskHubName, "\r\n") { + return fmt.Errorf("DTS task hub name cannot contain newlines") + } + if o.WorkerID != strings.TrimSpace(o.WorkerID) || strings.ContainsAny(o.WorkerID, "\r\n") { + return fmt.Errorf("DTS worker ID cannot contain leading/trailing whitespace or newlines") + } + if o.UserAgent != strings.TrimSpace(o.UserAgent) || strings.ContainsAny(o.UserAgent, "\r\n") { + return fmt.Errorf("DTS user agent cannot contain leading/trailing whitespace or newlines") + } + if o.HelloTimeout < 0 { + return fmt.Errorf("DTS Hello timeout cannot be negative") + } + if o.MaxReceiveMessageSize < 0 { + return fmt.Errorf("DTS maximum receive message size cannot be negative") + } + if o.MaxReceiveMessageSize > 0 && o.MaxReceiveMessageSize < minimumGRPCMessageSize { + return fmt.Errorf("DTS maximum receive message size cannot be less than %d", minimumGRPCMessageSize) + } + if o.MaxSendMessageSize < 0 { + return fmt.Errorf("DTS maximum send message size cannot be negative") + } + if o.MaxSendMessageSize > 0 && o.MaxSendMessageSize < minimumGRPCMessageSize { + return fmt.Errorf("DTS maximum send message size cannot be less than %d", minimumGRPCMessageSize) + } + if o.KeepaliveTime < 0 { + return fmt.Errorf("DTS keepalive time cannot be negative") + } + if o.KeepaliveTime > 0 && o.KeepaliveTime < minimumKeepaliveTime { + return fmt.Errorf("DTS keepalive time cannot be less than %s", minimumKeepaliveTime) + } + if o.KeepaliveTimeout < 0 { + return fmt.Errorf("DTS keepalive timeout cannot be negative") + } + if o.KeepaliveTime > 0 && o.KeepaliveTimeout >= o.KeepaliveTime { + return fmt.Errorf("DTS keepalive timeout must be less than keepalive time") + } + if o.MaximumTimerInterval < 0 { + return fmt.Errorf("DTS maximum timer interval cannot be negative") + } + if o.ChannelRecreateMinInterval < 0 { + return fmt.Errorf("DTS channel recreate minimum interval cannot be negative") + } + if strings.ContainsAny(o.ResourceID, "\r\n") { + return fmt.Errorf("DTS resource ID cannot contain newlines") + } + if o.ResourceID != "" && strings.TrimSpace(o.ResourceID) == "" { + return fmt.Errorf("DTS resource ID cannot be blank") + } + if _, err := api.NormalizeLargePayloadOptions(o.LargePayloads); err != nil { + return fmt.Errorf("invalid DTS large payload options: %w", err) + } + if o.Versioning != nil { + if err := o.Versioning.Validate(); err != nil { + return fmt.Errorf("invalid DTS versioning options: %w", err) + } + } + for _, interceptor := range o.UnaryInterceptors { + if interceptor == nil { + return fmt.Errorf("DTS unary interceptors cannot contain nil") + } + } + for _, interceptor := range o.StreamInterceptors { + if interceptor == nil { + return fmt.Errorf("DTS stream interceptors cannot contain nil") + } + } + + endpoint, err := normalizeEndpoint(o.EndpointAddress) + if err != nil { + return err + } + authentication := o.Authentication + if authentication == "" { + authentication = AuthenticationDefaultAzure + } + switch authentication { + case AuthenticationNone: + if o.Credential != nil { + return fmt.Errorf("DTS credential must be nil when Authentication is None") + } + if fields := o.identityFieldsInUse(); len(fields) > 0 { + return fmt.Errorf( + "DTS Authentication None does not use %s", + strings.Join(fields, ", "), + ) + } + case AuthenticationDefaultAzure: + if o.Credential != nil { + return fmt.Errorf("use Authentication TokenCredential for an explicit DTS credential") + } + case AuthenticationManagedIdentity, + AuthenticationWorkloadIdentity, + AuthenticationEnvironment, + AuthenticationAzureCLI, + AuthenticationAzurePowerShell, + AuthenticationInteractiveBrowser: + if o.Credential != nil { + return fmt.Errorf("DTS credential must be nil when Authentication is %s", o.Authentication) + } + case AuthenticationTokenCredential: + if o.Credential == nil { + return fmt.Errorf("DTS TokenCredential authentication requires a credential") + } + if fields := o.identityFieldsInUse(); len(fields) > 0 { + return fmt.Errorf( + "DTS Authentication TokenCredential does not use %s; configure them on the credential itself", + strings.Join(fields, ", "), + ) + } + default: + return fmt.Errorf("unsupported DTS authentication type %q", o.Authentication) + } + if endpoint.Scheme == "http" { + if !o.AllowInsecureConnection { + return fmt.Errorf("plaintext DTS endpoint requires AllowInsecureConnection") + } + if authentication != AuthenticationNone { + return fmt.Errorf("plaintext DTS endpoint cannot be used with credentials") + } + } + return nil +} + +// identityFieldsInUse lists the Azure Identity configuration fields that carry a +// value. It is used to reject configuration that the selected authentication +// mode can never consume. +func (o *Options) identityFieldsInUse() []string { + var fields []string + if strings.TrimSpace(o.ClientID) != "" { + fields = append(fields, "ClientID") + } + if strings.TrimSpace(o.TenantID) != "" { + fields = append(fields, "TenantID") + } + if strings.TrimSpace(o.TokenFilePath) != "" { + fields = append(fields, "TokenFilePath") + } + if len(normalizeAdditionallyAllowedTenants(o.AdditionallyAllowedTenants)) > 0 { + fields = append(fields, "AdditionallyAllowedTenants") + } + return fields +} + +func normalizeEndpoint(endpointAddress string) (*url.URL, error) { + endpointAddress = strings.TrimSpace(endpointAddress) + if endpointAddress == "" { + return nil, fmt.Errorf("DTS endpoint is required") + } + if !strings.Contains(endpointAddress, "://") { + endpointAddress = "https://" + endpointAddress + } + endpoint, err := url.Parse(endpointAddress) + if err != nil { + return nil, fmt.Errorf("invalid DTS endpoint: %w", err) + } + if endpoint.Scheme != "https" && endpoint.Scheme != "http" { + return nil, fmt.Errorf("DTS endpoint scheme must be https or http") + } + if endpoint.Hostname() == "" { + return nil, fmt.Errorf("DTS endpoint must include a host") + } + if endpoint.User != nil || endpoint.RawQuery != "" || endpoint.Fragment != "" { + return nil, fmt.Errorf("DTS endpoint cannot include user info, query parameters, or a fragment") + } + if endpoint.Path != "" && endpoint.Path != "/" { + return nil, fmt.Errorf("DTS endpoint cannot include a path") + } + return endpoint, nil +} diff --git a/durabletaskscheduler/options_test.go b/durabletaskscheduler/options_test.go new file mode 100644 index 00000000..d27bd465 --- /dev/null +++ b/durabletaskscheduler/options_test.go @@ -0,0 +1,661 @@ +package durabletaskscheduler + +import ( + "testing" + "time" + + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestNewOptionsFromConnectionString(t *testing.T) { + tests := []struct { + name string + value string + wantEndpoint string + wantTaskHub string + wantAuth AuthenticationType + wantUnsafe bool + wantClientID string + wantTenantID string + wantTokenFile string + wantTenants []string + wantErr string + }{ + { + name: "default azure", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=DefaultAzure", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationDefaultAzure, + }, + { + name: "default azure with tenant scoping", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=DefaultAzure;TenantID=tenant;AdditionallyAllowedTenants=*", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationDefaultAzure, + wantTenantID: "tenant", + wantTenants: []string{"*"}, + }, + { + name: "emulator", + value: "Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None", + wantEndpoint: "http://127.0.0.1:8080", + wantTaskHub: "default", + wantAuth: AuthenticationNone, + wantUnsafe: true, + }, + { + name: "managed identity", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=ManagedIdentity;ClientID=client", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationManagedIdentity, + wantClientID: "client", + }, + { + name: "system-assigned managed identity", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=ManagedIdentity", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationManagedIdentity, + }, + { + name: "workload identity", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=WorkloadIdentity;ClientID=client;TenantID=tenant;TokenFilePath=/token;AdditionallyAllowedTenants=one, two", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationWorkloadIdentity, + wantClientID: "client", + wantTenantID: "tenant", + wantTokenFile: "/token", + wantTenants: []string{"one", "two"}, + }, + { + name: "environment", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=Environment", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationEnvironment, + }, + { + name: "Azure CLI", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=AzureCLI;TenantID=tenant", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationAzureCLI, + wantTenantID: "tenant", + }, + { + name: "Azure PowerShell", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=AzurePowerShell;TenantID=tenant", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationAzurePowerShell, + wantTenantID: "tenant", + }, + { + name: "interactive browser", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=InteractiveBrowser;ClientID=client;TenantID=tenant", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationInteractiveBrowser, + wantClientID: "client", + wantTenantID: "tenant", + }, + { + name: "surrounding whitespace and empty segments are ignored", + value: " Endpoint = scheduler.example.com ;; TaskHub =\thub\t; Authentication = AzureCLI ;", + wantEndpoint: "scheduler.example.com", + wantTaskHub: "hub", + wantAuth: AuthenticationAzureCLI, + }, + { + name: "additionally allowed tenants drops empty entries", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=AzureCLI;AdditionallyAllowedTenants=one,, ,two,", + wantAuth: AuthenticationAzureCLI, + wantTenants: []string{"one", "two"}, + }, + { + name: "empty additionally allowed tenants", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=AzureCLI;AdditionallyAllowedTenants=", + wantAuth: AuthenticationAzureCLI, + }, + { + name: "unsupported Visual Studio", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=VisualStudio", + wantErr: "no Azure Identity for Go equivalent", + }, + { + name: "unsupported Visual Studio Code", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=VisualStudioCode", + wantErr: "no Azure Identity for Go equivalent", + }, + { + name: "TokenCredential is programmatic only", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=TokenCredential", + wantErr: "NewOptionsWithCredential", + }, + { + name: "missing authentication", + value: "Endpoint=https://scheduler.example.com;TaskHub=hub", + wantErr: "missing required Authentication", + }, + { + name: "blank authentication", + value: "Endpoint=https://scheduler.example.com;TaskHub=hub;Authentication= ", + wantErr: "missing required Authentication", + }, + { + name: "missing endpoint", + value: "TaskHub=hub;Authentication=None", + wantErr: "missing required Endpoint", + }, + { + name: "blank endpoint", + value: "Endpoint= ;TaskHub=hub;Authentication=None", + wantErr: "missing required Endpoint", + }, + { + name: "missing task hub", + value: "Endpoint=https://scheduler.example.com;Authentication=None", + wantErr: "missing required TaskHub", + }, + { + name: "blank task hub", + value: "Endpoint=https://scheduler.example.com;TaskHub= ;Authentication=None", + wantErr: "missing required TaskHub", + }, + { + name: "empty connection string", + value: "", + wantErr: "missing required Endpoint", + }, + { + name: "segment without a separator", + value: "Endpoint=https://scheduler.example.com;TaskHub;Authentication=None", + wantErr: `invalid connection string segment "TaskHub"`, + }, + { + name: "segment with an empty key", + value: "Endpoint=https://scheduler.example.com;=orphan;TaskHub=hub;Authentication=None", + wantErr: `unsupported connection string key ""`, + }, + { + name: "unknown key", + value: "Endpoint=https://scheduler.example.com;TaskHub=hub;Authentication=None;Typo=value", + wantErr: `unsupported connection string key "Typo"`, + }, + { + name: "unsupported authentication", + value: "Endpoint=https://scheduler.example.com;TaskHub=hub;Authentication=Password", + wantErr: `unsupported Authentication value "Password"`, + }, + { + name: "endpoint with a query string", + value: "Endpoint=https://scheduler.example.com?tenant=a;TaskHub=hub;Authentication=None", + wantErr: "query parameters", + }, + { + name: "endpoint with a path", + value: "Endpoint=https://scheduler.example.com/hub;TaskHub=hub;Authentication=None", + wantErr: "cannot include a path", + }, + { + name: "identity fields are rejected for None", + value: "Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None;ClientID=client", + wantErr: "Authentication None does not use ClientID", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options, err := NewOptionsFromConnectionString(tt.value) + if tt.wantErr != "" { + require.Nil(t, options) + require.ErrorContains(t, err, tt.wantErr) + return + } + require.NoError(t, err) + if tt.wantEndpoint != "" { + require.Equal(t, tt.wantEndpoint, options.EndpointAddress) + } + if tt.wantTaskHub != "" { + require.Equal(t, tt.wantTaskHub, options.TaskHubName) + } + require.Equal(t, tt.wantAuth, options.Authentication) + require.Equal(t, tt.wantUnsafe, options.AllowInsecureConnection) + require.Nil(t, options.Credential) + require.Equal(t, DefaultResourceID, options.ResourceID) + require.Equal(t, task.DefaultMaximumTimerInterval, options.MaximumTimerInterval) + require.Equal(t, tt.wantClientID, options.ClientID) + require.Equal(t, tt.wantTenantID, options.TenantID) + require.Equal(t, tt.wantTokenFile, options.TokenFilePath) + require.Equal(t, tt.wantTenants, options.AdditionallyAllowedTenants) + require.Equal(t, 30*time.Second, options.HelloTimeout) + require.Equal(t, DefaultMaxReceiveMessageSize, options.MaxReceiveMessageSize) + require.Equal(t, DefaultMaxSendMessageSize, options.MaxSendMessageSize) + require.Equal(t, DefaultKeepaliveTime, options.KeepaliveTime) + require.Equal(t, DefaultKeepaliveTimeout, options.KeepaliveTimeout) + require.Equal(t, 5, options.ChannelRecreateFailureThreshold) + require.Equal(t, 30*time.Second, options.ChannelRecreateMinInterval) + require.Empty(t, options.WorkerID) + require.Empty(t, options.UserAgent) + require.NoError(t, options.Validate()) + }) + } +} + +// TestNewOptionsFromConnectionStringKeysAreCaseInsensitive covers every +// supported key in lower, upper, and mixed case. +func TestNewOptionsFromConnectionStringKeysAreCaseInsensitive(t *testing.T) { + tests := []struct { + name string + value string + }{ + { + name: "canonical", + value: "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=WorkloadIdentity;ClientID=client;TenantID=tenant;TokenFilePath=/token;AdditionallyAllowedTenants=one", + }, + { + name: "lower", + value: "endpoint=scheduler.example.com;taskhub=hub;authentication=WorkloadIdentity;clientid=client;tenantid=tenant;tokenfilepath=/token;additionallyallowedtenants=one", + }, + { + name: "upper", + value: "ENDPOINT=scheduler.example.com;TASKHUB=hub;AUTHENTICATION=WorkloadIdentity;CLIENTID=client;TENANTID=tenant;TOKENFILEPATH=/token;ADDITIONALLYALLOWEDTENANTS=one", + }, + { + name: "mixed", + value: "EnDpOiNt=scheduler.example.com;tAsKhUb=hub;AuThEnTiCaTiOn=WorkloadIdentity;cLiEnTiD=client;TeNaNtId=tenant;ToKeNfIlEpAtH=/token;AdDiTiOnAlLyAlLoWeDtEnAnTs=one", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options, err := NewOptionsFromConnectionString(tt.value) + require.NoError(t, err) + require.Equal(t, "scheduler.example.com", options.EndpointAddress) + require.Equal(t, "hub", options.TaskHubName) + require.Equal(t, AuthenticationWorkloadIdentity, options.Authentication) + require.Equal(t, "client", options.ClientID) + require.Equal(t, "tenant", options.TenantID) + require.Equal(t, "/token", options.TokenFilePath) + require.Equal(t, []string{"one"}, options.AdditionallyAllowedTenants) + }) + } +} + +// TestNewOptionsFromConnectionStringAuthenticationValuesAreCaseInsensitive +// covers every mode a connection string can name. +func TestNewOptionsFromConnectionStringAuthenticationValuesAreCaseInsensitive(t *testing.T) { + tests := []struct { + values []string + want AuthenticationType + }{ + {values: []string{"None", "none", "NONE", "nOnE"}, want: AuthenticationNone}, + { + values: []string{"DefaultAzure", "defaultazure", "DEFAULTAZURE", "dEfAuLtAzUrE"}, + want: AuthenticationDefaultAzure, + }, + { + values: []string{"ManagedIdentity", "managedidentity", "MANAGEDIDENTITY"}, + want: AuthenticationManagedIdentity, + }, + { + values: []string{"WorkloadIdentity", "workloadidentity", "WORKLOADIDENTITY"}, + want: AuthenticationWorkloadIdentity, + }, + {values: []string{"Environment", "environment", "ENVIRONMENT"}, want: AuthenticationEnvironment}, + {values: []string{"AzureCLI", "azurecli", "AZURECLI", "AzureCli"}, want: AuthenticationAzureCLI}, + { + values: []string{"AzurePowerShell", "azurepowershell", "AZUREPOWERSHELL"}, + want: AuthenticationAzurePowerShell, + }, + { + values: []string{"InteractiveBrowser", "interactivebrowser", "INTERACTIVEBROWSER"}, + want: AuthenticationInteractiveBrowser, + }, + } + + for _, tt := range tests { + t.Run(string(tt.want), func(t *testing.T) { + for _, value := range tt.values { + endpoint := "scheduler.example.com" + if tt.want == AuthenticationNone { + endpoint = "http://127.0.0.1:8080" + } + options, err := NewOptionsFromConnectionString( + "Endpoint=" + endpoint + ";TaskHub=hub;Authentication=" + value, + ) + require.NoErrorf(t, err, "authentication value %q", value) + require.Equalf(t, tt.want, options.Authentication, "authentication value %q", value) + } + }) + } +} + +// TestNewOptionsFromConnectionStringDuplicateKeysUseLastValue matches the .NET +// connection-string builder, where a repeated key overwrites earlier values. +func TestNewOptionsFromConnectionStringDuplicateKeysUseLastValue(t *testing.T) { + options, err := NewOptionsFromConnectionString( + "Endpoint=first.example.com;TaskHub=first;Authentication=DefaultAzure;ClientID=first;" + + "TenantID=first;TokenFilePath=/first;AdditionallyAllowedTenants=first;" + + "endpoint=second.example.com;TASKHUB=second;authentication=WorkloadIdentity;" + + "clientid=second;TenantId=second;tokenfilepath=/second;AdditionallyAllowedTenants=second", + ) + require.NoError(t, err) + require.Equal(t, "second.example.com", options.EndpointAddress) + require.Equal(t, "second", options.TaskHubName) + require.Equal(t, AuthenticationWorkloadIdentity, options.Authentication) + require.Equal(t, "second", options.ClientID) + require.Equal(t, "second", options.TenantID) + require.Equal(t, "/second", options.TokenFilePath) + require.Equal(t, []string{"second"}, options.AdditionallyAllowedTenants) +} + +// TestNewOptionsFromConnectionStringDuplicateKeyCanBlankAValue confirms the last +// value wins even when it clears an earlier one. +func TestNewOptionsFromConnectionStringDuplicateKeyCanBlankAValue(t *testing.T) { + options, err := NewOptionsFromConnectionString( + "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=ManagedIdentity;ClientID=client;ClientID=", + ) + require.NoError(t, err) + require.Empty(t, options.ClientID) + + _, err = NewOptionsFromConnectionString( + "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=AzureCLI;Authentication=", + ) + require.ErrorContains(t, err, "missing required Authentication") +} + +// TestNewOptionsFromConnectionStringValuesMayContainSeparators documents that +// only the first '=' separates a key from its value. +func TestNewOptionsFromConnectionStringValuesMayContainSeparators(t *testing.T) { + options, err := NewOptionsFromConnectionString( + "Endpoint=scheduler.example.com;TaskHub=hub;Authentication=WorkloadIdentity;TokenFilePath=/run/secrets/token=v2", + ) + require.NoError(t, err) + require.Equal(t, "/run/secrets/token=v2", options.TokenFilePath) +} + +// TestNewOptionsFromConnectionStringDoesNotForceInsecureForHTTPS keeps the +// insecure opt-in tied to Authentication=None only. +func TestNewOptionsFromConnectionStringDoesNotForceInsecureForHTTPS(t *testing.T) { + options, err := NewOptionsFromConnectionString( + "Endpoint=https://scheduler.example.com;TaskHub=hub;Authentication=DefaultAzure", + ) + require.NoError(t, err) + require.False(t, options.AllowInsecureConnection) +} + +func TestNewOptionsDefaults(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + require.Equal(t, "scheduler.example.com", options.EndpointAddress) + require.Equal(t, "hub", options.TaskHubName) + require.Equal(t, AuthenticationDefaultAzure, options.Authentication) + require.Equal(t, DefaultResourceID, options.ResourceID) + require.Equal(t, 30*time.Second, options.HelloTimeout) + require.Equal(t, DefaultMaxReceiveMessageSize, options.MaxReceiveMessageSize) + require.Equal(t, DefaultMaxSendMessageSize, options.MaxSendMessageSize) + require.Equal(t, DefaultKeepaliveTime, options.KeepaliveTime) + require.Equal(t, DefaultKeepaliveTimeout, options.KeepaliveTimeout) + require.Equal(t, task.DefaultMaximumTimerInterval, options.MaximumTimerInterval) + require.Equal(t, 5, options.ChannelRecreateFailureThreshold) + require.Equal(t, 30*time.Second, options.ChannelRecreateMinInterval) + require.False(t, options.AllowInsecureConnection) + require.Nil(t, options.Credential) + require.NoError(t, options.Validate()) +} + +func TestOptionsValidatePlaintextGuard(t *testing.T) { + options := NewOptions("http://127.0.0.1:8080", "default") + options.AllowInsecureConnection = true + require.ErrorContains(t, options.Validate(), "cannot be used with credentials") + + options.Authentication = AuthenticationNone + options.AllowInsecureConnection = false + require.ErrorContains(t, options.Validate(), "AllowInsecureConnection") + + options.AllowInsecureConnection = true + require.NoError(t, options.Validate()) +} + +func TestOptionsValidateEndpoint(t *testing.T) { + tests := []struct { + endpoint string + wantErr string + }{ + {endpoint: "scheduler.example.com"}, + {endpoint: "https://scheduler.example.com:443"}, + {endpoint: "scheduler.example.com:443"}, + {endpoint: " scheduler.example.com "}, + {endpoint: "https://scheduler.example.com/"}, + {endpoint: "", wantErr: "endpoint is required"}, + {endpoint: " ", wantErr: "endpoint is required"}, + {endpoint: "ftp://scheduler.example.com", wantErr: "scheme"}, + {endpoint: "grpc://scheduler.example.com", wantErr: "scheme"}, + {endpoint: "https://", wantErr: "must include a host"}, + {endpoint: "https://scheduler.example.com/path", wantErr: "path"}, + {endpoint: "https://user@scheduler.example.com", wantErr: "user info"}, + {endpoint: "https://user:pass@scheduler.example.com", wantErr: "user info"}, + {endpoint: "https://scheduler.example.com?a=b", wantErr: "query parameters"}, + {endpoint: "https://scheduler.example.com#frag", wantErr: "fragment"}, + {endpoint: "https://scheduler.example.com:notaport", wantErr: "invalid DTS endpoint"}, + } + + for _, tt := range tests { + t.Run(tt.endpoint, func(t *testing.T) { + options := NewOptions(tt.endpoint, "hub") + err := options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + +// TestOptionsValidateTaskHubWorkerIDAndUserAgent covers the header-injection and +// required-field guards for values placed on outgoing metadata. +func TestOptionsValidateTaskHubWorkerIDAndUserAgent(t *testing.T) { + tests := []struct { + name string + taskHub string + workerID string + userAgent string + wantErr string + }{ + {name: "defaults", taskHub: "hub"}, + {name: "explicit worker ID and user agent", taskHub: "hub", workerID: "worker", userAgent: "agent/1.0"}, + {name: "missing task hub", wantErr: "task hub name is required"}, + {name: "blank task hub", taskHub: " ", wantErr: "task hub name is required"}, + {name: "padded task hub", taskHub: " hub", wantErr: "leading or trailing whitespace"}, + {name: "task hub newline", taskHub: "hub\nx", wantErr: "cannot contain newlines"}, + {name: "task hub carriage return", taskHub: "hub\rx", wantErr: "cannot contain newlines"}, + {name: "padded worker ID", taskHub: "hub", workerID: "worker ", wantErr: "worker ID"}, + {name: "worker ID newline", taskHub: "hub", workerID: "worker\nid", wantErr: "worker ID"}, + {name: "padded user agent", taskHub: "hub", userAgent: " agent", wantErr: "user agent"}, + {name: "user agent injection", taskHub: "hub", userAgent: "agent\r\nx: y", wantErr: "user agent"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", tt.taskHub) + options.WorkerID = tt.workerID + options.UserAgent = tt.userAgent + err := options.Validate() + if tt.wantErr == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantErr) + } + }) + } +} + +func TestOptionsValidateNilReceiver(t *testing.T) { + var options *Options + require.ErrorContains(t, options.Validate(), "options are required") + + _, err := prepareOptions(nil) + require.ErrorContains(t, err, "options are required") +} + +func TestPrepareOptionsAppliesDefaultsAndCopiesTenants(t *testing.T) { + options := &Options{ + EndpointAddress: "scheduler.example.com", + TaskHubName: "hub", + Authentication: AuthenticationDefaultAzure, + AdditionallyAllowedTenants: []string{"tenant"}, + } + prepared, err := prepareOptions(options) + require.NoError(t, err) + require.Equal(t, DefaultResourceID, prepared.ResourceID) + require.Equal(t, 30*time.Second, prepared.HelloTimeout) + require.Equal(t, DefaultMaxReceiveMessageSize, prepared.MaxReceiveMessageSize) + require.Equal(t, DefaultMaxSendMessageSize, prepared.MaxSendMessageSize) + require.Equal(t, task.DefaultMaximumTimerInterval, prepared.MaximumTimerInterval) + options.AdditionallyAllowedTenants[0] = "changed" + require.Equal(t, []string{"tenant"}, prepared.AdditionallyAllowedTenants) +} + +// TestPrepareOptionsResourceIDNormalization pins the documented contract: +// exactly empty defaults, surrounding whitespace on a real value is trimmed, +// and a whitespace-only value is rejected instead of collapsing to the default. +func TestPrepareOptionsResourceIDNormalization(t *testing.T) { + for _, test := range []struct { + name string + resourceID string + want string + wantScope string + wantErr string + }{ + { + name: "empty defaults", + want: DefaultResourceID, + wantScope: DefaultResourceID + "/.default", + }, + { + name: "surrounding whitespace is trimmed", + resourceID: " https://custom.example.com ", + want: "https://custom.example.com", + wantScope: "https://custom.example.com/.default", + }, + { + name: "exact value is preserved", + resourceID: "https://custom.example.com", + want: "https://custom.example.com", + wantScope: "https://custom.example.com/.default", + }, + {name: "spaces only", resourceID: " ", wantErr: "resource ID cannot be blank"}, + {name: "tab only", resourceID: "\t", wantErr: "resource ID cannot be blank"}, + {name: "newline only", resourceID: "\n", wantErr: "resource ID cannot contain newlines"}, + } { + t.Run(test.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.ResourceID = test.resourceID + options.Authentication = AuthenticationNone + options.EndpointAddress = "http://127.0.0.1:8080" + options.AllowInsecureConnection = true + + prepared, err := prepareOptions(options) + if test.wantErr != "" { + require.ErrorContains(t, err, test.wantErr) + return + } + require.NoError(t, err) + require.Equal(t, test.want, prepared.ResourceID) + require.Equal(t, test.wantScope, newPerRPCCredentials(&prepared, clientRole, "").scope) + }) + } +} + +func TestOptionsValidateRejectsInvalidUserAgent(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.UserAgent = "agent\r\ninjected" + require.ErrorContains(t, options.Validate(), "user agent") +} + +func TestOptionsValidateRejectsNegativeMaximumTimerInterval(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.MaximumTimerInterval = -time.Second + require.ErrorContains(t, options.Validate(), "maximum timer interval") +} + +func TestOptionsValidateRejectsNegativeChannelRecreateInterval(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.ChannelRecreateMinInterval = -time.Second + require.ErrorContains(t, options.Validate(), "channel recreate minimum interval") +} + +func TestOptionsValidateRejectsInvalidTransportLimits(t *testing.T) { + tests := []struct { + name string + mutate func(*Options) + message string + }{ + { + name: "negative receive size", + mutate: func(options *Options) { options.MaxReceiveMessageSize = -1 }, + message: "maximum receive message size", + }, + { + name: "receive size below minimum", + mutate: func(options *Options) { options.MaxReceiveMessageSize = minimumGRPCMessageSize - 1 }, + message: "maximum receive message size", + }, + { + name: "negative send size", + mutate: func(options *Options) { options.MaxSendMessageSize = -1 }, + message: "maximum send message size", + }, + { + name: "send size below minimum", + mutate: func(options *Options) { options.MaxSendMessageSize = minimumGRPCMessageSize - 1 }, + message: "maximum send message size", + }, + { + name: "negative keepalive time", + mutate: func(options *Options) { options.KeepaliveTime = -time.Second }, + message: "keepalive time", + }, + { + name: "aggressive keepalive time", + mutate: func(options *Options) { options.KeepaliveTime = minimumKeepaliveTime - time.Second }, + message: "keepalive time", + }, + { + name: "negative keepalive timeout", + mutate: func(options *Options) { options.KeepaliveTimeout = -time.Second }, + message: "keepalive timeout", + }, + { + name: "keepalive timeout not below interval", + mutate: func(options *Options) { + options.KeepaliveTimeout = options.KeepaliveTime + }, + message: "must be less than keepalive time", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + test.mutate(options) + require.ErrorContains(t, options.Validate(), test.message) + }) + } +} + +func TestOptionsValidateRejectsNilInterceptors(t *testing.T) { + options := NewOptions("scheduler.example.com", "hub") + options.UnaryInterceptors = []grpc.UnaryClientInterceptor{nil} + require.ErrorContains(t, options.Validate(), "unary interceptors") + + options.UnaryInterceptors = nil + options.StreamInterceptors = []grpc.StreamClientInterceptor{nil} + require.ErrorContains(t, options.Validate(), "stream interceptors") +} diff --git a/durabletaskscheduler/scheduled_tasks.go b/durabletaskscheduler/scheduled_tasks.go new file mode 100644 index 00000000..a8ca3ec0 --- /dev/null +++ b/durabletaskscheduler/scheduled_tasks.go @@ -0,0 +1,1642 @@ +package durabletaskscheduler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "math/bits" + "reflect" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/tagcodec" + "github.com/microsoft/durabletask-go/task" +) + +const ( + // ScheduleEntityName is the system entity name used to persist schedules. + ScheduleEntityName = "Schedule" + // ExecuteScheduleOperationOrchestratorName is the system orchestrator used + // for schedule mutations. + ExecuteScheduleOperationOrchestratorName = "ExecuteScheduleOperationOrchestrator" + // ExecuteScheduledTaskOrchestratorName applies orchestration-only options + // such as tags, context fields, and retries to a scheduled target. + ExecuteScheduledTaskOrchestratorName = "ExecuteScheduledTaskOrchestrator" + + createScheduleOperation = "CreateSchedule" + updateScheduleOperation = "UpdateSchedule" + pauseScheduleOperation = "PauseSchedule" + resumeScheduleOperation = "ResumeSchedule" + runScheduleOperation = "RunSchedule" + deleteScheduleOperation = "delete" +) + +var ( + // ErrScheduleNotFound identifies attempts to access a missing schedule. + ErrScheduleNotFound = errors.New("schedule not found") + // ErrScheduleInvalidTransition identifies invalid schedule state changes. + ErrScheduleInvalidTransition = errors.New("invalid schedule state transition") + // ErrScheduleValidation identifies invalid schedule configuration. + ErrScheduleValidation = errors.New("invalid schedule configuration") + // ErrScheduleOperationFailed identifies a failed schedule mutation. + ErrScheduleOperationFailed = errors.New("schedule operation failed") +) + +const ( + scheduleValidationErrorType api.ErrorType = "ScheduleClientValidationException" + scheduleInvalidTransitionType api.ErrorType = "ScheduleInvalidTransitionException" + scheduleOperationFailedErrorType api.ErrorType = "ScheduleOperationFailed" +) + +// ScheduleStatus is the persisted status of a schedule. +type ScheduleStatus int + +const ( + ScheduleStatusUninitialized ScheduleStatus = iota + ScheduleStatusActive + ScheduleStatusPaused +) + +// ScheduleNotFoundError is returned when a schedule does not exist. +type ScheduleNotFoundError struct{ ScheduleID string } + +func (e *ScheduleNotFoundError) Error() string { + return fmt.Sprintf("schedule with ID %q was not found", e.ScheduleID) +} +func (e *ScheduleNotFoundError) Unwrap() error { return ErrScheduleNotFound } + +// ScheduleInvalidTransitionError is returned for an invalid state transition. +type ScheduleInvalidTransitionError struct { + ScheduleID string + From ScheduleStatus + To ScheduleStatus + Operation string +} + +func (e *ScheduleInvalidTransitionError) Error() string { + return fmt.Sprintf("invalid state transition attempted for schedule %q: cannot transition from %s to %s during %s operation", + e.ScheduleID, e.From, e.To, e.Operation) +} +func (e *ScheduleInvalidTransitionError) Unwrap() error { return ErrScheduleInvalidTransition } + +// ScheduleValidationError is returned when schedule options are invalid. +type ScheduleValidationError struct{ Message string } + +func (e *ScheduleValidationError) Error() string { + return "schedule validation failed: " + e.Message +} +func (e *ScheduleValidationError) Unwrap() error { return ErrScheduleValidation } + +func (*ScheduleValidationError) DurableTaskErrorType() api.ErrorType { + return scheduleValidationErrorType +} + +func (e *ScheduleValidationError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"message": e.Message} +} + +func (*ScheduleInvalidTransitionError) DurableTaskErrorType() api.ErrorType { + return scheduleInvalidTransitionType +} + +func (e *ScheduleInvalidTransitionError) DurableTaskErrorProperties() map[string]any { + return map[string]any{ + "scheduleId": e.ScheduleID, + "from": int(e.From), + "to": int(e.To), + "operation": e.Operation, + } +} + +// ScheduleOperationError describes a schedule mutation orchestration that did +// not complete successfully. +type ScheduleOperationError struct { + ScheduleID string + Operation string + RuntimeStatus api.OrchestrationStatus + FailureDetails *api.FailureDetails +} + +func (e *ScheduleOperationError) Error() string { + message := "" + if e.FailureDetails != nil { + message = e.FailureDetails.ErrorMessage + } + if message == "" { + return fmt.Sprintf("schedule %q operation %s failed with status %v", e.ScheduleID, e.Operation, e.RuntimeStatus) + } + return fmt.Sprintf("schedule %q operation %s failed: %s", e.ScheduleID, e.Operation, message) +} + +func (e *ScheduleOperationError) Unwrap() error { return ErrScheduleOperationFailed } + +func (*ScheduleOperationError) DurableTaskErrorType() api.ErrorType { + return scheduleOperationFailedErrorType +} + +func (s ScheduleStatus) String() string { + switch s { + case ScheduleStatusUninitialized: + return "Uninitialized" + case ScheduleStatusActive: + return "Active" + case ScheduleStatusPaused: + return "Paused" + default: + return fmt.Sprintf("ScheduleStatus(%d)", s) + } +} + +// ScheduleCreationOptions configures a recurring interval schedule. +// +// OrchestrationInput is already serialized input. TypedOrchestrationInput is +// serialized by the configured DataConverter and takes precedence when non-nil. +// Effective orchestration instance IDs must not use the reserved entity format. +// Retry-backed schedules generate separate instance IDs for each attempt. +// Cron schedules are not supported. +type ScheduleCreationOptions struct { + ScheduleID string + OrchestrationName string + Interval time.Duration + OrchestrationInput string + TypedOrchestrationInput any + OrchestrationInstanceID string + OrchestrationVersion string + StartAt time.Time + EndAt time.Time + StartImmediatelyIfLate bool + Tags map[string]string + ContextFields api.ContextFields + RetryPolicy *ScheduleRetryPolicy +} + +// ScheduleUpdateOptions applies a sparse update to a schedule. Nil pointers +// mean the corresponding field is unchanged. Empty string values are ignored. +// Nil maps leave stored values unchanged; non-nil empty maps clear them. +type ScheduleUpdateOptions struct { + OrchestrationName *string + OrchestrationInput *string + TypedOrchestrationInput any + OrchestrationInstanceID *string + OrchestrationVersion *string + StartAt *time.Time + EndAt *time.Time + Interval *time.Duration + StartImmediatelyIfLate *bool + Tags map[string]string + ContextFields api.ContextFields + RetryPolicy *ScheduleRetryPolicy +} + +// ScheduleRetryPolicy is the serializable subset of [task.RetryPolicy] used +// when a scheduled target orchestration fails. +type ScheduleRetryPolicy struct { + MaxAttempts int + InitialRetryInterval time.Duration + BackoffCoefficient float64 + MaxRetryInterval time.Duration + RetryTimeout time.Duration +} + +// scheduleCreationWire is the .NET-compatible JSON shape of +// ScheduleCreationOptions. `omitempty` only affects marshaling, so the same +// shape decodes payloads written by other DTS SDKs. +type scheduleCreationWire struct { + ScheduleID string `json:"ScheduleId"` + OrchestrationName string `json:"OrchestrationName"` + Interval dotNetSpan `json:"Interval"` + OrchestrationInput string `json:"OrchestrationInput,omitempty"` + TypedOrchestrationInput any `json:"TypedOrchestrationInput,omitempty"` + OrchestrationInstanceID string `json:"OrchestrationInstanceId,omitempty"` + OrchestrationVersion string `json:"OrchestrationVersion,omitempty"` + StartAt *time.Time `json:"StartAt,omitempty"` + EndAt *time.Time `json:"EndAt,omitempty"` + StartImmediatelyIfLate bool `json:"StartImmediatelyIfLate"` + Tags map[string]string `json:"Tags,omitempty"` + ContextFields api.ContextFields `json:"ContextFields,omitempty"` + RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` +} + +// scheduleUpdateWire is the .NET-compatible JSON shape of ScheduleUpdateOptions. +type scheduleUpdateWire struct { + OrchestrationName *string `json:"OrchestrationName,omitempty"` + OrchestrationInput *string `json:"OrchestrationInput,omitempty"` + TypedOrchestrationInput any `json:"TypedOrchestrationInput,omitempty"` + OrchestrationInstanceID *string `json:"OrchestrationInstanceId,omitempty"` + OrchestrationVersion *string `json:"OrchestrationVersion,omitempty"` + StartAt *time.Time `json:"StartAt,omitempty"` + EndAt *time.Time `json:"EndAt,omitempty"` + Interval *dotNetSpan `json:"Interval,omitempty"` + StartImmediatelyIfLate *bool `json:"StartImmediatelyIfLate,omitempty"` + Tags map[string]string `json:"Tags,omitzero"` + ContextFields api.ContextFields `json:"ContextFields,omitzero"` + RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` +} + +func (o ScheduleCreationOptions) MarshalJSON() ([]byte, error) { + retry, err := scheduleRetryPolicyFromPublic(o.RetryPolicy) + if err != nil { + return nil, err + } + return json.Marshal(scheduleCreationWire{ + ScheduleID: o.ScheduleID, + OrchestrationName: o.OrchestrationName, + Interval: dotNetSpan(o.Interval), + OrchestrationInput: o.OrchestrationInput, + TypedOrchestrationInput: o.TypedOrchestrationInput, + OrchestrationInstanceID: o.OrchestrationInstanceID, + OrchestrationVersion: o.OrchestrationVersion, + StartAt: optionalTime(o.StartAt), + EndAt: optionalTime(o.EndAt), + StartImmediatelyIfLate: o.StartImmediatelyIfLate, + Tags: o.Tags, + ContextFields: o.ContextFields, + RetryPolicy: retry, + }) +} + +func (o *ScheduleCreationOptions) UnmarshalJSON(data []byte) error { + var decoded scheduleCreationWire + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *o = ScheduleCreationOptions{ + ScheduleID: decoded.ScheduleID, + OrchestrationName: decoded.OrchestrationName, + Interval: time.Duration(decoded.Interval), + OrchestrationInput: decoded.OrchestrationInput, + TypedOrchestrationInput: decoded.TypedOrchestrationInput, + OrchestrationInstanceID: decoded.OrchestrationInstanceID, + OrchestrationVersion: decoded.OrchestrationVersion, + StartImmediatelyIfLate: decoded.StartImmediatelyIfLate, + Tags: decoded.Tags, + ContextFields: decoded.ContextFields, + RetryPolicy: decoded.RetryPolicy.public(), + } + if decoded.StartAt != nil { + o.StartAt = *decoded.StartAt + } + if decoded.EndAt != nil { + o.EndAt = *decoded.EndAt + } + return nil +} + +func (o ScheduleUpdateOptions) MarshalJSON() ([]byte, error) { + var interval *dotNetSpan + if o.Interval != nil { + value := dotNetSpan(*o.Interval) + interval = &value + } + retry, err := scheduleRetryPolicyFromPublic(o.RetryPolicy) + if err != nil { + return nil, err + } + return json.Marshal(scheduleUpdateWire{ + OrchestrationName: o.OrchestrationName, + OrchestrationInput: o.OrchestrationInput, + TypedOrchestrationInput: o.TypedOrchestrationInput, + OrchestrationInstanceID: o.OrchestrationInstanceID, + OrchestrationVersion: o.OrchestrationVersion, + StartAt: o.StartAt, + EndAt: o.EndAt, + Interval: interval, + StartImmediatelyIfLate: o.StartImmediatelyIfLate, + Tags: o.Tags, + ContextFields: o.ContextFields, + RetryPolicy: retry, + }) +} + +func (o *ScheduleUpdateOptions) UnmarshalJSON(data []byte) error { + var decoded scheduleUpdateWire + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *o = ScheduleUpdateOptions{ + OrchestrationName: decoded.OrchestrationName, + OrchestrationInput: decoded.OrchestrationInput, + TypedOrchestrationInput: decoded.TypedOrchestrationInput, + OrchestrationInstanceID: decoded.OrchestrationInstanceID, + OrchestrationVersion: decoded.OrchestrationVersion, + StartAt: decoded.StartAt, + EndAt: decoded.EndAt, + StartImmediatelyIfLate: decoded.StartImmediatelyIfLate, + Tags: decoded.Tags, + ContextFields: decoded.ContextFields, + RetryPolicy: decoded.RetryPolicy.public(), + } + if decoded.Interval != nil { + value := time.Duration(*decoded.Interval) + o.Interval = &value + } + return nil +} + +// ScheduleDescription describes a schedule's current configuration and state. +type ScheduleDescription struct { + ScheduleID string + OrchestrationName string + OrchestrationInput string + OrchestrationInstanceID string + OrchestrationVersion string + StartAt time.Time + EndAt time.Time + Interval time.Duration + StartImmediatelyIfLate bool + Status ScheduleStatus + ExecutionToken string + LastRunAt time.Time + NextRunAt time.Time + CreatedAt time.Time + LastModifiedAt time.Time + Tags map[string]string + ContextFields api.ContextFields + RetryPolicy *ScheduleRetryPolicy + Converter api.DataConverter `json:"-"` +} + +// ReadInput deserializes the scheduled orchestration input. +func (d *ScheduleDescription) ReadInput(target any) error { + if d == nil || target == nil || d.OrchestrationInput == "" { + return nil + } + return api.NormalizeDataConverter(d.Converter).Deserialize(d.OrchestrationInput, target) +} + +// ScheduleQuery filters a single page of schedules. Filters are applied after +// DTS returns the entity page, so a page can contain fewer than PageSize items. +type ScheduleQuery struct { + Status *ScheduleStatus + ScheduleIDPrefix string + // CreatedFrom is an exclusive lower creation-time bound, matching .NET. + CreatedFrom time.Time + // CreatedTo is an exclusive upper creation-time bound, matching .NET. + CreatedTo time.Time + PageSize int32 + ContinuationToken string +} + +// ScheduleQueryResult is one page of schedules. +type ScheduleQueryResult struct { + Schedules []*ScheduleDescription + ContinuationToken string +} + +// ScheduleOperationRequest is the input consumed by +// ExecuteScheduleOperationOrchestrator. +type ScheduleOperationRequest struct { + EntityID api.EntityID `json:"EntityId"` + OperationName string `json:"OperationName"` + Input any `json:"Input,omitempty"` +} + +// RegisterScheduledTasks registers DTS's system schedule entity and operation +// orchestrator. The operation orchestrator is deliberately unversioned so it +// remains reachable when application default versioning is enabled. +func RegisterScheduledTasks(registry *task.TaskRegistry) error { + return registerScheduledTasks(registry, "") +} + +// RegisterScheduledTasksWithDefaultVersion registers scheduled-task system +// handlers and applies defaultVersion to scheduled target orchestrations that +// do not specify ScheduleCreationOptions.OrchestrationVersion. +func RegisterScheduledTasksWithDefaultVersion(registry *task.TaskRegistry, defaultVersion string) error { + return registerScheduledTasks(registry, defaultVersion) +} + +func registerScheduledTasks(registry *task.TaskRegistry, defaultVersion string) error { + if registry == nil { + return fmt.Errorf("task registry is required") + } + if err := registry.AddEntityN(ScheduleEntityName, func(ctx *task.EntityContext) (any, error) { + return scheduleEntityWithDefaultVersion(ctx, defaultVersion) + }); err != nil { + return err + } + if err := registry.AddOrchestratorN( + ExecuteScheduleOperationOrchestratorName, + ExecuteScheduleOperationOrchestrator, + ); err != nil { + return err + } + return registry.AddOrchestratorN(ExecuteScheduledTaskOrchestratorName, executeScheduledTaskOrchestrator) +} + +// WithScheduledTasks advertises the DTS scheduled-task capability. Use it with +// RegisterScheduledTasks; registrations are then included by auto work-item +// filters derived from the registry. +func WithScheduledTasks() durabletaskclient.TaskHubGrpcWorkerOption { + return durabletaskclient.CombineTaskHubGrpcWorkerOptions( + durabletaskclient.WithScheduledTaskCapability(true), + durabletaskclient.WithUnversionedOrchestratorNames( + ExecuteScheduleOperationOrchestratorName, + ExecuteScheduledTaskOrchestratorName, + ), + ) +} + +// ExecuteScheduleOperationOrchestrator invokes one schedule entity operation. +// It must remain unversioned because schedule clients explicitly target the +// unversioned system orchestrator. +func ExecuteScheduleOperationOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var request ScheduleOperationRequest + if err := ctx.GetInput(&request); err != nil { + return nil, err + } + if request.OperationName == "" { + return nil, &ScheduleValidationError{Message: "operation name is required"} + } + var result any + if err := ctx.CallEntity(request.EntityID, request.OperationName, task.WithEntityInput(request.Input)).Await(&result); err != nil { + return nil, err + } + return result, nil +} + +type scheduledTaskRequest struct { + OrchestrationName string `json:"OrchestrationName"` + OrchestrationInput string `json:"OrchestrationInput,omitempty"` + OrchestrationInstanceID string `json:"OrchestrationInstanceId"` + OrchestrationVersion string `json:"OrchestrationVersion,omitempty"` + Tags map[string]string `json:"Tags,omitempty"` + ContextFields api.ContextFields `json:"ContextFields,omitempty"` + RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` +} + +func executeScheduledTaskOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var request scheduledTaskRequest + if err := ctx.GetInput(&request); err != nil { + return nil, err + } + options := []task.SubOrchestratorOption{} + if request.RetryPolicy == nil { + options = append(options, task.WithSubOrchestrationInstanceID(request.OrchestrationInstanceID)) + } + if request.OrchestrationInput != "" { + options = append(options, task.WithRawSubOrchestratorInput(request.OrchestrationInput)) + } + if request.OrchestrationVersion != "" { + options = append(options, task.WithSubOrchestrationVersion(request.OrchestrationVersion)) + } + if len(request.Tags) > 0 { + options = append(options, task.WithSubOrchestrationTags(request.Tags)) + } + if len(request.ContextFields) > 0 { + options = append(options, task.WithSubOrchestrationContextFields(request.ContextFields)) + } + if retry := request.RetryPolicy.taskPolicy(); retry != nil { + options = append(options, task.WithSubOrchestrationRetryPolicy(retry)) + } + if err := ctx.CallSubOrchestrator(request.OrchestrationName, options...).Await(nil); err != nil { + return nil, err + } + return nil, nil +} + +type scheduleState struct { + Status ScheduleStatus `json:"Status"` + ExecutionToken string `json:"ExecutionToken"` + LastRunAt time.Time `json:"LastRunAt,omitempty"` + NextRunAt time.Time `json:"NextRunAt,omitempty"` + ScheduleCreatedAt time.Time `json:"ScheduleCreatedAt,omitempty"` + ScheduleLastModifiedAt time.Time `json:"ScheduleLastModifiedAt,omitempty"` + ScheduleConfiguration *scheduleConfiguration `json:"ScheduleConfiguration,omitempty"` +} + +func (s scheduleState) MarshalJSON() ([]byte, error) { + type stateJSON struct { + Status ScheduleStatus `json:"Status"` + ExecutionToken string `json:"ExecutionToken"` + LastRunAt *time.Time `json:"LastRunAt"` + NextRunAt *time.Time `json:"NextRunAt"` + ScheduleCreatedAt *time.Time `json:"ScheduleCreatedAt"` + ScheduleLastModifiedAt *time.Time `json:"ScheduleLastModifiedAt"` + ScheduleConfiguration *scheduleConfiguration `json:"ScheduleConfiguration"` + } + return json.Marshal(stateJSON{ + Status: s.Status, + ExecutionToken: s.ExecutionToken, + LastRunAt: optionalTime(s.LastRunAt), + NextRunAt: optionalTime(s.NextRunAt), + ScheduleCreatedAt: optionalTime(s.ScheduleCreatedAt), + ScheduleLastModifiedAt: optionalTime(s.ScheduleLastModifiedAt), + ScheduleConfiguration: s.ScheduleConfiguration, + }) +} + +type scheduleConfiguration struct { + OrchestrationName string `json:"OrchestrationName"` + ScheduleID string `json:"ScheduleId"` + OrchestrationInput string `json:"OrchestrationInput,omitempty"` + OrchestrationInstanceID string `json:"OrchestrationInstanceId,omitempty"` + OrchestrationVersion string `json:"OrchestrationVersion,omitempty"` + StartAt time.Time `json:"StartAt,omitempty"` + EndAt time.Time `json:"EndAt,omitempty"` + Interval dotNetSpan `json:"Interval"` + StartImmediatelyIfLate bool `json:"StartImmediatelyIfLate"` + Tags map[string]string `json:"Tags,omitempty"` + ContextFields api.ContextFields `json:"ContextFields,omitempty"` + RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` +} + +func (c scheduleConfiguration) MarshalJSON() ([]byte, error) { + type configJSON struct { + OrchestrationName string `json:"OrchestrationName"` + ScheduleID string `json:"ScheduleId"` + OrchestrationInput string `json:"OrchestrationInput,omitempty"` + OrchestrationInstanceID string `json:"OrchestrationInstanceId,omitempty"` + OrchestrationVersion string `json:"OrchestrationVersion,omitempty"` + StartAt *time.Time `json:"StartAt"` + EndAt *time.Time `json:"EndAt"` + Interval dotNetSpan `json:"Interval"` + StartImmediatelyIfLate bool `json:"StartImmediatelyIfLate"` + Tags map[string]string `json:"Tags,omitempty"` + ContextFields api.ContextFields `json:"ContextFields,omitempty"` + RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` + } + return json.Marshal(configJSON{ + OrchestrationName: c.OrchestrationName, + ScheduleID: c.ScheduleID, + OrchestrationInput: c.OrchestrationInput, + OrchestrationInstanceID: c.OrchestrationInstanceID, + OrchestrationVersion: c.OrchestrationVersion, + StartAt: optionalTime(c.StartAt), + EndAt: optionalTime(c.EndAt), + Interval: c.Interval, + StartImmediatelyIfLate: c.StartImmediatelyIfLate, + Tags: c.Tags, + ContextFields: c.ContextFields, + RetryPolicy: c.RetryPolicy, + }) +} + +type scheduleRetryPolicy struct { + MaxAttempts int `json:"MaxAttempts"` + InitialRetryInterval dotNetSpan `json:"InitialRetryInterval"` + BackoffCoefficient float64 `json:"BackoffCoefficient"` + MaxRetryInterval dotNetSpan `json:"MaxRetryInterval"` + RetryTimeout dotNetSpan `json:"RetryTimeout"` +} + +func scheduleRetryPolicyFromPublic(policy *ScheduleRetryPolicy) (*scheduleRetryPolicy, error) { + if policy == nil { + return nil, nil + } + taskPolicy, err := policy.taskPolicy().Normalized() + if err != nil { + return nil, &ScheduleValidationError{Message: err.Error()} + } + return &scheduleRetryPolicy{ + MaxAttempts: taskPolicy.MaxAttempts, + InitialRetryInterval: dotNetSpan(taskPolicy.InitialRetryInterval), + BackoffCoefficient: taskPolicy.BackoffCoefficient, + MaxRetryInterval: dotNetSpan(taskPolicy.MaxRetryInterval), + RetryTimeout: dotNetSpan(taskPolicy.RetryTimeout), + }, nil +} + +func (p *scheduleRetryPolicy) public() *ScheduleRetryPolicy { + if p == nil { + return nil + } + return &ScheduleRetryPolicy{ + MaxAttempts: p.MaxAttempts, + InitialRetryInterval: time.Duration(p.InitialRetryInterval), + BackoffCoefficient: p.BackoffCoefficient, + MaxRetryInterval: time.Duration(p.MaxRetryInterval), + RetryTimeout: time.Duration(p.RetryTimeout), + } +} + +func (p *scheduleRetryPolicy) taskPolicy() *task.RetryPolicy { + if p == nil { + return nil + } + return p.public().taskPolicy() +} + +func (p *ScheduleRetryPolicy) taskPolicy() *task.RetryPolicy { + if p == nil { + return nil + } + return &task.RetryPolicy{ + MaxAttempts: p.MaxAttempts, + InitialRetryInterval: p.InitialRetryInterval, + BackoffCoefficient: p.BackoffCoefficient, + MaxRetryInterval: p.MaxRetryInterval, + RetryTimeout: p.RetryTimeout, + } +} + +func optionalTime(value time.Time) *time.Time { + if value.IsZero() { + return nil + } + return &value +} + +// dotNetSpan uses the System.TimeSpan JSON format so persisted schedule state +// remains readable by DTS clients that use the .NET default serializer. +type dotNetSpan time.Duration + +func (d dotNetSpan) MarshalJSON() ([]byte, error) { + duration := time.Duration(d) + if duration < 0 { + return nil, fmt.Errorf("negative TimeSpan is not supported") + } + days := duration / (24 * time.Hour) + duration %= 24 * time.Hour + hours := duration / time.Hour + duration %= time.Hour + minutes := duration / time.Minute + duration %= time.Minute + seconds := duration / time.Second + fraction := duration % time.Second + prefix := "" + if days > 0 { + prefix = fmt.Sprintf("%d.", days) + } + if fraction == 0 { + return []byte(fmt.Sprintf("%q", fmt.Sprintf("%s%02d:%02d:%02d", prefix, hours, minutes, seconds))), nil + } + return []byte(fmt.Sprintf("%q", fmt.Sprintf("%s%02d:%02d:%02d.%07d", prefix, hours, minutes, seconds, fraction/100))), nil +} + +func (d *dotNetSpan) UnmarshalJSON(data []byte) error { + var value string + if err := json.Unmarshal(data, &value); err != nil { + return err + } + invalid := fmt.Errorf("invalid TimeSpan %q", value) + if strings.HasPrefix(value, "-") { + return invalid + } + days := int64(0) + timePart := value + if daySeparator := strings.Index(value, "."); daySeparator >= 0 && + (daySeparator < strings.Index(value, ":")) { + dayPart, rest := value[:daySeparator], value[daySeparator+1:] + parsedDays, err := strconv.ParseInt(dayPart, 10, 64) + if err != nil || parsedDays < 0 { + return invalid + } + days, timePart = parsedDays, rest + } + parts := strings.Split(timePart, ":") + if len(parts) != 3 { + return invalid + } + hours, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil || hours < 0 || (days > 0 && hours > 23) { + return invalid + } + minutes, err := strconv.ParseInt(parts[1], 10, 64) + if err != nil || minutes < 0 || minutes > 59 { + return invalid + } + secondPart, fraction, _ := strings.Cut(parts[2], ".") + seconds, err := strconv.ParseInt(secondPart, 10, 64) + if err != nil || seconds < 0 || seconds > 59 { + return invalid + } + ticks := int64(0) + if fraction != "" { + if len(fraction) > 7 { + return invalid + } + parsedTicks, err := strconv.ParseInt(fraction, 10, 64) + if err != nil { + return invalid + } + for i := len(fraction); i < 7; i++ { + parsedTicks *= 10 + } + ticks = parsedTicks + } + const day = 24 * time.Hour + if days > int64((time.Duration(1<<63-1))/day) { + return invalid + } + valueDuration := time.Duration(days)*day + time.Duration(hours)*time.Hour + + time.Duration(minutes)*time.Minute + time.Duration(seconds)*time.Second + time.Duration(ticks)*100 + if valueDuration < 0 { + return invalid + } + *d = dotNetSpan(valueDuration) + return nil +} + +func scheduleEntityWithDefaultVersion(ctx *task.EntityContext, defaultVersion string) (any, error) { + var state scheduleState + if rawState, ok := ctx.GetRawState(); ok { + if err := json.Unmarshal([]byte(rawState), &state); err != nil { + return nil, fmt.Errorf("failed to deserialize schedule state: %w", err) + } + } + + switch { + case strings.EqualFold(ctx.Operation, deleteScheduleOperation): + var token string + if ctx.HasInput() { + if err := ctx.GetInput(&token); err != nil { + return nil, err + } + } + if token != "" && token != state.ExecutionToken { + return nil, nil + } + ctx.DeleteState() + return nil, nil + case strings.EqualFold(ctx.Operation, createScheduleOperation): + var options ScheduleCreationOptions + if err := ctx.GetInput(&options); err != nil { + return nil, err + } + if err := createSchedule(ctx, &state, options); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, updateScheduleOperation): + var options ScheduleUpdateOptions + if err := ctx.GetInput(&options); err != nil { + return nil, err + } + if err := updateSchedule(ctx, &state, options); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, pauseScheduleOperation): + if err := pauseSchedule(&state); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, resumeScheduleOperation): + if err := resumeSchedule(ctx, &state); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, runScheduleOperation): + var token string + if err := ctx.GetInput(&token); err != nil { + return nil, err + } + if err := runSchedule(ctx, &state, token, defaultVersion); err != nil { + return nil, err + } + if state.Status == ScheduleStatusUninitialized { + return nil, nil + } + default: + return nil, fmt.Errorf("schedule does not support operation %q", ctx.Operation) + } + payload, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("failed to serialize schedule state: %w", err) + } + ctx.SetRawState(string(payload)) + return nil, nil +} + +func createSchedule(ctx *task.EntityContext, state *scheduleState, options ScheduleCreationOptions) error { + if err := validateCreation(options); err != nil { + return err + } + config, err := scheduleConfigFromCreate(ctx, options) + if err != nil { + return err + } + now := scheduleNow(ctx) + createdAt := state.ScheduleCreatedAt + lastRunAt := state.LastRunAt + *state = scheduleState{ + Status: ScheduleStatusActive, + ExecutionToken: newExecutionToken(), + LastRunAt: lastRunAt, + ScheduleCreatedAt: now, + ScheduleLastModifiedAt: now, + ScheduleConfiguration: config, + } + if !createdAt.IsZero() { + state.ScheduleCreatedAt = createdAt + } + return signalRun(ctx, config.ScheduleID, state.ExecutionToken, time.Time{}) +} + +func updateSchedule(ctx *task.EntityContext, state *scheduleState, options ScheduleUpdateOptions) error { + if state.Status != ScheduleStatusActive && state.Status != ScheduleStatusPaused { + return invalidTransition(scheduleID(state), state.Status, state.Status, updateScheduleOperation) + } + if state.ScheduleConfiguration == nil { + return &ScheduleValidationError{Message: "schedule configuration is missing"} + } + config := *state.ScheduleConfiguration + changed, resetNext, err := applyScheduleUpdate(ctx, &config, options) + if err != nil { + return err + } + if !changed { + return nil + } + state.ScheduleConfiguration = &config + state.ScheduleLastModifiedAt = scheduleNow(ctx) + state.ExecutionToken = newExecutionToken() + if resetNext { + state.NextRunAt = time.Time{} + } + if state.Status == ScheduleStatusActive { + return signalRun(ctx, config.ScheduleID, state.ExecutionToken, time.Time{}) + } + return nil +} + +func pauseSchedule(state *scheduleState) error { + if state.Status != ScheduleStatusActive { + return invalidTransition(scheduleID(state), state.Status, ScheduleStatusPaused, pauseScheduleOperation) + } + if state.ScheduleConfiguration == nil { + return &ScheduleValidationError{Message: "schedule configuration is missing"} + } + state.Status = ScheduleStatusPaused + state.NextRunAt = time.Time{} + state.ExecutionToken = newExecutionToken() + return nil +} + +func resumeSchedule(ctx *task.EntityContext, state *scheduleState) error { + if state.Status != ScheduleStatusPaused { + return invalidTransition(scheduleID(state), state.Status, ScheduleStatusActive, resumeScheduleOperation) + } + if state.ScheduleConfiguration == nil { + return &ScheduleValidationError{Message: "schedule configuration is missing"} + } + state.Status = ScheduleStatusActive + state.NextRunAt = time.Time{} + return signalRun(ctx, state.ScheduleConfiguration.ScheduleID, state.ExecutionToken, time.Time{}) +} + +func runSchedule(ctx *task.EntityContext, state *scheduleState, token, defaultVersion string) error { + if state.Status == ScheduleStatusUninitialized { + ctx.DeleteState() + return nil + } + config := state.ScheduleConfiguration + if config == nil { + return &ScheduleValidationError{Message: "schedule configuration is missing"} + } + if token != state.ExecutionToken { + return nil + } + if state.Status != ScheduleStatusActive { + return invalidTransition(config.ScheduleID, state.Status, ScheduleStatusActive, runScheduleOperation) + } + now := scheduleNow(ctx) + if !config.EndAt.IsZero() && now.After(config.EndAt) { + state.NextRunAt = time.Time{} + return ctx.SignalEntity(api.NewEntityID(ScheduleEntityName, config.ScheduleID), deleteScheduleOperation, state.ExecutionToken) + } + next, err := determineNextRun(state, config, now) + if err != nil { + return err + } + state.NextRunAt = next + if !next.After(now) { + if err := startScheduleOrchestration(ctx, config, next, defaultVersion); err != nil { + return err + } + state.LastRunAt = next + state.NextRunAt = time.Time{} + state.NextRunAt, err = determineNextRun(state, config, now) + if err != nil { + return err + } + } + return signalRun(ctx, config.ScheduleID, state.ExecutionToken, state.NextRunAt) +} + +func signalRun(ctx *task.EntityContext, scheduleID, token string, at time.Time) error { + entityID := api.NewEntityID(ScheduleEntityName, scheduleID) + if at.IsZero() { + return ctx.SignalEntity(entityID, runScheduleOperation, token) + } + return ctx.SignalEntityAt(entityID, at, runScheduleOperation, token) +} + +func startScheduleOrchestration(ctx *task.EntityContext, config *scheduleConfiguration, at time.Time, defaultVersion string) error { + instanceID := scheduledOrchestrationInstanceID(config.ScheduleID, config.OrchestrationInstanceID, at) + version := config.OrchestrationVersion + if version == "" { + version = defaultVersion + } + if len(config.Tags) > 0 || len(config.ContextFields) > 0 || config.RetryPolicy != nil { + request := scheduledTaskRequest{ + OrchestrationName: config.OrchestrationName, + OrchestrationInput: config.OrchestrationInput, + OrchestrationInstanceID: instanceID, + OrchestrationVersion: version, + Tags: config.Tags, + ContextFields: config.ContextFields, + RetryPolicy: config.RetryPolicy, + } + payload, err := ctx.SerializeInput(request) + if err != nil { + return fmt.Errorf("failed to serialize scheduled orchestration request: %w", err) + } + return ctx.StartNewOrchestration( + ExecuteScheduledTaskOrchestratorName, + task.WithRawEntityStartOrchestrationInput(payload), + task.WithEntityStartOrchestrationVersion(task.UnversionedTaskVersion), + ) + } + options := []task.EntityStartOrchestrationOption{ + task.WithEntityStartOrchestrationInstanceID(instanceID), + } + if config.OrchestrationInput != "" { + options = append(options, task.WithRawEntityStartOrchestrationInput(config.OrchestrationInput)) + } + if version != "" { + options = append(options, task.WithEntityStartOrchestrationVersion(version)) + } + return ctx.StartNewOrchestration(config.OrchestrationName, options...) +} + +func scheduledOrchestrationInstanceID(scheduleID, instanceID string, at time.Time) string { + if instanceID != "" { + return instanceID + } + return scheduleID + "-" + at.UTC().Format("2006-01-02T15:04:05.0000000-07:00") +} + +func determineNextRun(state *scheduleState, config *scheduleConfiguration, now time.Time) (time.Time, error) { + if !state.NextRunAt.IsZero() { + return state.NextRunAt, nil + } + start := config.StartAt + if start.IsZero() { + start = state.ScheduleCreatedAt + } + if start.IsZero() { + start = now + } + if now.Before(start) { + return start, nil + } + if state.LastRunAt.IsZero() && config.StartImmediatelyIfLate { + return now, nil + } + interval := time.Duration(config.Interval) + if interval < time.Second { + return time.Time{}, &ScheduleValidationError{Message: "interval must be at least one second"} + } + // Land on the same instant .NET computes as + // start + interval*(floor((now-start)/interval)+1) without relying on + // time.Duration arithmetic, which saturates for extreme start times. + delay := interval - intervalRemainder(start, now, interval) + next := now.Add(delay) + // Go can represent instants far beyond .NET's DateTimeOffset.MaxValue, so a + // schedule running near the end of year 9999 would otherwise produce a next + // run that only fails later when the state is serialized for the service. + if next.Before(now) || next.After(maxScheduleTime) { + return time.Time{}, &ScheduleValidationError{Message: "next scheduled time overflows time range"} + } + return next, nil +} + +// maxScheduleTime is DateTimeOffset.MaxValue, the largest instant the .NET +// schedule wire format can carry. .NET timestamps have 100ns resolution, so the +// last representable tick is 999999900ns into the final second. +var maxScheduleTime = time.Date(9999, time.December, 31, 23, 59, 59, 999999900, time.UTC) + +// intervalRemainder returns (now-start) mod interval exactly. It uses 128-bit +// arithmetic because the elapsed nanoseconds between the minimum and maximum +// representable timestamps do not fit in a time.Duration. Inputs outside the +// domain of that division clamp to zero: a non-positive interval, which would +// divide by zero, and a start after now, which would make the elapsed time +// negative. +func intervalRemainder(start, now time.Time, interval time.Duration) time.Duration { + if interval <= 0 { + return 0 + } + seconds := now.Unix() - start.Unix() + nanos := int64(now.Nanosecond()) - int64(start.Nanosecond()) + if nanos < 0 { + seconds-- + nanos += int64(time.Second) + } + if seconds < 0 { + return 0 + } + high, low := bits.Mul64(uint64(seconds), uint64(time.Second)) + low, carry := bits.Add64(low, uint64(nanos), 0) + high += carry + divisor := uint64(interval) + // Reducing the high word first keeps bits.Div64 in range and leaves the + // remainder unchanged, since the discarded multiple of 2^64 divides evenly. + _, remainder := bits.Div64(high%divisor, low, divisor) + return time.Duration(remainder) +} + +func scheduleConfigFromCreate(ctx *task.EntityContext, options ScheduleCreationOptions) (*scheduleConfiguration, error) { + input, err := serializeScheduleInput(ctx, options.OrchestrationInput, options.TypedOrchestrationInput) + if err != nil { + return nil, err + } + retry, err := scheduleRetryPolicyFromPublic(options.RetryPolicy) + if err != nil { + return nil, err + } + config := &scheduleConfiguration{ + OrchestrationName: options.OrchestrationName, + ScheduleID: options.ScheduleID, + OrchestrationInput: input, + OrchestrationInstanceID: options.OrchestrationInstanceID, + OrchestrationVersion: options.OrchestrationVersion, + StartAt: options.StartAt.UTC(), + EndAt: options.EndAt.UTC(), + Interval: dotNetSpan(options.Interval), + StartImmediatelyIfLate: options.StartImmediatelyIfLate, + Tags: cloneStrings(options.Tags), + ContextFields: api.ContextFields(cloneStrings(options.ContextFields)), + RetryPolicy: retry, + } + if err := validateConfig(config); err != nil { + return nil, err + } + return config, nil +} + +func applyScheduleUpdate(ctx *task.EntityContext, config *scheduleConfiguration, options ScheduleUpdateOptions) (changed, resetNext bool, err error) { + changed = updateNonEmptyString(&config.OrchestrationName, options.OrchestrationName) || changed + changed = updateNonEmptyString(&config.OrchestrationInput, options.OrchestrationInput) || changed + if options.TypedOrchestrationInput != nil { + input, inputErr := serializeScheduleInput(ctx, "", options.TypedOrchestrationInput) + if inputErr != nil { + return false, false, inputErr + } + if input != config.OrchestrationInput { + config.OrchestrationInput = input + changed = true + } + } + changed = updateNonEmptyString(&config.OrchestrationInstanceID, options.OrchestrationInstanceID) || changed + changed = updateNonEmptyString(&config.OrchestrationVersion, options.OrchestrationVersion) || changed + if updateTime(&config.StartAt, options.StartAt) { + changed, resetNext = true, true + } + if updateTime(&config.EndAt, options.EndAt) { + changed = true + } + if options.Interval != nil && *options.Interval != time.Duration(config.Interval) { + if err := validateInterval(*options.Interval); err != nil { + return false, false, err + } + config.Interval = dotNetSpan(*options.Interval) + changed, resetNext = true, true + } + if updateComparable(&config.StartImmediatelyIfLate, options.StartImmediatelyIfLate) { + changed, resetNext = true, true + } + if options.Tags != nil && !maps.Equal(options.Tags, config.Tags) { + if err := validateScheduleTags(options.Tags); err != nil { + return false, false, err + } + config.Tags = cloneStrings(options.Tags) + changed = true + } + if options.ContextFields != nil && !maps.Equal(options.ContextFields, config.ContextFields) { + if err := validateScheduleContextFields(options.ContextFields); err != nil { + return false, false, err + } + config.ContextFields = api.ContextFields(cloneStrings(options.ContextFields)) + changed = true + } + if options.RetryPolicy != nil { + retry, retryErr := scheduleRetryPolicyFromPublic(options.RetryPolicy) + if retryErr != nil { + return false, false, retryErr + } + if !reflect.DeepEqual(retry, config.RetryPolicy) { + config.RetryPolicy = retry + changed = true + } + } + if err := validateConfig(config); err != nil { + return false, false, err + } + return changed, resetNext, nil +} + +func updateNonEmptyString(target *string, value *string) bool { + if value == nil || *value == "" || *value == *target { + return false + } + *target = *value + return true +} + +func updateTime(target *time.Time, value *time.Time) bool { + if value == nil || value.Equal(*target) { + return false + } + *target = value.UTC() + return true +} + +func updateComparable[T comparable](target *T, value *T) bool { + if value == nil || *value == *target { + return false + } + *target = *value + return true +} + +func serializeScheduleInput(ctx *task.EntityContext, raw string, typed any) (string, error) { + if typed == nil { + return raw, nil + } + return ctx.SerializeInput(typed) +} + +func validateCreation(options ScheduleCreationOptions) error { + if options.ScheduleID == "" { + return &ScheduleValidationError{Message: "schedule ID is required"} + } + if options.OrchestrationName == "" { + return &ScheduleValidationError{Message: "orchestration name is required"} + } + if err := validateInterval(options.Interval); err != nil { + return err + } + if !options.StartAt.IsZero() && !options.EndAt.IsZero() && options.StartAt.After(options.EndAt) { + return &ScheduleValidationError{Message: "start time cannot be later than end time"} + } + if options.RetryPolicy != nil && options.OrchestrationInstanceID != "" { + return &ScheduleValidationError{ + Message: "a fixed orchestration instance ID cannot be combined with a retry policy", + } + } + if options.RetryPolicy == nil { + // The timestamp suffix contains no '@', so its value cannot change + // whether the derived target uses the reserved entity namespace. + instanceID := scheduledOrchestrationInstanceID(options.ScheduleID, options.OrchestrationInstanceID, options.StartAt) + if err := helpers.ValidateOrchestrationInstanceID(instanceID); err != nil { + return &ScheduleValidationError{Message: err.Error()} + } + } + if err := validateScheduleTags(options.Tags); err != nil { + return err + } + if err := validateScheduleContextFields(options.ContextFields); err != nil { + return err + } + if _, err := scheduleRetryPolicyFromPublic(options.RetryPolicy); err != nil { + return err + } + return nil +} + +func validateConfig(config *scheduleConfiguration) error { + return validateCreation(ScheduleCreationOptions{ + ScheduleID: config.ScheduleID, + OrchestrationName: config.OrchestrationName, + OrchestrationInstanceID: config.OrchestrationInstanceID, + Interval: time.Duration(config.Interval), + StartAt: config.StartAt, + EndAt: config.EndAt, + Tags: config.Tags, + ContextFields: config.ContextFields, + RetryPolicy: config.RetryPolicy.public(), + }) +} + +func validateInterval(interval time.Duration) error { + if interval <= 0 { + return &ScheduleValidationError{Message: "interval must be positive"} + } + if interval < time.Second { + return &ScheduleValidationError{Message: "interval must be at least one second"} + } + return nil +} + +func invalidTransition(scheduleID string, from, to ScheduleStatus, operation string) error { + return &ScheduleInvalidTransitionError{ScheduleID: scheduleID, From: from, To: to, Operation: operation} +} + +func scheduleID(state *scheduleState) string { + if state.ScheduleConfiguration == nil { + return "" + } + return state.ScheduleConfiguration.ScheduleID +} + +func scheduleNow(ctx *task.EntityContext) time.Time { + if now := ctx.CurrentTimeUTC(); !now.IsZero() { + return now.UTC() + } + return time.Now().UTC() +} + +func newExecutionToken() string { + return strings.ReplaceAll(uuid.NewString(), "-", "") +} + +func validateScheduleTags(tags map[string]string) error { + for key := range tags { + if key == "" { + return &ScheduleValidationError{Message: "tag key cannot be empty"} + } + if err := checkUnreservedScheduleKey("tag", key); err != nil { + return err + } + } + return nil +} + +func validateScheduleContextFields(fields api.ContextFields) error { + for key := range fields { + if key == "" { + return &ScheduleValidationError{Message: "context field key cannot be empty"} + } + if err := checkUnreservedScheduleKey("context field", key); err != nil { + return err + } + } + return nil +} + +// checkUnreservedScheduleKey rejects keys that collide with the reserved +// prefixes used to carry orchestration context on the wire. +func checkUnreservedScheduleKey(kind, key string) error { + if strings.HasPrefix(key, api.ReservedContextFieldPrefix) || + strings.HasPrefix(key, tagcodec.UserTagPrefix) { + return &ScheduleValidationError{Message: fmt.Sprintf("%s %q uses a reserved prefix", kind, key)} + } + return nil +} + +func cloneStrings[M ~map[string]string](values M) map[string]string { + if len(values) == 0 { + return nil + } + return maps.Clone(values) +} + +type scheduledTaskBackend interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + GetEntity(context.Context, api.EntityID, ...api.GetEntityOptions) (*api.EntityMetadata, error) + QueryEntities(context.Context, api.EntityQuery) (*api.EntityQueryResults, error) +} + +// ScheduledTaskClient manages all schedules in a DTS task hub. +type ScheduledTaskClient struct { + client scheduledTaskBackend + converter api.DataConverter +} + +// NewScheduledTaskClient creates a scheduled-task client with the default JSON +// data converter. +func NewScheduledTaskClient(client *durabletaskclient.TaskHubGrpcClient) *ScheduledTaskClient { + return NewScheduledTaskClientWithDataConverter(client, nil) +} + +// NewScheduledTaskClientWithDataConverter creates a scheduled-task client with +// an explicit data converter. +func NewScheduledTaskClientWithDataConverter( + client *durabletaskclient.TaskHubGrpcClient, + converter api.DataConverter, +) *ScheduledTaskClient { + if client == nil { + return nil + } + return &ScheduledTaskClient{client: client, converter: api.NormalizeDataConverter(converter)} +} + +// ScheduledTasks returns a scheduled-task client backed by c. +func (c *Client) ScheduledTasks() *ScheduledTaskClient { + if c == nil { + return nil + } + return NewScheduledTaskClientWithDataConverter(c.TaskHubGrpcClient, c.converter) +} + +// GetScheduleClient returns a handle for scheduleID. +func (c *ScheduledTaskClient) GetScheduleClient(scheduleID string) (*ScheduleClient, error) { + if c == nil || c.client == nil { + return nil, fmt.Errorf("scheduled-task client is required") + } + if scheduleID == "" { + return nil, &ScheduleValidationError{Message: "schedule ID is required"} + } + return &ScheduleClient{client: c.client, converter: c.converter, scheduleID: scheduleID}, nil +} + +// Create creates or replaces a schedule and returns its handle. +func (c *ScheduledTaskClient) Create(ctx context.Context, options ScheduleCreationOptions) (*ScheduleClient, error) { + handle, err := c.GetScheduleClient(options.ScheduleID) + if err != nil { + return nil, err + } + if err := handle.Create(ctx, options); err != nil { + return nil, err + } + return handle, nil +} + +// Get returns nil, nil if scheduleID has no schedule. +func (c *ScheduledTaskClient) Get(ctx context.Context, scheduleID string) (*ScheduleDescription, error) { + handle, err := c.GetScheduleClient(scheduleID) + if err != nil { + return nil, err + } + description, err := handle.Describe(ctx) + if errors.Is(err, ErrScheduleNotFound) { + return nil, nil + } + return description, err +} + +// List returns one page of schedules matching query. +func (c *ScheduledTaskClient) List(ctx context.Context, query ScheduleQuery) (*ScheduleQueryResult, error) { + if c == nil || c.client == nil { + return nil, fmt.Errorf("scheduled-task client is required") + } + pageSize := query.PageSize + if pageSize < 0 { + return nil, &ScheduleValidationError{Message: "page size cannot be negative"} + } + if pageSize == 0 { + pageSize = 100 + } + if pageSize > api.MaxInstanceQueryPageSize { + return nil, &ScheduleValidationError{ + Message: fmt.Sprintf("page size cannot exceed %d", api.MaxInstanceQueryPageSize), + } + } + entities, err := c.client.QueryEntities(ctx, api.EntityQuery{ + InstanceIDStartsWith: "@schedule@" + query.ScheduleIDPrefix, + PageSize: pageSize, + ContinuationToken: query.ContinuationToken, + }) + if err != nil { + return nil, err + } + if entities == nil { + return nil, errors.New("schedule query returned no result") + } + result := &ScheduleQueryResult{ContinuationToken: entities.ContinuationToken} + for _, entity := range entities.Entities { + description, err := scheduleDescription(entity, c.converter) + if err != nil { + return nil, err + } + if description == nil || !matchesScheduleQuery(description, query) { + continue + } + result.Schedules = append(result.Schedules, description) + } + return result, nil +} + +// ScheduleClient manages one DTS schedule. +type ScheduleClient struct { + client scheduledTaskBackend + converter api.DataConverter + scheduleID string +} + +// ID returns the schedule identifier managed by this handle. +func (c *ScheduleClient) ID() string { + if c == nil { + return "" + } + return c.scheduleID +} + +// Create creates or replaces this schedule. +func (c *ScheduleClient) Create(ctx context.Context, options ScheduleCreationOptions) error { + if c == nil || c.client == nil { + return fmt.Errorf("schedule client is required") + } + if options.ScheduleID != c.scheduleID { + return &ScheduleValidationError{Message: "creation schedule ID does not match schedule client"} + } + return c.operate(ctx, createScheduleOperation, options) +} + +// Describe returns the current schedule description. +func (c *ScheduleClient) Describe(ctx context.Context) (*ScheduleDescription, error) { + if c == nil || c.client == nil { + return nil, fmt.Errorf("schedule client is required") + } + entity, err := c.client.GetEntity(ctx, scheduleEntityID(c.scheduleID)) + if err != nil { + return nil, err + } + if entity == nil { + return nil, &ScheduleNotFoundError{ScheduleID: c.scheduleID} + } + description, err := scheduleDescription(entity, c.converter) + if err != nil { + return nil, err + } + if description == nil { + return nil, &ScheduleNotFoundError{ScheduleID: c.scheduleID} + } + return description, nil +} + +// Delete removes this schedule without affecting orchestration runs already started. +func (c *ScheduleClient) Delete(ctx context.Context) error { + return c.operate(ctx, deleteScheduleOperation, nil) +} + +// Pause stops future executions without affecting runs already started. +func (c *ScheduleClient) Pause(ctx context.Context) error { + return c.operate(ctx, pauseScheduleOperation, nil) +} + +// Resume restarts future executions. +func (c *ScheduleClient) Resume(ctx context.Context) error { + return c.operate(ctx, resumeScheduleOperation, nil) +} + +// Update applies a sparse configuration update. +func (c *ScheduleClient) Update(ctx context.Context, options ScheduleUpdateOptions) error { + return c.operate(ctx, updateScheduleOperation, options) +} + +func (c *ScheduleClient) operate(ctx context.Context, operation string, input any) error { + if c == nil || c.client == nil { + return fmt.Errorf("schedule client is required") + } + preparedInput, err := prepareScheduleOperationInput(c.converter, input) + if err != nil { + return err + } + instanceID, err := c.client.ScheduleNewOrchestration(ctx, ExecuteScheduleOperationOrchestratorName, + api.WithInput(ScheduleOperationRequest{ + EntityID: scheduleEntityID(c.scheduleID), + OperationName: operation, + Input: preparedInput, + }), + // System operations must not inherit application default versions. + api.WithVersion(""), + ) + if err != nil { + return err + } + metadata, err := c.client.WaitForOrchestrationCompletion(ctx, instanceID) + if err != nil { + return err + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_COMPLETED { + return scheduleOperationFailure(c.scheduleID, operation, metadata) + } + return nil +} + +func scheduleEntityID(scheduleID string) api.EntityID { + return api.NewEntityID(ScheduleEntityName, scheduleID) +} + +func scheduleDescription( + metadata *api.EntityMetadata, + converter api.DataConverter, +) (*ScheduleDescription, error) { + if metadata == nil || metadata.SerializedState == "" { + return nil, nil + } + var state scheduleState + if err := json.Unmarshal([]byte(metadata.SerializedState), &state); err != nil { + return nil, fmt.Errorf("failed to deserialize schedule state: %w", err) + } + description := &ScheduleDescription{ + ScheduleID: metadata.InstanceID.Key, + Status: state.Status, + ExecutionToken: state.ExecutionToken, + LastRunAt: state.LastRunAt, + NextRunAt: state.NextRunAt, + CreatedAt: state.ScheduleCreatedAt, + LastModifiedAt: state.ScheduleLastModifiedAt, + Converter: api.NormalizeDataConverter(converter), + } + if config := state.ScheduleConfiguration; config != nil { + description.ScheduleID = config.ScheduleID + description.OrchestrationName = config.OrchestrationName + description.OrchestrationInput = config.OrchestrationInput + description.OrchestrationInstanceID = config.OrchestrationInstanceID + description.OrchestrationVersion = config.OrchestrationVersion + description.StartAt = config.StartAt + description.EndAt = config.EndAt + description.Interval = time.Duration(config.Interval) + description.StartImmediatelyIfLate = config.StartImmediatelyIfLate + description.Tags = cloneStrings(config.Tags) + description.ContextFields = api.ContextFields(cloneStrings(config.ContextFields)) + description.RetryPolicy = config.RetryPolicy.public() + } + return description, nil +} + +func matchesScheduleQuery(description *ScheduleDescription, query ScheduleQuery) bool { + if query.Status != nil && description.Status != *query.Status { + return false + } + if !query.CreatedFrom.IsZero() && !description.CreatedAt.After(query.CreatedFrom) { + return false + } + if !query.CreatedTo.IsZero() && !description.CreatedAt.Before(query.CreatedTo) { + return false + } + return true +} + +func prepareScheduleOperationInput(converter api.DataConverter, input any) (any, error) { + switch options := input.(type) { + case ScheduleCreationOptions: + if options.TypedOrchestrationInput != nil { + payload, err := api.SerializeData(converter, options.TypedOrchestrationInput) + if err != nil { + return nil, fmt.Errorf("failed to serialize scheduled orchestration input: %w", err) + } + options.OrchestrationInput = payload + options.TypedOrchestrationInput = nil + } + return options, nil + case ScheduleUpdateOptions: + if options.TypedOrchestrationInput != nil { + payload, err := api.SerializeData(converter, options.TypedOrchestrationInput) + if err != nil { + return nil, fmt.Errorf("failed to serialize scheduled orchestration input: %w", err) + } + options.OrchestrationInput = &payload + options.TypedOrchestrationInput = nil + } + return options, nil + default: + return input, nil + } +} + +func scheduleOperationFailure( + scheduleID string, + operation string, + metadata *api.OrchestrationMetadata, +) error { + if metadata != nil { + if details := findScheduleFailure(metadata.FailureDetails, scheduleInvalidTransitionType); details != nil { + return &ScheduleInvalidTransitionError{ + ScheduleID: stringProperty(details.Properties, "scheduleId", scheduleID), + From: ScheduleStatus(intProperty(details.Properties, "from")), + To: ScheduleStatus(intProperty(details.Properties, "to")), + Operation: stringProperty(details.Properties, "operation", operation), + } + } + if details := findScheduleFailure(metadata.FailureDetails, scheduleValidationErrorType); details != nil { + return &ScheduleValidationError{Message: stringProperty(details.Properties, "message", details.ErrorMessage)} + } + return &ScheduleOperationError{ + ScheduleID: scheduleID, + Operation: operation, + RuntimeStatus: metadata.RuntimeStatus, + FailureDetails: metadata.FailureDetails, + } + } + return &ScheduleOperationError{ScheduleID: scheduleID, Operation: operation} +} + +func findScheduleFailure(details *api.FailureDetails, errorType api.ErrorType) *api.FailureDetails { + for current := details; current != nil; current = current.InnerFailure { + if current.ErrorType == errorType || + strings.HasSuffix(string(current.ErrorType), "."+string(errorType)) { + return current + } + } + return nil +} + +func stringProperty(properties map[string]any, name, fallback string) string { + if value, ok := properties[name].(string); ok && value != "" { + return value + } + return fallback +} + +func intProperty(properties map[string]any, name string) int { + switch value := properties[name].(type) { + case int: + return value + case int32: + return int(value) + case int64: + return int(value) + case float64: + return int(value) + default: + return 0 + } +} diff --git a/durabletaskscheduler/scheduled_tasks_parity_test.go b/durabletaskscheduler/scheduled_tasks_parity_test.go new file mode 100644 index 00000000..e20e707d --- /dev/null +++ b/durabletaskscheduler/scheduled_tasks_parity_test.go @@ -0,0 +1,1977 @@ +package durabletaskscheduler + +import ( + "context" + "encoding/base64" + "encoding/json" + "math" + "math/big" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// minTimestamp and maxTimestamp mirror DateTimeOffset.MinValue/MaxValue. The +// exact minimum is Go's zero time, which the schedule wire format encodes as an +// absent value, so the boundary cases use the smallest representable non-zero +// instant instead. +var ( + minTimestamp = time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC) + nearMinTimestamp = minTimestamp.Add(100 * time.Nanosecond) + maxTimestamp = time.Date(9999, time.December, 31, 23, 59, 59, 999999900, time.UTC) +) + +// --------------------------------------------------------------------------- +// Next-run calculation +// --------------------------------------------------------------------------- + +func TestDetermineNextRunMatrix(t *testing.T) { + now := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + minute := time.Minute + + tests := []struct { + name string + startAt time.Time + createdAt time.Time + lastRunAt time.Time + nextRunAt time.Time + interval time.Duration + immediate bool + want time.Time + wantErrorIs error + }{ + { + name: "start in future is honored", + startAt: now.Add(5 * minute), + interval: 5 * minute, + want: now.Add(5 * minute), + }, + { + name: "start in future ignores start immediately if late", + startAt: now.Add(5 * minute), + interval: 5 * minute, + immediate: true, + want: now.Add(5 * minute), + }, + { + name: "start in future with prior run still waits for start", + startAt: now.Add(90 * time.Second), + lastRunAt: now.Add(-time.Hour), + interval: 5 * minute, + want: now.Add(90 * time.Second), + }, + { + name: "late start runs immediately on first run", + startAt: now.Add(-5 * minute), + interval: 5 * minute, + immediate: true, + want: now, + }, + { + name: "late start without immediate waits for next boundary", + startAt: now.Add(-5 * minute), + interval: 3 * minute, + want: now.Add(-5 * minute).Add(6 * minute), + }, + { + name: "immediate is ignored once a run happened", + startAt: now.Add(-10 * minute), + lastRunAt: now.Add(-minute), + interval: 3 * minute, + immediate: true, + want: now.Add(-10 * minute).Add(12 * minute), + }, + { + name: "exact interval boundary advances one full interval", + startAt: now.Add(-9 * minute), + interval: 3 * minute, + want: now.Add(3 * minute), + }, + { + name: "start equal to now advances one full interval", + startAt: now, + interval: 5 * minute, + want: now.Add(5 * minute), + }, + { + name: "fractional interval keeps sub-second precision", + startAt: now.Add(-4 * time.Second), + interval: 1500 * time.Millisecond, + want: now.Add(-4 * time.Second).Add(4500 * time.Millisecond), + }, + { + name: "creation time anchors the schedule when start is unset", + createdAt: now.Add(-10 * minute), + interval: 3 * minute, + want: now.Add(-10 * minute).Add(12 * minute), + }, + { + name: "now anchors the schedule when start and creation are unset", + interval: 5 * minute, + want: now.Add(5 * minute), + }, + { + name: "existing next run is returned unchanged", + startAt: now.Add(-time.Hour), + nextRunAt: now.Add(-time.Hour), + interval: 5 * minute, + immediate: true, + want: now.Add(-time.Hour), + }, + { + name: "near-minimum start stays on the interval grid", + startAt: nearMinTimestamp, + interval: time.Second, + want: now.Add(100 * time.Nanosecond), + }, + { + name: "maximum end time does not affect the next run", + startAt: now.Add(-time.Hour), + interval: 45 * minute, + want: now.Add(-time.Hour).Add(90 * minute), + }, + { + name: "interval below one second is rejected", + interval: 500 * time.Millisecond, + wantErrorIs: ErrScheduleValidation, + }, + { + name: "zero interval is rejected", + wantErrorIs: ErrScheduleValidation, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + state := &scheduleState{ + LastRunAt: test.lastRunAt, + NextRunAt: test.nextRunAt, + ScheduleCreatedAt: test.createdAt, + } + config := &scheduleConfiguration{ + StartAt: test.startAt, + Interval: dotNetSpan(test.interval), + StartImmediatelyIfLate: test.immediate, + } + next, err := determineNextRun(state, config, now) + if test.wantErrorIs != nil { + require.ErrorIs(t, err, test.wantErrorIs) + return + } + require.NoError(t, err) + require.True(t, test.want.Equal(next), "want %s, got %s", test.want, next) + }) + } +} + +// TestDetermineNextRunLandsOnIntervalGrid verifies the .NET formula +// start + interval*(elapsed/interval + 1) using independent big.Int math, which +// time.Duration cannot express for extreme start times. +func TestDetermineNextRunLandsOnIntervalGrid(t *testing.T) { + now := time.Date(2026, time.August, 27, 12, 34, 56, 789012300, time.UTC) + + tests := []struct { + name string + startAt time.Time + interval time.Duration + }{ + {name: "near minimum start with one second interval", startAt: nearMinTimestamp, interval: time.Second}, + {name: "near minimum start with fractional interval", startAt: nearMinTimestamp, interval: 1500 * time.Millisecond}, + {name: "year 1200 start", startAt: time.Date(1200, time.June, 1, 3, 4, 5, 600, time.UTC), interval: 7 * time.Hour}, + {name: "recent start", startAt: now.Add(-987654321 * time.Nanosecond), interval: 90 * time.Second}, + {name: "century old start", startAt: now.AddDate(-100, 0, 0), interval: 13*time.Minute + 7*time.Second}, + {name: "very large interval", startAt: nearMinTimestamp, interval: 200000 * time.Hour}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + next, err := determineNextRun( + &scheduleState{}, + &scheduleConfiguration{StartAt: test.startAt, Interval: dotNetSpan(test.interval)}, + now, + ) + require.NoError(t, err) + require.True(t, next.After(now), "next run %s must be after %s", next, now) + require.False(t, next.After(now.Add(test.interval)), "next run must be within one interval") + offset := new(big.Int).Sub(nanosecondsSinceEpoch(next), nanosecondsSinceEpoch(test.startAt)) + remainder := new(big.Int).Mod(offset, big.NewInt(int64(test.interval))) + require.Equal(t, "0", remainder.String(), "next run must sit on the interval grid") + }) + } +} + +// TestDetermineNextRunRejectsTimesBeyondTheSupportedRange pins the upper +// boundary of the .NET schedule wire format: a next run one tick past +// DateTimeOffset.MaxValue must be rejected up front rather than serialized as a +// year-10000 timestamp that only fails later. +func TestDetermineNextRunRejectsTimesBeyondTheSupportedRange(t *testing.T) { + tests := []struct { + name string + now time.Time + interval time.Duration + want time.Time + }{ + { + name: "next run exactly at the maximum is allowed", + now: maxTimestamp.Add(-time.Hour), + interval: time.Hour, + want: maxTimestamp, + }, + { + name: "next run one tick past the maximum is rejected", + now: maxTimestamp.Add(-time.Hour).Add(100 * time.Nanosecond), + interval: time.Hour, + }, + { + name: "next run far past the maximum is rejected", + now: maxTimestamp.Add(-time.Second), + interval: 24 * time.Hour, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + next, err := determineNextRun( + &scheduleState{}, + // Anchoring the start at now makes the next run land exactly + // one interval later. + &scheduleConfiguration{StartAt: test.now, Interval: dotNetSpan(test.interval)}, + test.now, + ) + if test.want.IsZero() { + require.ErrorIs(t, err, ErrScheduleValidation) + return + } + require.NoError(t, err) + require.True(t, test.want.Equal(next), "want %s, got %s", test.want, next) + }) + } +} + +func TestIntervalRemainderIsExact(t *testing.T) { + now := time.Date(2026, time.August, 27, 12, 0, 0, 500, time.UTC) + tests := []struct { + name string + start time.Time + interval time.Duration + }{ + {name: "zero elapsed", start: now, interval: time.Minute}, + {name: "sub interval", start: now.Add(-999 * time.Millisecond), interval: time.Second}, + {name: "exact multiple", start: now.Add(-3 * time.Hour), interval: time.Hour}, + {name: "near minimum start", start: nearMinTimestamp, interval: time.Second}, + {name: "near minimum start fractional", start: nearMinTimestamp, interval: 1234567891 * time.Nanosecond}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got := intervalRemainder(test.start, now, test.interval) + elapsed := new(big.Int).Sub(nanosecondsSinceEpoch(now), nanosecondsSinceEpoch(test.start)) + want := new(big.Int).Mod(elapsed, big.NewInt(int64(test.interval))) + require.Equal(t, want.String(), big.NewInt(int64(got)).String()) + }) + } +} + +// TestIntervalRemainderClampsInputsOutsideTheDivisionDomain pins the two guards +// that keep the 128-bit division well defined: a non-positive interval would +// divide by zero, and a start after now would make the elapsed time negative. +func TestIntervalRemainderClampsInputsOutsideTheDivisionDomain(t *testing.T) { + now := time.Date(2026, time.August, 27, 12, 0, 0, 500, time.UTC) + require.Zero(t, intervalRemainder(now.Add(-time.Hour), now, 0)) + require.Zero(t, intervalRemainder(now.Add(-time.Hour), now, -time.Minute)) + require.Zero(t, intervalRemainder(now.Add(time.Hour), now, time.Minute)) +} + +func nanosecondsSinceEpoch(value time.Time) *big.Int { + total := new(big.Int).Mul(big.NewInt(value.Unix()), big.NewInt(int64(time.Second))) + return total.Add(total, big.NewInt(int64(value.Nanosecond()))) +} + +// --------------------------------------------------------------------------- +// .NET-compatible serialization boundaries +// --------------------------------------------------------------------------- + +func TestDotNetSpanRoundTrip(t *testing.T) { + tests := []struct { + name string + duration time.Duration + json string + }{ + {name: "zero", duration: 0, json: `"00:00:00"`}, + {name: "one second", duration: time.Second, json: `"00:00:01"`}, + {name: "fractional", duration: 1500 * time.Millisecond, json: `"00:00:01.5000000"`}, + {name: "one tick", duration: 100 * time.Nanosecond, json: `"00:00:00.0000001"`}, + {name: "one day", duration: 24 * time.Hour, json: `"1.00:00:00"`}, + {name: "days hours minutes", duration: 26*time.Hour + 3*time.Minute + 4*time.Second + 500*time.Millisecond, json: `"1.02:03:04.5000000"`}, + {name: "maximum tick aligned duration", duration: (math.MaxInt64 / 100) * 100, json: `"106751.23:47:16.8547758"`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + payload, err := json.Marshal(dotNetSpan(test.duration)) + require.NoError(t, err) + require.Equal(t, test.json, string(payload)) + + var decoded dotNetSpan + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.Equal(t, test.duration, time.Duration(decoded)) + }) + } +} + +func TestDotNetSpanRejectsInvalidValues(t *testing.T) { + for _, value := range []string{ + `"-00:00:01"`, + `"00:00"`, + `"00:60:00"`, + `"00:00:60"`, + `"00:00:00.00000001"`, + `"1.24:00:00"`, + `"abc"`, + `"1.aa:00:00"`, + `5`, + } { + t.Run(value, func(t *testing.T) { + var decoded dotNetSpan + require.Error(t, json.Unmarshal([]byte(value), &decoded)) + }) + } + _, err := json.Marshal(dotNetSpan(-time.Second)) + require.Error(t, err) +} + +func TestScheduleCreationOptionsJSONBoundaries(t *testing.T) { + tests := []struct { + name string + options ScheduleCreationOptions + want string + }{ + { + name: "minimum start is encoded as absent", + options: ScheduleCreationOptions{ + ScheduleID: "min", OrchestrationName: "Run", Interval: time.Second, StartAt: minTimestamp, + }, + want: `{"ScheduleId":"min","OrchestrationName":"Run","Interval":"00:00:01","StartImmediatelyIfLate":false}`, + }, + { + name: "near minimum start round-trips", + options: ScheduleCreationOptions{ + ScheduleID: "near-min", OrchestrationName: "Run", Interval: time.Second, StartAt: nearMinTimestamp, + }, + want: `{"ScheduleId":"near-min","OrchestrationName":"Run","Interval":"00:00:01","StartAt":"0001-01-01T00:00:00.0000001Z","StartImmediatelyIfLate":false}`, + }, + { + name: "maximum end round-trips", + options: ScheduleCreationOptions{ + ScheduleID: "max", OrchestrationName: "Run", Interval: time.Second, EndAt: maxTimestamp, + }, + want: `{"ScheduleId":"max","OrchestrationName":"Run","Interval":"00:00:01","EndAt":"9999-12-31T23:59:59.9999999Z","StartImmediatelyIfLate":false}`, + }, + { + name: "unicode schedule id and json input", + options: ScheduleCreationOptions{ + ScheduleID: "日程-🌟", OrchestrationName: "Run", Interval: time.Second, + OrchestrationInput: `{"key":"value with \"quotes\""}`, + }, + want: `{"ScheduleId":"日程-🌟","OrchestrationName":"Run","Interval":"00:00:01","OrchestrationInput":"{\"key\":\"value with \\\"quotes\\\"\"}","StartImmediatelyIfLate":false}`, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + payload, err := json.Marshal(test.options) + require.NoError(t, err) + require.JSONEq(t, test.want, string(payload)) + + var decoded ScheduleCreationOptions + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.Equal(t, test.options.ScheduleID, decoded.ScheduleID) + require.Equal(t, test.options.Interval, decoded.Interval) + require.Equal(t, test.options.OrchestrationInput, decoded.OrchestrationInput) + if test.options.StartAt.Equal(minTimestamp) { + require.True(t, decoded.StartAt.IsZero()) + } else { + require.True(t, test.options.StartAt.Equal(decoded.StartAt)) + } + require.True(t, test.options.EndAt.Equal(decoded.EndAt)) + }) + } +} + +func TestScheduleUpdateOptionsOmitsUnsetFields(t *testing.T) { + payload, err := json.Marshal(ScheduleUpdateOptions{}) + require.NoError(t, err) + require.JSONEq(t, `{}`, string(payload)) + + name := "" + payload, err = json.Marshal(ScheduleUpdateOptions{OrchestrationName: &name}) + require.NoError(t, err) + require.JSONEq(t, `{"OrchestrationName":""}`, string(payload)) + + var decoded ScheduleUpdateOptions + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.NotNil(t, decoded.OrchestrationName) + require.Empty(t, *decoded.OrchestrationName) +} + +func TestScheduleUpdateMapPresenceAcrossClientOrchestratorAndEntity(t *testing.T) { + for _, clearMaps := range []bool{false, true} { + name := "nil maps preserve values" + if clearMaps { + name = "empty maps clear values" + } + t.Run(name, func(t *testing.T) { + backend := &recordingScheduleBackend{} + handle, err := (&ScheduledTaskClient{client: backend}).GetScheduleClient("daily") + require.NoError(t, err) + options := ScheduleUpdateOptions{} + if clearMaps { + options.Tags = map[string]string{} + options.ContextFields = api.ContextFields{} + } + require.NoError(t, handle.Update(context.Background(), options)) + payload, err := api.DefaultDataConverter().Serialize(backend.request) + require.NoError(t, err) + registry := newScheduleRegistry(t) + response := executeScheduleOperationTurn(t, registry, nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(ExecuteScheduleOperationOrchestratorName, "op", + wrapperspb.String(payload), nil, nil, nil), + }) + require.Len(t, response.Actions, 1) + called := response.Actions[0].GetSendEntityMessage().GetEntityOperationCalled() + require.NotNil(t, called) + + state := seededScheduleState(ScheduleStatusActive) + state.ScheduleConfiguration.Tags = map[string]string{"tag": "old"} + state.ScheduleConfiguration.ContextFields = api.ContextFields{"field": "old"} + state.ScheduleConfiguration.StartImmediatelyIfLate = true + executor := newScheduleExecutor(t) + updated, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: called.TargetInstanceId.GetValue(), + EntityState: marshalScheduleState(state), + Operations: []*protos.OperationRequest{{ + RequestId: called.RequestId, + Operation: called.Operation, + Input: called.Input, + }}, + }) + require.NoError(t, err) + requireOperationSucceeded(t, updated) + current := decodeScheduleState(t, updated.EntityState) + if clearMaps { + require.Empty(t, current.ScheduleConfiguration.Tags) + require.Empty(t, current.ScheduleConfiguration.ContextFields) + require.NotEqual(t, state.ExecutionToken, current.ExecutionToken) + } else { + require.Equal(t, state.ScheduleConfiguration.Tags, current.ScheduleConfiguration.Tags) + require.Equal(t, state.ScheduleConfiguration.ContextFields, current.ScheduleConfiguration.ContextFields) + require.Equal(t, state.ExecutionToken, current.ExecutionToken) + } + run := runScheduleOperationBatch(t, executor, "daily", updated.EntityState, runScheduleOperation, current.ExecutionToken) + requireOperationSucceeded(t, run) + var started *protos.StartNewOrchestrationAction + for _, action := range run.Actions { + if start := action.GetStartNewOrchestration(); start != nil { + started = start + } + } + require.NotNil(t, started) + if clearMaps { + require.Equal(t, "Backup", started.Name) + } else { + require.Equal(t, ExecuteScheduledTaskOrchestratorName, started.Name) + } + }) + } +} + +// --------------------------------------------------------------------------- +// Entity state transitions +// --------------------------------------------------------------------------- + +func TestScheduleEntityTransitionMatrix(t *testing.T) { + tests := []struct { + name string + status ScheduleStatus + operation string + input any + useToken bool + wantInvalidTransition bool + wantStateGone bool + }{ + {name: "create from uninitialized", status: ScheduleStatusUninitialized, operation: createScheduleOperation, input: validCreationOptions()}, + {name: "create from active", status: ScheduleStatusActive, operation: createScheduleOperation, input: validCreationOptions()}, + {name: "create from paused", status: ScheduleStatusPaused, operation: createScheduleOperation, input: validCreationOptions()}, + + {name: "update from uninitialized", status: ScheduleStatusUninitialized, operation: updateScheduleOperation, input: intervalUpdate(10 * time.Minute), wantInvalidTransition: true}, + {name: "update from active", status: ScheduleStatusActive, operation: updateScheduleOperation, input: intervalUpdate(10 * time.Minute)}, + {name: "update from paused", status: ScheduleStatusPaused, operation: updateScheduleOperation, input: intervalUpdate(10 * time.Minute)}, + + {name: "pause from uninitialized", status: ScheduleStatusUninitialized, operation: pauseScheduleOperation, wantInvalidTransition: true}, + {name: "pause from active", status: ScheduleStatusActive, operation: pauseScheduleOperation}, + {name: "pause from paused", status: ScheduleStatusPaused, operation: pauseScheduleOperation, wantInvalidTransition: true}, + + {name: "resume from uninitialized", status: ScheduleStatusUninitialized, operation: resumeScheduleOperation, wantInvalidTransition: true}, + {name: "resume from active", status: ScheduleStatusActive, operation: resumeScheduleOperation, wantInvalidTransition: true}, + {name: "resume from paused", status: ScheduleStatusPaused, operation: resumeScheduleOperation}, + + {name: "run from uninitialized deletes state", status: ScheduleStatusUninitialized, operation: runScheduleOperation, useToken: true, wantStateGone: true}, + {name: "run from active", status: ScheduleStatusActive, operation: runScheduleOperation, useToken: true}, + {name: "run from paused", status: ScheduleStatusPaused, operation: runScheduleOperation, useToken: true, wantInvalidTransition: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + state := seededScheduleState(test.status) + input := test.input + if test.useToken { + input = state.ExecutionToken + } + result := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(state), test.operation, input) + if test.wantInvalidTransition { + requireOperationErrorType(t, result, scheduleInvalidTransitionType) + return + } + requireOperationSucceeded(t, result) + if test.wantStateGone { + require.Nil(t, result.EntityState) + return + } + require.NotNil(t, result.EntityState) + }) + } +} + +func TestScheduleEntityUpdateNoOpInvariants(t *testing.T) { + sameValues := ScheduleUpdateOptions{ + OrchestrationName: pointerTo("Backup"), + Interval: pointerTo(5 * time.Minute), + } + emptyStrings := ScheduleUpdateOptions{ + OrchestrationName: pointerTo(""), + OrchestrationInput: pointerTo(""), + OrchestrationInstanceID: pointerTo(""), + OrchestrationVersion: pointerTo(""), + } + unchangedFlags := ScheduleUpdateOptions{ + StartImmediatelyIfLate: pointerTo(false), + EndAt: pointerTo(time.Time{}), + } + + tests := []struct { + name string + status ScheduleStatus + options ScheduleUpdateOptions + }{ + {name: "all fields nil on active", status: ScheduleStatusActive, options: ScheduleUpdateOptions{}}, + {name: "all fields nil on paused", status: ScheduleStatusPaused, options: ScheduleUpdateOptions{}}, + {name: "same values on active", status: ScheduleStatusActive, options: sameValues}, + {name: "same values on paused", status: ScheduleStatusPaused, options: sameValues}, + {name: "empty strings are ignored", status: ScheduleStatusActive, options: emptyStrings}, + {name: "unchanged flags are ignored", status: ScheduleStatusActive, options: unchangedFlags}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + before := seededScheduleState(test.status) + before.LastRunAt = time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + before.NextRunAt = time.Date(2026, time.August, 1, 0, 5, 0, 0, time.UTC) + + result := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(before), + updateScheduleOperation, test.options) + requireOperationSucceeded(t, result) + after := decodeScheduleState(t, result.EntityState) + + require.Empty(t, result.Actions, "a no-op update must not resignal the schedule") + require.Equal(t, before.ExecutionToken, after.ExecutionToken, "a no-op update must not rotate the token") + require.True(t, before.ScheduleLastModifiedAt.Equal(after.ScheduleLastModifiedAt)) + require.True(t, before.NextRunAt.Equal(after.NextRunAt)) + require.True(t, before.LastRunAt.Equal(after.LastRunAt)) + require.Equal(t, before.Status, after.Status) + require.Equal(t, before.ScheduleConfiguration, after.ScheduleConfiguration) + }) + } +} + +func TestScheduleEntityUpdateRotatesTokenAndResetsNextRun(t *testing.T) { + tests := []struct { + name string + status ScheduleStatus + options ScheduleUpdateOptions + wantResetNext bool + wantSignal bool + }{ + { + name: "interval change resets next run and resignals", + status: ScheduleStatusActive, + options: intervalUpdate(6 * time.Minute), + wantResetNext: true, + wantSignal: true, + }, + { + name: "start change resets next run", + status: ScheduleStatusActive, + options: ScheduleUpdateOptions{StartAt: pointerTo(time.Date(2030, time.January, 1, 0, 0, 0, 0, time.UTC))}, + wantResetNext: true, + wantSignal: true, + }, + { + name: "start immediately change resets next run", + status: ScheduleStatusActive, + options: ScheduleUpdateOptions{StartImmediatelyIfLate: pointerTo(true)}, + wantResetNext: true, + wantSignal: true, + }, + { + name: "end change keeps next run", + status: ScheduleStatusActive, + options: ScheduleUpdateOptions{EndAt: pointerTo(time.Date(2030, time.January, 1, 0, 0, 0, 0, time.UTC))}, + wantSignal: true, + }, + { + name: "orchestration name change keeps next run", + status: ScheduleStatusActive, + options: ScheduleUpdateOptions{OrchestrationName: pointerTo("Restore")}, + wantSignal: true, + }, + { + name: "paused schedules rotate the token without resignaling", + status: ScheduleStatusPaused, + options: intervalUpdate(6 * time.Minute), + wantResetNext: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + before := seededScheduleState(test.status) + before.NextRunAt = time.Date(2026, time.August, 1, 0, 5, 0, 0, time.UTC) + + result := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(before), + updateScheduleOperation, test.options) + requireOperationSucceeded(t, result) + after := decodeScheduleState(t, result.EntityState) + + require.NotEqual(t, before.ExecutionToken, after.ExecutionToken) + require.False(t, after.ScheduleLastModifiedAt.Before(before.ScheduleLastModifiedAt)) + if test.wantResetNext { + require.True(t, after.NextRunAt.IsZero()) + } else { + require.True(t, before.NextRunAt.Equal(after.NextRunAt)) + } + if test.wantSignal { + require.Len(t, result.Actions, 1) + signal := result.Actions[0].GetSendSignal() + require.NotNil(t, signal) + require.Equal(t, runScheduleOperation, signal.GetName()) + require.Equal(t, after.ExecutionToken, decodeSignalToken(t, signal)) + } else { + require.Empty(t, result.Actions) + } + }) + } +} + +func TestScheduleEntityRunTokenMatrix(t *testing.T) { + tests := []struct { + name string + token func(current string) string + }{ + {name: "empty token", token: func(string) string { return "" }}, + {name: "whitespace token", token: func(string) string { return " " }}, + {name: "superseded token", token: func(string) string { return "0123456789abcdef0123456789abcdef" }}, + {name: "token with different case", token: strings.ToUpper}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + before := seededScheduleState(ScheduleStatusActive) + before.ScheduleConfiguration.StartImmediatelyIfLate = true + + result := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(before), + runScheduleOperation, test.token(before.ExecutionToken)) + requireOperationSucceeded(t, result) + after := decodeScheduleState(t, result.EntityState) + + require.Empty(t, result.Actions, "a stale run must not start work or resignal") + require.True(t, after.LastRunAt.IsZero()) + require.Equal(t, before.ExecutionToken, after.ExecutionToken) + require.Equal(t, ScheduleStatusActive, after.Status) + }) + } +} + +func TestScheduleEntityRunSupersededAfterUpdate(t *testing.T) { + executor := newScheduleExecutor(t) + created := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + Interval: 5 * time.Minute, + StartImmediatelyIfLate: true, + }) + requireOperationSucceeded(t, created) + initialToken := decodeScheduleState(t, created.EntityState).ExecutionToken + + updated := runScheduleOperationBatch(t, executor, "daily", created.EntityState, + updateScheduleOperation, intervalUpdate(6*time.Minute)) + requireOperationSucceeded(t, updated) + currentToken := decodeScheduleState(t, updated.EntityState).ExecutionToken + require.NotEqual(t, initialToken, currentToken) + + stale := runScheduleOperationBatch(t, executor, "daily", updated.EntityState, runScheduleOperation, initialToken) + requireOperationSucceeded(t, stale) + require.Empty(t, stale.Actions) + require.True(t, decodeScheduleState(t, stale.EntityState).LastRunAt.IsZero()) + + current := runScheduleOperationBatch(t, executor, "daily", stale.EntityState, runScheduleOperation, currentToken) + requireOperationSucceeded(t, current) + require.NotNil(t, current.Actions[0].GetStartNewOrchestration()) + require.False(t, decodeScheduleState(t, current.EntityState).LastRunAt.IsZero()) +} + +func TestScheduleEntityRunTimingMatrix(t *testing.T) { + tests := []struct { + name string + startOffset time.Duration + endOffset time.Duration + hasStart bool + hasEnd bool + // maxEnd sets the end to DateTimeOffset.MaxValue instead of an offset + // from now. + maxEnd bool + immediate bool + wantStarted bool + wantRetired bool + }{ + {name: "future start does not run", startOffset: 5 * time.Minute, hasStart: true}, + {name: "future start ignores immediate", startOffset: 5 * time.Minute, hasStart: true, immediate: true}, + {name: "late start with immediate runs", startOffset: -5 * time.Minute, hasStart: true, immediate: true, wantStarted: true}, + {name: "late start without immediate waits", startOffset: -5 * time.Minute, hasStart: true}, + {name: "no start with immediate runs", immediate: true, wantStarted: true}, + {name: "no start without immediate waits"}, + {name: "expired end retires the schedule", endOffset: -time.Second, hasEnd: true, immediate: true, wantRetired: true}, + {name: "expired end retires even without immediate", endOffset: -5 * time.Minute, hasEnd: true, wantRetired: true}, + {name: "future end still runs", endOffset: time.Hour, hasEnd: true, immediate: true, wantStarted: true}, + {name: "maximum end still runs", maxEnd: true, immediate: true, wantStarted: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + now := time.Now().UTC() + options := ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + Interval: 5 * time.Minute, + StartImmediatelyIfLate: test.immediate, + } + if test.hasStart { + options.StartAt = now.Add(test.startOffset) + } + switch { + case test.maxEnd: + options.EndAt = maxTimestamp + case test.hasEnd: + options.EndAt = now.Add(test.endOffset) + } + + created := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, options) + requireOperationSucceeded(t, created) + token := decodeScheduleState(t, created.EntityState).ExecutionToken + + ran := runScheduleOperationBatch(t, executor, "daily", created.EntityState, runScheduleOperation, token) + requireOperationSucceeded(t, ran) + after := decodeScheduleState(t, ran.EntityState) + + switch { + case test.wantRetired: + require.Len(t, ran.Actions, 1) + require.Equal(t, deleteScheduleOperation, ran.Actions[0].GetSendSignal().GetName()) + require.True(t, after.LastRunAt.IsZero()) + require.True(t, after.NextRunAt.IsZero()) + case test.wantStarted: + require.Len(t, ran.Actions, 2) + require.NotNil(t, ran.Actions[0].GetStartNewOrchestration()) + require.Equal(t, runScheduleOperation, ran.Actions[1].GetSendSignal().GetName()) + require.False(t, after.LastRunAt.IsZero()) + require.True(t, after.NextRunAt.After(after.LastRunAt)) + default: + // The schedule is not due yet, so it only reschedules itself. + require.Len(t, ran.Actions, 1) + signal := ran.Actions[0].GetSendSignal() + require.Equal(t, runScheduleOperation, signal.GetName()) + require.True(t, after.LastRunAt.IsZero()) + require.True(t, after.NextRunAt.After(now)) + require.True(t, signal.GetScheduledTime().AsTime().Equal(after.NextRunAt)) + if test.hasStart && test.startOffset > 0 { + require.True(t, after.NextRunAt.Equal(options.StartAt.UTC())) + } + } + }) + } +} + +func TestScheduleEntityRunInstanceIDMatrix(t *testing.T) { + tests := []struct { + name string + configured string + want func(after scheduleState) string + }{ + { + name: "derived from the run time when unconfigured", + want: func(after scheduleState) string { + // The .NET round-trip format keeps 100ns precision and an + // explicit UTC offset. + return "daily-" + after.LastRunAt.UTC().Format("2006-01-02T15:04:05.0000000-07:00") + }, + }, + { + name: "configured instance id is used verbatim", + configured: "fixed-instance", + want: func(scheduleState) string { return "fixed-instance" }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + state := seededScheduleState(ScheduleStatusActive) + state.ScheduleConfiguration.StartImmediatelyIfLate = true + state.ScheduleConfiguration.OrchestrationInstanceID = test.configured + + ran := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(state), + runScheduleOperation, state.ExecutionToken) + requireOperationSucceeded(t, ran) + + start := ran.Actions[0].GetStartNewOrchestration() + require.NotNil(t, start) + require.Equal(t, test.want(decodeScheduleState(t, ran.EntityState)), start.InstanceId) + }) + } +} + +func TestScheduleEntityPauseAndResumeLifecycle(t *testing.T) { + executor := newScheduleExecutor(t) + created := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, validCreationOptions()) + requireOperationSucceeded(t, created) + createdState := decodeScheduleState(t, created.EntityState) + require.Equal(t, ScheduleStatusActive, createdState.Status) + require.Len(t, created.Actions, 1) + + paused := runScheduleOperationBatch(t, executor, "daily", created.EntityState, pauseScheduleOperation, nil) + requireOperationSucceeded(t, paused) + pausedState := decodeScheduleState(t, paused.EntityState) + require.Equal(t, ScheduleStatusPaused, pausedState.Status) + require.True(t, pausedState.NextRunAt.IsZero()) + require.NotEqual(t, createdState.ExecutionToken, pausedState.ExecutionToken) + require.Empty(t, paused.Actions, "pausing must not signal a run") + + resumed := runScheduleOperationBatch(t, executor, "daily", paused.EntityState, resumeScheduleOperation, nil) + requireOperationSucceeded(t, resumed) + resumedState := decodeScheduleState(t, resumed.EntityState) + require.Equal(t, ScheduleStatusActive, resumedState.Status) + require.Equal(t, pausedState.ExecutionToken, resumedState.ExecutionToken, "resume keeps the paused token") + require.Len(t, resumed.Actions, 1) + require.Equal(t, resumedState.ExecutionToken, decodeSignalToken(t, resumed.Actions[0].GetSendSignal())) +} + +func TestScheduleEntityCreateValidationMatrix(t *testing.T) { + now := time.Now().UTC() + tests := []struct { + name string + options ScheduleCreationOptions + }{ + {name: "missing schedule id", options: ScheduleCreationOptions{OrchestrationName: "Backup", Interval: time.Minute}}, + {name: "missing orchestration name", options: ScheduleCreationOptions{ScheduleID: "daily", Interval: time.Minute}}, + {name: "zero interval", options: ScheduleCreationOptions{ScheduleID: "daily", OrchestrationName: "Backup"}}, + {name: "sub second interval", options: ScheduleCreationOptions{ScheduleID: "daily", OrchestrationName: "Backup", Interval: 500 * time.Millisecond}}, + { + name: "end before start", + options: ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + StartAt: now.Add(2 * time.Hour), EndAt: now.Add(time.Hour), + }, + }, + { + name: "reserved tag prefix", + options: ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + Tags: map[string]string{tagcodec.UserTagPrefix + "schedule": "daily"}, + }, + }, + { + name: "empty tag key", + options: ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + Tags: map[string]string{"": "daily"}, + }, + }, + { + name: "reserved context field prefix", + options: ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + ContextFields: api.ContextFields{api.ReservedContextFieldPrefix + "tenant": "north"}, + }, + }, + { + name: "fixed instance id with retry policy", + options: ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + OrchestrationInstanceID: "fixed", + RetryPolicy: &ScheduleRetryPolicy{MaxAttempts: 2, InitialRetryInterval: time.Second}, + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + result := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, test.options) + requireOperationErrorType(t, result, scheduleValidationErrorType) + require.Nil(t, result.EntityState, "a rejected creation must not persist state") + }) + } +} + +func TestScheduleRejectsReservedTargetsBeforeMutation(t *testing.T) { + for _, path := range []string{"direct", "tags", "context"} { + for _, target := range []string{"explicit", "derived"} { + t.Run(path+"/"+target, func(t *testing.T) { + options := validCreationOptions() + if target == "explicit" { + options.OrchestrationInstanceID = "@counter@one" + } else { + options.ScheduleID = "@counter@one" + } + switch path { + case "tags": + options.Tags = map[string]string{"schedule": "daily"} + case "context": + options.ContextFields = api.ContextFields{"tenant": "one"} + } + executor := newScheduleExecutor(t) + for _, status := range []ScheduleStatus{ScheduleStatusUninitialized, ScheduleStatusActive, ScheduleStatusPaused} { + t.Run("create/"+status.String(), func(t *testing.T) { + var before *wrapperspb.StringValue + if status != ScheduleStatusUninitialized { + state := seededScheduleState(status) + state.ScheduleConfiguration.ScheduleID = options.ScheduleID + before = marshalScheduleState(state) + } + result := runScheduleOperationBatch(t, executor, options.ScheduleID, before, createScheduleOperation, options) + requireOperationErrorType(t, result, scheduleValidationErrorType) + require.Equal(t, before == nil, result.EntityState == nil) + require.Equal(t, before.GetValue(), result.EntityState.GetValue()) + require.Empty(t, result.Actions) + + backend := &recordingScheduleBackend{completion: &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: failure.FromProto(result.Results[0].GetFailure().GetFailureDetails()), + }} + client := &ScheduledTaskClient{client: backend, converter: api.DefaultDataConverter()} + _, err := client.Create(t.Context(), options) + require.ErrorIs(t, err, ErrScheduleValidation) + var typed *ScheduleValidationError + require.ErrorAs(t, err, &typed) + require.Contains(t, typed.Message, "reserved entity format") + }) + } + for _, status := range []ScheduleStatus{ScheduleStatusActive, ScheduleStatusPaused} { + t.Run("update/"+status.String(), func(t *testing.T) { + state := seededScheduleState(status) + state.ScheduleConfiguration.ScheduleID = options.ScheduleID + state.ScheduleConfiguration.Tags = options.Tags + state.ScheduleConfiguration.ContextFields = options.ContextFields + before := marshalScheduleState(state) + update := ScheduleUpdateOptions{ + OrchestrationName: pointerTo("Changed"), + Interval: pointerTo(6 * time.Minute), + } + if target == "explicit" { + update.OrchestrationInstanceID = &options.OrchestrationInstanceID + } + result := runScheduleOperationBatch(t, executor, options.ScheduleID, before, updateScheduleOperation, update) + requireOperationErrorType(t, result, scheduleValidationErrorType) + require.Equal(t, before.GetValue(), result.EntityState.GetValue()) + require.Empty(t, result.Actions) + }) + } + err := validateCreation(options) + require.ErrorIs(t, err, ErrScheduleValidation) + var typed *ScheduleValidationError + require.ErrorAs(t, err, &typed) + }) + } + } +} + +func TestScheduleValidatesOnlyEffectiveTargets(t *testing.T) { + for _, test := range []struct { + name, scheduleID, instanceID string + wrapper, retry bool + }{ + {name: "safe override", scheduleID: "@counter@one", instanceID: "safe"}, + {name: "wrapped safe override", scheduleID: "@counter@one", instanceID: "safe", wrapper: true}, + {name: "unused retry derived target", scheduleID: "@counter@one", retry: true}, + {name: "single at sign", scheduleID: "@counter"}, + {name: "empty entity name", scheduleID: "@@one"}, + {name: "wrapped derived target", scheduleID: "daily", wrapper: true}, + } { + t.Run(test.name, func(t *testing.T) { + options := validCreationOptions() + options.ScheduleID = test.scheduleID + options.OrchestrationInstanceID = test.instanceID + options.StartImmediatelyIfLate = true + if test.wrapper { + options.Tags = map[string]string{"schedule": "daily"} + options.ContextFields = api.ContextFields{"tenant": "one"} + } + if test.retry { + options.RetryPolicy = &ScheduleRetryPolicy{MaxAttempts: 2, InitialRetryInterval: time.Second} + } + require.NoError(t, validateCreation(options)) + registry := newScheduleRegistry(t) + executor := task.NewTaskExecutor(registry).(entityExecutor) + created := runScheduleOperationBatch(t, executor, options.ScheduleID, nil, createScheduleOperation, options) + requireOperationSucceeded(t, created) + updated := runScheduleOperationBatch(t, executor, options.ScheduleID, created.EntityState, + updateScheduleOperation, intervalUpdate(6*time.Minute)) + requireOperationSucceeded(t, updated) + state := decodeScheduleState(t, updated.EntityState) + ran := runScheduleOperationBatch(t, executor, options.ScheduleID, updated.EntityState, runScheduleOperation, state.ExecutionToken) + requireOperationSucceeded(t, ran) + require.Len(t, ran.Actions, 2) + started := ran.Actions[0].GetStartNewOrchestration() + require.NotNil(t, started) + require.NoError(t, helpers.ValidateOrchestrationInstanceID(started.InstanceId)) + targetID := started.InstanceId + if test.wrapper || test.retry { + require.Equal(t, ExecuteScheduledTaskOrchestratorName, started.Name) + response := executeScheduleOperationTurn(t, registry, nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(started.Name, "op", started.Input, nil, nil, nil), + }) + require.Len(t, response.Actions, 1) + target := response.Actions[0].GetCreateSubOrchestration() + require.NotNil(t, target) + require.Equal(t, "Backup", target.Name) + targetID = target.InstanceId + } + require.NoError(t, helpers.ValidateOrchestrationInstanceID(targetID)) + switch { + case test.retry: + require.Equal(t, "op:0000", targetID) + case test.instanceID != "": + require.Equal(t, test.instanceID, targetID) + default: + require.True(t, strings.HasPrefix(targetID, test.scheduleID+"-")) + } + require.Equal(t, runScheduleOperation, ran.Actions[1].GetSendSignal().GetName()) + }) + } +} + +func TestScheduleRejectsEmptyContextKeysBeforeMutation(t *testing.T) { + for _, status := range []ScheduleStatus{ScheduleStatusUninitialized, ScheduleStatusActive, ScheduleStatusPaused} { + for _, operation := range []string{createScheduleOperation, updateScheduleOperation} { + if status == ScheduleStatusUninitialized && operation == updateScheduleOperation { + continue + } + t.Run(operation+"/"+status.String(), func(t *testing.T) { + var before *wrapperspb.StringValue + if status != ScheduleStatusUninitialized { + before = marshalScheduleState(seededScheduleState(status)) + } + options := validCreationOptions() + options.ContextFields = api.ContextFields{"": "value"} + var input any = options + if operation == updateScheduleOperation { + input = ScheduleUpdateOptions{ + OrchestrationName: pointerTo("Changed"), + ContextFields: options.ContextFields, + } + } + result := runScheduleOperationBatch(t, newScheduleExecutor(t), options.ScheduleID, before, operation, input) + requireOperationErrorType(t, result, scheduleValidationErrorType) + require.Equal(t, before == nil, result.EntityState == nil) + require.Equal(t, before.GetValue(), result.EntityState.GetValue()) + require.Empty(t, result.Actions) + err := scheduleOperationFailure(options.ScheduleID, operation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: failure.FromProto(result.Results[0].GetFailure().GetFailureDetails()), + }) + require.ErrorIs(t, err, ErrScheduleValidation) + var typed *ScheduleValidationError + require.ErrorAs(t, err, &typed) + require.Equal(t, "context field key cannot be empty", typed.Message) + }) + } + } +} + +func TestScheduleNegativeIntervalIsRejectedBeforeTheWire(t *testing.T) { + require.ErrorIs(t, validateCreation(ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: -time.Minute, + }), ErrScheduleValidation) + + // The .NET TimeSpan wire format cannot express a negative interval, so the + // options never reach the entity. + _, err := json.Marshal(ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: -time.Minute, + }) + require.Error(t, err) +} + +func TestScheduleEntityInvalidTransitionCarriesScheduleIdentity(t *testing.T) { + tests := []struct { + name string + status ScheduleStatus + operation string + input any + useToken bool + to ScheduleStatus + }{ + {name: "update", status: ScheduleStatusUninitialized, operation: updateScheduleOperation, + input: intervalUpdate(time.Minute), to: ScheduleStatusUninitialized}, + {name: "pause", status: ScheduleStatusPaused, operation: pauseScheduleOperation, + to: ScheduleStatusPaused}, + {name: "resume", status: ScheduleStatusActive, operation: resumeScheduleOperation, + to: ScheduleStatusActive}, + {name: "run", status: ScheduleStatusPaused, operation: runScheduleOperation, useToken: true, + to: ScheduleStatusActive}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + state := seededScheduleState(test.status) + input := test.input + if test.useToken { + input = state.ExecutionToken + } + result := runScheduleOperationBatch(t, executor, "daily", marshalScheduleState(state), + test.operation, input) + details := result.Results[0].GetFailure().GetFailureDetails() + require.Equal(t, string(scheduleInvalidTransitionType), details.GetErrorType()) + require.Contains(t, details.GetErrorMessage(), `"daily"`) + + properties := decodeErrorProperties(t, details) + require.Equal(t, "daily", properties["scheduleId"]) + require.EqualValues(t, int(test.status), properties["from"]) + require.EqualValues(t, int(test.to), properties["to"]) + require.Equal(t, test.operation, properties["operation"]) + }) + } +} + +func TestScheduleEntityCreateAcceptsEqualStartAndEnd(t *testing.T) { + executor := newScheduleExecutor(t) + at := time.Now().UTC().Add(time.Hour) + result := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, StartAt: at, EndAt: at, + }) + requireOperationSucceeded(t, result) + state := decodeScheduleState(t, result.EntityState) + require.True(t, state.ScheduleConfiguration.StartAt.Equal(at)) + require.True(t, state.ScheduleConfiguration.EndAt.Equal(at)) +} + +func TestScheduleEntityRecreatePreservesCreationTimeAndClearsEnd(t *testing.T) { + executor := newScheduleExecutor(t) + created := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: 5 * time.Minute, + EndAt: time.Now().UTC().Add(time.Hour), + }) + requireOperationSucceeded(t, created) + first := decodeScheduleState(t, created.EntityState) + require.False(t, first.ScheduleConfiguration.EndAt.IsZero()) + + recreated := runScheduleOperationBatch(t, executor, "daily", created.EntityState, + createScheduleOperation, validCreationOptions()) + requireOperationSucceeded(t, recreated) + second := decodeScheduleState(t, recreated.EntityState) + require.True(t, second.ScheduleConfiguration.EndAt.IsZero(), "recreating must clear the end time") + require.True(t, first.ScheduleCreatedAt.Equal(second.ScheduleCreatedAt)) + require.NotEqual(t, first.ExecutionToken, second.ExecutionToken) + require.True(t, second.NextRunAt.IsZero()) +} + +func TestScheduleEntityRejectsUnknownOperation(t *testing.T) { + executor := newScheduleExecutor(t) + result := runScheduleOperationBatch(t, executor, "daily", + marshalScheduleState(seededScheduleState(ScheduleStatusActive)), "NotAnOperation", nil) + require.NotNil(t, result.Results[0].GetFailure()) + require.Contains(t, result.Results[0].GetFailure().GetFailureDetails().GetErrorMessage(), "NotAnOperation") +} + +// --------------------------------------------------------------------------- +// Text handling +// --------------------------------------------------------------------------- + +func TestScheduleEntityPreservesTextPayloads(t *testing.T) { + tests := []struct { + name string + scheduleID string + input string + }{ + {name: "special characters", scheduleID: "test@schedule#123$%^", input: `plain`}, + {name: "unicode", scheduleID: "スケジュール-日程-🌟", input: `{"名前":"値","emoji":"🌟"}`}, + {name: "maximum length id", scheduleID: strings.Repeat("a", 1000), input: `"payload"`}, + {name: "json special characters", scheduleID: "json", input: `{"key":"value with \"quotes\" and \\backslash\\ and /slash/"}`}, + {name: "html special characters", scheduleID: "html", input: `&<>"'`}, + {name: "multiline", scheduleID: "multiline", input: "line one\nline two\r\nline three\ttabbed"}, + {name: "base64", scheduleID: "base64", input: base64.StdEncoding.EncodeToString([]byte("scheduled task payload"))}, + {name: "large input", scheduleID: "large", input: strings.Repeat("x", 100000)}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := newScheduleExecutor(t) + created := runScheduleOperationBatch(t, executor, test.scheduleID, nil, createScheduleOperation, + ScheduleCreationOptions{ + ScheduleID: test.scheduleID, + OrchestrationName: "Backup", + Interval: 5 * time.Minute, + OrchestrationInput: test.input, + }) + requireOperationSucceeded(t, created) + createdState := decodeScheduleState(t, created.EntityState) + require.Equal(t, test.scheduleID, createdState.ScheduleConfiguration.ScheduleID) + require.Equal(t, test.input, createdState.ScheduleConfiguration.OrchestrationInput) + + updatedInput := test.input + "-updated" + updated := runScheduleOperationBatch(t, executor, test.scheduleID, created.EntityState, + updateScheduleOperation, ScheduleUpdateOptions{OrchestrationInput: pointerTo(updatedInput)}) + requireOperationSucceeded(t, updated) + updatedState := decodeScheduleState(t, updated.EntityState) + require.Equal(t, updatedInput, updatedState.ScheduleConfiguration.OrchestrationInput) + require.NotEqual(t, createdState.ExecutionToken, updatedState.ExecutionToken) + + description, err := scheduleDescription(&api.EntityMetadata{ + InstanceID: api.NewEntityID(ScheduleEntityName, test.scheduleID), + StateIncluded: true, + HasState: true, + SerializedState: updated.EntityState.GetValue(), + }, nil) + require.NoError(t, err) + require.Equal(t, test.scheduleID, description.ScheduleID) + require.Equal(t, updatedInput, description.OrchestrationInput) + }) + } +} + +func TestScheduleEntityUpdatesLongInstanceID(t *testing.T) { + executor := newScheduleExecutor(t) + created := runScheduleOperationBatch(t, executor, "daily", nil, createScheduleOperation, validCreationOptions()) + requireOperationSucceeded(t, created) + + longInstanceID := strings.Repeat("i", 1000) + updated := runScheduleOperationBatch(t, executor, "daily", created.EntityState, updateScheduleOperation, + ScheduleUpdateOptions{OrchestrationInstanceID: pointerTo(longInstanceID)}) + requireOperationSucceeded(t, updated) + require.Equal(t, longInstanceID, + decodeScheduleState(t, updated.EntityState).ScheduleConfiguration.OrchestrationInstanceID) +} + +// --------------------------------------------------------------------------- +// Operation orchestrator +// --------------------------------------------------------------------------- + +func TestExecuteScheduleOperationOrchestratorCallsEntity(t *testing.T) { + tests := []struct { + name string + input any + }{ + {name: "with input", input: ScheduleUpdateOptions{Interval: pointerTo(10 * time.Minute)}}, + {name: "without input", input: nil}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registry := newScheduleRegistry(t) + request := ScheduleOperationRequest{ + EntityID: scheduleEntityID("test-schedule"), + OperationName: updateScheduleOperation, + Input: test.input, + } + payload, err := api.DefaultDataConverter().Serialize(request) + require.NoError(t, err) + + response := executeScheduleOperationTurn(t, registry, nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(ExecuteScheduleOperationOrchestratorName, "op", wrapperspb.String(payload), nil, nil, nil), + }) + require.Len(t, response.Actions, 1) + called := response.Actions[0].GetSendEntityMessage().GetEntityOperationCalled() + require.NotNil(t, called) + require.Equal(t, scheduleEntityID("test-schedule").String(), called.TargetInstanceId.GetValue()) + require.Equal(t, updateScheduleOperation, called.Operation) + if test.input == nil { + require.Empty(t, called.Input.GetValue()) + } else { + require.Contains(t, called.Input.GetValue(), "00:10:00") + } + }) + } +} + +func TestExecuteScheduleOperationOrchestratorPropagatesResultAndFailure(t *testing.T) { + registry := newScheduleRegistry(t) + payload, err := api.DefaultDataConverter().Serialize(ScheduleOperationRequest{ + EntityID: scheduleEntityID("test-schedule"), + OperationName: pauseScheduleOperation, + }) + require.NoError(t, err) + + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent( + ExecuteScheduleOperationOrchestratorName, "op", wrapperspb.String(payload), nil, nil, nil) + first := executeScheduleOperationTurn(t, registry, nil, []*protos.HistoryEvent{started, executionStarted}) + called := first.Actions[0].GetSendEntityMessage().GetEntityOperationCalled() + require.NotNil(t, called) + + history := []*protos.HistoryEvent{started, executionStarted, { + EventId: first.Actions[0].Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationCalled{EntityOperationCalled: called}, + }} + + t.Run("result is propagated", func(t *testing.T) { + completed := &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{ + RequestId: called.RequestId, + Output: wrapperspb.String(`{"Status":2}`), + }, + }, + } + response := executeScheduleOperationTurn(t, registry, history, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), completed}) + completion := scheduleCompletionAction(t, response) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, completion.OrchestrationStatus) + require.JSONEq(t, `{"Status":2}`, completion.Result.GetValue()) + }) + + t.Run("failure is propagated", func(t *testing.T) { + failed := &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationFailed{ + EntityOperationFailed: &protos.EntityOperationFailedEvent{ + RequestId: called.RequestId, + FailureDetails: &protos.TaskFailureDetails{ + ErrorType: string(scheduleInvalidTransitionType), + ErrorMessage: "cannot pause a paused schedule", + }, + }, + }, + } + response := executeScheduleOperationTurn(t, registry, history, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), failed}) + completion := scheduleCompletionAction(t, response) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completion.OrchestrationStatus) + require.Contains(t, completion.FailureDetails.GetErrorMessage(), "cannot pause a paused schedule") + }) +} + +func TestExecuteScheduleOperationOrchestratorRequiresOperationName(t *testing.T) { + registry := newScheduleRegistry(t) + payload, err := api.DefaultDataConverter().Serialize(ScheduleOperationRequest{ + EntityID: scheduleEntityID("test-schedule"), + }) + require.NoError(t, err) + response := executeScheduleOperationTurn(t, registry, nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(ExecuteScheduleOperationOrchestratorName, "op", wrapperspb.String(payload), nil, nil, nil), + }) + completion := scheduleCompletionAction(t, response) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, completion.OrchestrationStatus) + require.Equal(t, string(scheduleValidationErrorType), completion.FailureDetails.GetErrorType()) +} + +// --------------------------------------------------------------------------- +// Client behavior +// --------------------------------------------------------------------------- + +func TestScheduleClientOperationsTargetTheEntity(t *testing.T) { + tests := []struct { + name string + operation string + invoke func(*ScheduleClient) error + }{ + {name: "create", operation: createScheduleOperation, invoke: func(c *ScheduleClient) error { + return c.Create(context.Background(), ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + }) + }}, + {name: "update", operation: updateScheduleOperation, invoke: func(c *ScheduleClient) error { + return c.Update(context.Background(), ScheduleUpdateOptions{Interval: pointerTo(time.Minute)}) + }}, + {name: "pause", operation: pauseScheduleOperation, invoke: func(c *ScheduleClient) error { + return c.Pause(context.Background()) + }}, + {name: "resume", operation: resumeScheduleOperation, invoke: func(c *ScheduleClient) error { + return c.Resume(context.Background()) + }}, + {name: "delete", operation: deleteScheduleOperation, invoke: func(c *ScheduleClient) error { + return c.Delete(context.Background()) + }}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &recordingScheduleBackend{} + handle, err := (&ScheduledTaskClient{client: backend}).GetScheduleClient("daily") + require.NoError(t, err) + require.NoError(t, test.invoke(handle)) + + require.Equal(t, ExecuteScheduleOperationOrchestratorName, backend.orchestrator) + require.NotNil(t, backend.version) + require.Empty(t, backend.version.GetValue()) + require.Equal(t, test.operation, backend.request.OperationName) + require.Equal(t, scheduleEntityID("daily"), backend.request.EntityID) + require.Equal(t, 1, backend.waitCount) + }) + } +} + +func TestScheduleClientFailureMatrix(t *testing.T) { + tests := []struct { + name string + metadata *api.OrchestrationMetadata + wantIs error + assertion func(*testing.T, error) + }{ + { + name: "invalid transition", + metadata: &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: scheduleInvalidTransitionType, + Properties: map[string]any{ + "scheduleId": "daily", + "from": float64(ScheduleStatusPaused), + "to": float64(ScheduleStatusPaused), + "operation": pauseScheduleOperation, + }, + }, + }, + wantIs: ErrScheduleInvalidTransition, + assertion: func(t *testing.T, err error) { + var transition *ScheduleInvalidTransitionError + require.ErrorAs(t, err, &transition) + require.Equal(t, "daily", transition.ScheduleID) + require.Equal(t, ScheduleStatusPaused, transition.From) + }, + }, + { + name: "namespaced invalid transition", + metadata: &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: api.ErrorType("Microsoft.DurableTask.ScheduledTasks." + string(scheduleInvalidTransitionType)), + }, + }, + wantIs: ErrScheduleInvalidTransition, + }, + { + name: "nested validation failure", + metadata: &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: api.ErrorTypeEntityOperationFailed, + InnerFailure: &api.FailureDetails{ + ErrorType: scheduleValidationErrorType, + Properties: map[string]any{"message": "interval must be positive"}, + }, + }, + }, + wantIs: ErrScheduleValidation, + assertion: func(t *testing.T, err error) { + var validation *ScheduleValidationError + require.ErrorAs(t, err, &validation) + require.Equal(t, "interval must be positive", validation.Message) + }, + }, + { + name: "generic failure", + metadata: &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorMessage: "boom"}, + }, + wantIs: ErrScheduleOperationFailed, + }, + { + name: "terminated without failure details", + metadata: &api.OrchestrationMetadata{RuntimeStatus: api.RUNTIME_STATUS_TERMINATED}, + wantIs: ErrScheduleOperationFailed, + assertion: func(t *testing.T, err error) { + var operation *ScheduleOperationError + require.ErrorAs(t, err, &operation) + require.Equal(t, api.RUNTIME_STATUS_TERMINATED, operation.RuntimeStatus) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &recordingScheduleBackend{completion: test.metadata} + handle, err := (&ScheduledTaskClient{client: backend}).GetScheduleClient("daily") + require.NoError(t, err) + for _, operate := range []func() error{ + func() error { return handle.Pause(context.Background()) }, + func() error { return handle.Resume(context.Background()) }, + func() error { return handle.Delete(context.Background()) }, + func() error { return handle.Update(context.Background(), ScheduleUpdateOptions{}) }, + } { + err := operate() + require.ErrorIs(t, err, test.wantIs) + if test.assertion != nil { + test.assertion(t, err) + } + } + }) + } +} + +func TestScheduleClientNotFoundMatrix(t *testing.T) { + tests := []struct { + name string + backend *recordingScheduleBackend + wantErr bool + wantNil bool + wantDesc bool + }{ + {name: "nil metadata", backend: &recordingScheduleBackend{}, wantErr: true, wantNil: true}, + { + name: "empty serialized state", + backend: &recordingScheduleBackend{entity: &api.EntityMetadata{InstanceID: scheduleEntityID("daily")}}, + wantErr: true, wantNil: true, + }, + { + name: "present schedule", + backend: &recordingScheduleBackend{entity: &api.EntityMetadata{ + InstanceID: scheduleEntityID("daily"), + StateIncluded: true, + HasState: true, + SerializedState: marshalScheduleState(seededScheduleState(ScheduleStatusActive)).GetValue(), + }}, + wantDesc: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + client := &ScheduledTaskClient{client: test.backend} + handle, err := client.GetScheduleClient("daily") + require.NoError(t, err) + + description, err := handle.Describe(context.Background()) + if test.wantErr { + require.ErrorIs(t, err, ErrScheduleNotFound) + var notFound *ScheduleNotFoundError + require.ErrorAs(t, err, ¬Found) + require.Equal(t, "daily", notFound.ScheduleID) + require.Nil(t, description) + } else { + require.NoError(t, err) + require.NotNil(t, description) + } + + got, err := client.Get(context.Background(), "daily") + require.NoError(t, err) + if test.wantNil { + require.Nil(t, got) + } + if test.wantDesc { + require.NotNil(t, got) + require.Equal(t, "daily", got.ScheduleID) + require.Equal(t, ScheduleStatusActive, got.Status) + require.Equal(t, "Backup", got.OrchestrationName) + require.Equal(t, 5*time.Minute, got.Interval) + } + }) + } +} + +func TestScheduleClientValidationMatrix(t *testing.T) { + client := &ScheduledTaskClient{client: &recordingScheduleBackend{}} + + _, err := client.GetScheduleClient("") + require.ErrorIs(t, err, ErrScheduleValidation) + + _, err = client.Create(context.Background(), ScheduleCreationOptions{OrchestrationName: "Backup"}) + require.ErrorIs(t, err, ErrScheduleValidation) + + handle, err := client.GetScheduleClient("daily") + require.NoError(t, err) + require.ErrorIs(t, + handle.Create(context.Background(), ScheduleCreationOptions{ScheduleID: "other", OrchestrationName: "Backup"}), + ErrScheduleValidation) + + var nilTasks *ScheduledTaskClient + _, err = nilTasks.GetScheduleClient("daily") + require.Error(t, err) + _, err = nilTasks.List(context.Background(), ScheduleQuery{}) + require.Error(t, err) + + var nilHandle *ScheduleClient + require.Empty(t, nilHandle.ID()) + require.Error(t, nilHandle.Pause(context.Background())) + _, err = nilHandle.Describe(context.Background()) + require.Error(t, err) + + require.Nil(t, NewScheduledTaskClient(nil)) + var nilClient *Client + require.Nil(t, nilClient.ScheduledTasks()) +} + +func TestScheduleClientCreateReturnsHandle(t *testing.T) { + backend := &recordingScheduleBackend{} + client := &ScheduledTaskClient{client: backend, converter: api.DefaultDataConverter()} + handle, err := client.Create(context.Background(), ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: 5 * time.Minute, + }) + require.NoError(t, err) + require.Equal(t, "daily", handle.ID()) + require.Equal(t, createScheduleOperation, backend.request.OperationName) +} + +func TestScheduleClientSerializesTypedInputWithConverter(t *testing.T) { + backend := &recordingScheduleBackend{} + client := &ScheduledTaskClient{client: backend, converter: prefixedConverter{}} + handle, err := client.GetScheduleClient("daily") + require.NoError(t, err) + + require.NoError(t, handle.Create(context.Background(), ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Minute, + TypedOrchestrationInput: map[string]string{"database": "main"}, + })) + creation, ok := backend.request.Input.(ScheduleCreationOptions) + require.True(t, ok) + require.Equal(t, `converter:{"database":"main"}`, creation.OrchestrationInput) + require.Nil(t, creation.TypedOrchestrationInput) + + require.NoError(t, handle.Update(context.Background(), ScheduleUpdateOptions{ + TypedOrchestrationInput: map[string]string{"database": "replica"}, + })) + update, ok := backend.request.Input.(ScheduleUpdateOptions) + require.True(t, ok) + require.NotNil(t, update.OrchestrationInput) + require.Equal(t, `converter:{"database":"replica"}`, *update.OrchestrationInput) + require.Nil(t, update.TypedOrchestrationInput) +} + +func TestScheduleClientListFiltersAndValidates(t *testing.T) { + created := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + entities := []*api.EntityMetadata{ + listedSchedule(t, "alpha", ScheduleStatusActive, created), + listedSchedule(t, "beta", ScheduleStatusPaused, created.Add(time.Hour)), + {InstanceID: scheduleEntityID("gamma")}, + } + active := ScheduleStatusActive + + tests := []struct { + name string + query ScheduleQuery + want []string + wantErr bool + }{ + {name: "no filter skips stateless entities", query: ScheduleQuery{}, want: []string{"alpha", "beta"}}, + {name: "status filter", query: ScheduleQuery{Status: &active}, want: []string{"alpha"}}, + {name: "created from is exclusive", query: ScheduleQuery{CreatedFrom: created}, want: []string{"beta"}}, + {name: "created to is exclusive", query: ScheduleQuery{CreatedTo: created.Add(time.Hour)}, want: []string{"alpha"}}, + {name: "negative page size", query: ScheduleQuery{PageSize: -1}, wantErr: true}, + {name: "oversized page size", query: ScheduleQuery{PageSize: api.MaxInstanceQueryPageSize + 1}, wantErr: true}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &recordingScheduleBackend{entities: entities} + client := &ScheduledTaskClient{client: backend} + result, err := client.List(context.Background(), test.query) + if test.wantErr { + require.ErrorIs(t, err, ErrScheduleValidation) + return + } + require.NoError(t, err) + ids := make([]string, 0, len(result.Schedules)) + for _, schedule := range result.Schedules { + ids = append(ids, schedule.ScheduleID) + } + require.Equal(t, test.want, ids) + require.Equal(t, "@schedule@"+test.query.ScheduleIDPrefix, backend.query.InstanceIDStartsWith) + require.False(t, backend.query.ExcludeState) + }) + } +} + +func TestScheduleClientListRequiresResult(t *testing.T) { + client := &ScheduledTaskClient{client: &recordingScheduleBackend{nilQueryResult: true}} + _, err := client.List(context.Background(), ScheduleQuery{}) + require.Error(t, err) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func newScheduleRegistry(t *testing.T) *task.TaskRegistry { + t.Helper() + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + return registry +} + +func newScheduleExecutor(t *testing.T) entityExecutor { + t.Helper() + return task.NewTaskExecutor(newScheduleRegistry(t)).(entityExecutor) +} + +func runScheduleOperationBatch( + t *testing.T, + executor entityExecutor, + scheduleID string, + state *wrapperspb.StringValue, + operation string, + input any, +) *protos.EntityBatchResult { + t.Helper() + request := &protos.EntityBatchRequest{ + InstanceId: scheduleEntityID(scheduleID).String(), + EntityState: state, + Operations: []*protos.OperationRequest{{Operation: operation}}, + } + if input != nil { + payload, err := api.DefaultDataConverter().Serialize(input) + require.NoError(t, err) + request.Operations[0].Input = wrapperspb.String(payload) + } + result, err := executor.ExecuteEntity(context.Background(), request) + require.NoError(t, err) + require.Len(t, result.Results, 1) + return result +} + +func requireOperationSucceeded(t *testing.T, result *protos.EntityBatchResult) { + t.Helper() + if failure := result.Results[0].GetFailure(); failure != nil { + t.Fatalf("expected success, got %s: %s", + failure.GetFailureDetails().GetErrorType(), failure.GetFailureDetails().GetErrorMessage()) + } +} + +func requireOperationErrorType(t *testing.T, result *protos.EntityBatchResult, errorType api.ErrorType) { + t.Helper() + failure := result.Results[0].GetFailure() + require.NotNil(t, failure, "expected the operation to fail") + require.Equal(t, string(errorType), failure.GetFailureDetails().GetErrorType()) +} + +func decodeErrorProperties(t *testing.T, details *protos.TaskFailureDetails) map[string]any { + t.Helper() + require.NotNil(t, details) + properties := make(map[string]any, len(details.GetProperties())) + for key, value := range details.GetProperties() { + properties[key] = value.AsInterface() + } + return properties +} + +func decodeScheduleState(t *testing.T, state *wrapperspb.StringValue) scheduleState { + t.Helper() + require.NotNil(t, state) + var decoded scheduleState + require.NoError(t, json.Unmarshal([]byte(state.GetValue()), &decoded)) + return decoded +} + +// marshalScheduleState encodes state the way the service persists it. Encoding +// cannot fail for a well-formed scheduleState, so it panics rather than taking +// a *testing.T that most call sites would only forward. +func marshalScheduleState(state scheduleState) *wrapperspb.StringValue { + payload, err := json.Marshal(state) + if err != nil { + panic(err) + } + return wrapperspb.String(string(payload)) +} + +func decodeSignalToken(t *testing.T, signal *protos.SendSignalAction) string { + t.Helper() + require.NotNil(t, signal) + var token string + require.NoError(t, api.DefaultDataConverter().Deserialize(signal.GetInput().GetValue(), &token)) + return token +} + +func seededScheduleState(status ScheduleStatus) scheduleState { + created := time.Date(2026, time.August, 1, 0, 0, 0, 0, time.UTC) + return scheduleState{ + Status: status, + ExecutionToken: "0123456789abcdef0123456789abcdee", + ScheduleCreatedAt: created, + ScheduleLastModifiedAt: created, + ScheduleConfiguration: &scheduleConfiguration{ + ScheduleID: "daily", + OrchestrationName: "Backup", + Interval: dotNetSpan(5 * time.Minute), + }, + } +} + +func validCreationOptions() ScheduleCreationOptions { + return ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + Interval: 5 * time.Minute, + } +} + +func intervalUpdate(interval time.Duration) ScheduleUpdateOptions { + return ScheduleUpdateOptions{Interval: pointerTo(interval)} +} + +func listedSchedule(t *testing.T, id string, status ScheduleStatus, created time.Time) *api.EntityMetadata { + t.Helper() + state := seededScheduleState(status) + state.ScheduleCreatedAt = created + state.ScheduleConfiguration.ScheduleID = id + return &api.EntityMetadata{ + InstanceID: scheduleEntityID(id), + StateIncluded: true, + HasState: true, + SerializedState: marshalScheduleState(state).GetValue(), + } +} + +func executeScheduleOperationTurn( + t *testing.T, + registry *task.TaskRegistry, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, +) *protos.OrchestratorResponse { + t.Helper() + result, err := task.NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + api.InstanceID("op"), + oldEvents, + newEvents, + &protos.OrchestratorEntityParameters{EntityMessageReorderWindow: durationpb.New(0)}, + ) + + require.NoError(t, err) + return result.Response +} + +func scheduleCompletionAction(t *testing.T, response *protos.OrchestratorResponse) *protos.CompleteOrchestrationAction { + t.Helper() + for _, action := range response.Actions { + if completion := action.GetCompleteOrchestration(); completion != nil { + return completion + } + } + t.Fatal("expected the orchestration to complete") + return nil +} + +func pointerTo[T any](value T) *T { return &value } + +// recordingScheduleBackend captures the requests issued by the schedule clients. +type recordingScheduleBackend struct { + orchestrator string + version *wrapperspb.StringValue + request ScheduleOperationRequest + waitCount int + completion *api.OrchestrationMetadata + entity *api.EntityMetadata + fetchErr error + entities []*api.EntityMetadata + query api.EntityQuery + nilQueryResult bool +} + +func (b *recordingScheduleBackend) ScheduleNewOrchestration( + _ context.Context, + orchestrator string, + options ...api.NewOrchestrationOptions, +) (api.InstanceID, error) { + request := &protos.CreateInstanceRequest{} + for _, option := range options { + if err := option(request, api.DefaultDataConverter()); err != nil { + return "", err + } + } + b.orchestrator = orchestrator + b.version = request.Version + if err := json.Unmarshal([]byte(request.Input.GetValue()), &b.request); err != nil { + return "", err + } + // Re-decode the operation input into its concrete type so tests can assert + // on converter-produced payloads. + var envelope struct{ Input json.RawMessage } + if err := json.Unmarshal([]byte(request.Input.GetValue()), &envelope); err == nil && len(envelope.Input) > 0 { + switch b.request.OperationName { + case createScheduleOperation: + var options ScheduleCreationOptions + if err := json.Unmarshal(envelope.Input, &options); err == nil { + b.request.Input = options + } + case updateScheduleOperation: + var options ScheduleUpdateOptions + if err := json.Unmarshal(envelope.Input, &options); err == nil { + b.request.Input = options + } + } + } + return "operation", nil +} + +func (b *recordingScheduleBackend) WaitForOrchestrationCompletion( + context.Context, + api.InstanceID, + ...api.FetchOrchestrationMetadataOptions, +) (*api.OrchestrationMetadata, error) { + b.waitCount++ + if b.completion != nil { + return b.completion, nil + } + return &api.OrchestrationMetadata{RuntimeStatus: api.RUNTIME_STATUS_COMPLETED}, nil +} + +func (b *recordingScheduleBackend) GetEntity( + context.Context, + api.EntityID, + ...api.GetEntityOptions, +) (*api.EntityMetadata, error) { + return b.entity, b.fetchErr +} + +func (b *recordingScheduleBackend) QueryEntities( + _ context.Context, + query api.EntityQuery, +) (*api.EntityQueryResults, error) { + b.query = query + if b.nilQueryResult { + return nil, nil + } + return &api.EntityQueryResults{Entities: b.entities}, nil +} + +var _ scheduledTaskBackend = (*recordingScheduleBackend)(nil) diff --git a/durabletaskscheduler/scheduled_tasks_test.go b/durabletaskscheduler/scheduled_tasks_test.go new file mode 100644 index 00000000..21f5b81f --- /dev/null +++ b/durabletaskscheduler/scheduled_tasks_test.go @@ -0,0 +1,544 @@ +package durabletaskscheduler + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestRegisterScheduledTasksUsesUnversionedSystemNames(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + snapshot := registry.Snapshot() + require.Contains(t, snapshot.Entities, "schedule") + require.Contains(t, snapshot.Orchestrators, task.TaskRegistration{ + Name: ExecuteScheduleOperationOrchestratorName, + }) + require.Contains(t, snapshot.Orchestrators, task.TaskRegistration{ + Name: ExecuteScheduledTaskOrchestratorName, + }) + require.NotContains(t, snapshot.Orchestrators, task.TaskRegistration{ + Name: ExecuteScheduleOperationOrchestratorName, Version: "v1", + }) + require.NotNil(t, WithScheduledTasks()) +} + +func TestScheduleStateUsesDotNetCompatibleJSON(t *testing.T) { + state := scheduleState{ + Status: ScheduleStatusActive, + ExecutionToken: "0123456789abcdef0123456789abcdef", + ScheduleConfiguration: &scheduleConfiguration{ + ScheduleID: "nightly", + OrchestrationName: "Backup", + Interval: dotNetSpan(5 * time.Minute), + }, + } + payload, err := json.Marshal(state) + require.NoError(t, err) + require.JSONEq(t, `{"Status":1,"ExecutionToken":"0123456789abcdef0123456789abcdef","LastRunAt":null,"NextRunAt":null,"ScheduleCreatedAt":null,"ScheduleLastModifiedAt":null,"ScheduleConfiguration":{"OrchestrationName":"Backup","ScheduleId":"nightly","StartAt":null,"EndAt":null,"Interval":"00:05:00","StartImmediatelyIfLate":false}}`, string(payload)) + + var decoded scheduleState + require.NoError(t, json.Unmarshal([]byte(`{ + "Status": 2, + "ExecutionToken": "token", + "ScheduleConfiguration": { + "ScheduleId": "from-dotnet", + "OrchestrationName": "Run", + "Interval": "1.02:03:04.5000000", + "StartImmediatelyIfLate": true + } + }`), &decoded)) + require.Equal(t, ScheduleStatusPaused, decoded.Status) + require.Equal(t, 26*time.Hour+3*time.Minute+4*time.Second+500*time.Millisecond, + time.Duration(decoded.ScheduleConfiguration.Interval)) + span, err := json.Marshal(dotNetSpan(26*time.Hour + 3*time.Minute + 4*time.Second + 500*time.Millisecond)) + require.NoError(t, err) + require.Equal(t, `"1.02:03:04.5000000"`, string(span)) +} + +func TestScheduleOptionsUseDotNetCompatibleWireJSON(t *testing.T) { + payload, err := json.Marshal(ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + Interval: 5 * time.Minute, + }) + require.NoError(t, err) + require.JSONEq(t, `{ + "ScheduleId":"daily", + "OrchestrationName":"Backup", + "Interval":"00:05:00", + "StartImmediatelyIfLate":false + }`, string(payload)) + + var decoded ScheduleCreationOptions + require.NoError(t, json.Unmarshal([]byte(`{ + "ScheduleId":"dotnet", + "OrchestrationName":"Run", + "Interval":"1.02:03:04.5000000", + "StartImmediatelyIfLate":true + }`), &decoded)) + require.Equal(t, "dotnet", decoded.ScheduleID) + require.Equal(t, 26*time.Hour+3*time.Minute+4*time.Second+500*time.Millisecond, decoded.Interval) +} + +func TestDetermineNextRunHandlesLateStartWithoutOverflow(t *testing.T) { + now := time.Date(2026, 8, 27, 12, 0, 0, 0, time.UTC) + state := &scheduleState{ScheduleCreatedAt: now.Add(-100000 * time.Hour)} + config := &scheduleConfiguration{ + Interval: dotNetSpan(time.Second), + } + next, err := determineNextRun(state, config, now) + require.NoError(t, err) + require.Equal(t, now.Add(time.Second), next) + + state.LastRunAt = now + config.StartImmediatelyIfLate = true + next, err = determineNextRun(state, config, now) + require.NoError(t, err) + require.Equal(t, now.Add(time.Second), next) +} + +func TestScheduleErrorsSupportIsAndAs(t *testing.T) { + err := invalidTransition("id", ScheduleStatusActive, ScheduleStatusPaused, pauseScheduleOperation) + require.ErrorIs(t, err, ErrScheduleInvalidTransition) + var transition *ScheduleInvalidTransitionError + require.ErrorAs(t, err, &transition) + require.Equal(t, "id", transition.ScheduleID) + + notFound := &ScheduleNotFoundError{ScheduleID: "missing"} + require.ErrorIs(t, notFound, ErrScheduleNotFound) + var typedNotFound *ScheduleNotFoundError + require.ErrorAs(t, notFound, &typedNotFound) +} + +func TestScheduleClientTargetsUnversionedOperationOrchestrator(t *testing.T) { + backend := &scheduleClientBackend{} + client := &ScheduledTaskClient{client: backend} + handle, err := client.GetScheduleClient("daily") + require.NoError(t, err) + require.Equal(t, "daily", handle.ID()) + require.NoError(t, handle.Pause(context.Background())) + require.Equal(t, ExecuteScheduleOperationOrchestratorName, backend.orchestrator) + require.NotNil(t, backend.version) + require.Empty(t, backend.version.GetValue()) +} + +func TestScheduleClientGetReturnsNilForMissingSchedule(t *testing.T) { + backend := &scheduleClientBackend{} + client := &ScheduledTaskClient{client: backend} + description, err := client.Get(context.Background(), "missing") + require.NoError(t, err) + require.Nil(t, description) +} + +func TestScheduleEntityCreatesPascalStateAndRunSignal(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + executor := task.NewTaskExecutor(registry) + input, err := api.DefaultDataConverter().Serialize(ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + OrchestrationInput: `{"database":"main"}`, + Interval: time.Hour, + StartImmediatelyIfLate: true, + }) + require.NoError(t, err) + + result, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + Operations: []*protos.OperationRequest{{ + Operation: createScheduleOperation, + Input: wrapperspb.String(input), + }}, + }) + require.NoError(t, err) + require.Len(t, result.Actions, 1) + require.Equal(t, runScheduleOperation, result.Actions[0].GetSendSignal().GetName()) + require.Contains(t, result.EntityState.GetValue(), `"Status":1`) + require.Contains(t, result.EntityState.GetValue(), `"Interval":"01:00:00"`) +} + +func TestScheduleEntityOperationNamesAreCaseInsensitive(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + executor := task.NewTaskExecutor(registry) + input, err := api.DefaultDataConverter().Serialize(ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Hour, + }) + require.NoError(t, err) + result, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + Operations: []*protos.OperationRequest{{ + Operation: strings.ToLower(createScheduleOperation), + Input: wrapperspb.String(input), + }}, + }) + require.NoError(t, err) + require.NotNil(t, result.EntityState) +} + +func TestScheduleEntityRunsOnceAndCancelsStaleTokens(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasksWithDefaultVersion(registry, "v1")) + executor := task.NewTaskExecutor(registry) + input, err := api.DefaultDataConverter().Serialize(ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + OrchestrationInput: `{"database":"main"}`, + Interval: time.Hour, + StartAt: time.Now().Add(-time.Minute), + StartImmediatelyIfLate: true, + }) + require.NoError(t, err) + created, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + Operations: []*protos.OperationRequest{{ + Operation: createScheduleOperation, + Input: wrapperspb.String(input), + }}, + }) + require.NoError(t, err) + var state scheduleState + require.NoError(t, json.Unmarshal([]byte(created.EntityState.GetValue()), &state)) + + token, err := api.DefaultDataConverter().Serialize(state.ExecutionToken) + require.NoError(t, err) + ran, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: created.EntityState, + Operations: []*protos.OperationRequest{{ + Operation: runScheduleOperation, + Input: wrapperspb.String(token), + }}, + }) + require.NoError(t, err) + require.Len(t, ran.Actions, 2) + require.Equal(t, `{"database":"main"}`, ran.Actions[0].GetStartNewOrchestration().Input.GetValue()) + require.Equal(t, "v1", ran.Actions[0].GetStartNewOrchestration().Version.GetValue()) + require.Contains(t, ran.Actions[0].GetStartNewOrchestration().InstanceId, "+00:00") + require.Equal(t, runScheduleOperation, ran.Actions[1].GetSendSignal().GetName()) + + stale, err := api.DefaultDataConverter().Serialize("not-current") + require.NoError(t, err) + staleResult, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: ran.EntityState, + Operations: []*protos.OperationRequest{{ + Operation: runScheduleOperation, + Input: wrapperspb.String(stale), + }}, + }) + require.NoError(t, err) + require.Empty(t, staleResult.Actions) +} + +func TestScheduleEntityUsesWrapperForTagsContextAndRetries(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasksWithDefaultVersion(registry, "v2")) + executor := task.NewTaskExecutor( + registry, + task.WithUnversionedOrchestratorNames( + ExecuteScheduleOperationOrchestratorName, + ExecuteScheduledTaskOrchestratorName, + ), + ) + input, err := api.DefaultDataConverter().Serialize(ScheduleCreationOptions{ + ScheduleID: "daily", + OrchestrationName: "Backup", + OrchestrationInput: `{"database":"main"}`, + Interval: time.Hour, + StartAt: time.Now().Add(-time.Minute), + StartImmediatelyIfLate: true, + Tags: map[string]string{"schedule": "daily"}, + ContextFields: api.ContextFields{"tenant": "north"}, + RetryPolicy: &ScheduleRetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: time.Second, + BackoffCoefficient: 2, + MaxRetryInterval: time.Minute, + RetryTimeout: time.Hour, + }, + }) + require.NoError(t, err) + created, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + Operations: []*protos.OperationRequest{{ + Operation: createScheduleOperation, + Input: wrapperspb.String(input), + }}, + }) + require.NoError(t, err) + var state scheduleState + require.NoError(t, json.Unmarshal([]byte(created.EntityState.GetValue()), &state)) + token, err := api.DefaultDataConverter().Serialize(state.ExecutionToken) + require.NoError(t, err) + ran, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: created.EntityState, + Operations: []*protos.OperationRequest{{ + Operation: runScheduleOperation, + Input: wrapperspb.String(token), + }}, + }) + require.NoError(t, err) + start := ran.Actions[0].GetStartNewOrchestration() + require.NotNil(t, start) + require.Equal(t, ExecuteScheduledTaskOrchestratorName, start.Name) + require.NotNil(t, start.Version) + require.Empty(t, start.Version.GetValue()) + var request scheduledTaskRequest + require.NoError(t, api.DefaultDataConverter().Deserialize(start.Input.GetValue(), &request)) + require.Equal(t, "Backup", request.OrchestrationName) + require.Equal(t, "v2", request.OrchestrationVersion) + require.Equal(t, map[string]string{"schedule": "daily"}, request.Tags) + require.Equal(t, api.ContextFields{"tenant": "north"}, request.ContextFields) + require.Equal(t, 3, request.RetryPolicy.MaxAttempts) +} + +func TestScheduleEntityStateIsJSONWithCustomInputConverter(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + converter := prefixedConverter{} + executor := task.NewTaskExecutor(registry, task.WithDataConverter(converter)) + input, err := converter.Serialize(ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Hour, + TypedOrchestrationInput: map[string]string{ + "database": "main", + }, + }) + require.NoError(t, err) + result, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + Operations: []*protos.OperationRequest{{ + Operation: createScheduleOperation, + Input: wrapperspb.String(input), + }}, + }) + require.NoError(t, err) + require.False(t, strings.HasPrefix(result.EntityState.GetValue(), "converter:")) + var state scheduleState + require.NoError(t, json.Unmarshal([]byte(result.EntityState.GetValue()), &state)) + require.Equal(t, "daily", state.ScheduleConfiguration.ScheduleID) + require.Equal(t, `converter:{"database":"main"}`, state.ScheduleConfiguration.OrchestrationInput) +} + +func TestScheduleDescriptionReadsInputWithConverter(t *testing.T) { + state, err := json.Marshal(scheduleState{ + Status: ScheduleStatusActive, + ScheduleConfiguration: &scheduleConfiguration{ + ScheduleID: "daily", + OrchestrationName: "Backup", + OrchestrationInput: `converter:"main"`, + Interval: dotNetSpan(time.Hour), + }, + }) + require.NoError(t, err) + description, err := scheduleDescription(&api.EntityMetadata{ + InstanceID: api.NewEntityID(ScheduleEntityName, "daily"), + SerializedState: string(state), + }, prefixedConverter{}) + require.NoError(t, err) + var input string + require.NoError(t, description.ReadInput(&input)) + require.Equal(t, "main", input) +} + +func TestScheduleOperationFailureReturnsTypedErrors(t *testing.T) { + err := scheduleOperationFailure("daily", pauseScheduleOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: scheduleInvalidTransitionType, + Properties: map[string]any{ + "scheduleId": "daily", + "from": float64(ScheduleStatusPaused), + "to": float64(ScheduleStatusPaused), + "operation": pauseScheduleOperation, + }, + }, + }) + require.ErrorIs(t, err, ErrScheduleInvalidTransition) + var transition *ScheduleInvalidTransitionError + require.ErrorAs(t, err, &transition) + require.Equal(t, ScheduleStatusPaused, transition.From) + + err = scheduleOperationFailure("daily", "Run", &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorMessage: "failed"}, + }) + require.ErrorIs(t, err, ErrScheduleOperationFailed) + var operation *ScheduleOperationError + require.ErrorAs(t, err, &operation) + require.Equal(t, "failed", operation.FailureDetails.ErrorMessage) +} + +func TestScheduleEntityDoesNotRecreateDeletedStateForStaleRun(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + executor := task.NewTaskExecutor(registry) + token, err := api.DefaultDataConverter().Serialize("stale") + require.NoError(t, err) + state, err := json.Marshal(scheduleState{Status: ScheduleStatusUninitialized}) + require.NoError(t, err) + result, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: wrapperspb.String(string(state)), + Operations: []*protos.OperationRequest{{ + Operation: runScheduleOperation, + Input: wrapperspb.String(token), + }}, + }) + require.NoError(t, err) + require.Nil(t, result.EntityState) +} + +func TestScheduleEntityIgnoresStaleRetirementDelete(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, RegisterScheduledTasks(registry)) + executor := task.NewTaskExecutor(registry) + state, err := json.Marshal(scheduleState{ + Status: ScheduleStatusActive, + ExecutionToken: "current", + ScheduleConfiguration: &scheduleConfiguration{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: dotNetSpan(time.Hour), + }, + }) + require.NoError(t, err) + staleToken, err := api.DefaultDataConverter().Serialize("stale") + require.NoError(t, err) + result, err := executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: wrapperspb.String(string(state)), + Operations: []*protos.OperationRequest{{ + Operation: deleteScheduleOperation, + Input: wrapperspb.String(staleToken), + }}, + }) + require.NoError(t, err) + require.NotNil(t, result.EntityState) + + result, err = executor.(entityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID(ScheduleEntityName, "daily").String(), + EntityState: result.EntityState, + Operations: []*protos.OperationRequest{{ + Operation: deleteScheduleOperation, + }}, + }) + require.NoError(t, err) + require.Nil(t, result.EntityState) +} + +func TestCreateReactivatesPausedSchedule(t *testing.T) { + now := time.Now().UTC() + state := scheduleState{ + Status: ScheduleStatusPaused, + ScheduleCreatedAt: now.Add(-time.Hour), + ScheduleConfiguration: &scheduleConfiguration{ + ScheduleID: "daily", + }, + } + ctx := &task.EntityContext{ID: api.NewEntityID(ScheduleEntityName, "daily")} + require.NoError(t, createSchedule(ctx, &state, ScheduleCreationOptions{ + ScheduleID: "daily", OrchestrationName: "Backup", Interval: time.Hour, + })) + require.Equal(t, ScheduleStatusActive, state.Status) + require.Equal(t, now.Add(-time.Hour).Unix(), state.ScheduleCreatedAt.Unix()) +} + +func TestValidateCreation(t *testing.T) { + err := validateCreation(ScheduleCreationOptions{ + ScheduleID: "id", + OrchestrationName: "run", + Interval: time.Millisecond, + }) + require.ErrorIs(t, err, ErrScheduleValidation) + require.False(t, errors.Is(err, ErrScheduleNotFound)) + + err = validateCreation(ScheduleCreationOptions{ + ScheduleID: "id", + OrchestrationName: "run", + OrchestrationInstanceID: "fixed", + Interval: time.Second, + RetryPolicy: &ScheduleRetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: time.Second, + }, + }) + require.ErrorIs(t, err, ErrScheduleValidation) +} + +type scheduleClientBackend struct { + orchestrator string + version *wrapperspb.StringValue + fetchErr error +} + +func (b *scheduleClientBackend) ScheduleNewOrchestration(_ context.Context, orchestrator string, options ...api.NewOrchestrationOptions) (api.InstanceID, error) { + request := &protos.CreateInstanceRequest{} + for _, option := range options { + if err := option(request, api.DefaultDataConverter()); err != nil { + return "", err + } + } + b.orchestrator = orchestrator + b.version = request.Version + return "operation", nil +} + +func (b *scheduleClientBackend) WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) { + return &api.OrchestrationMetadata{RuntimeStatus: api.RUNTIME_STATUS_COMPLETED}, nil +} + +func (b *scheduleClientBackend) GetEntity(context.Context, api.EntityID, ...api.GetEntityOptions) (*api.EntityMetadata, error) { + return nil, b.fetchErr +} + +func (b *scheduleClientBackend) QueryEntities(context.Context, api.EntityQuery) (*api.EntityQueryResults, error) { + return nil, nil +} + +type entityExecutor interface { + ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) +} + +type prefixedConverter struct{} + +func (prefixedConverter) Serialize(value any) (string, error) { + payload, err := json.Marshal(value) + return "converter:" + string(payload), err +} + +func (prefixedConverter) Deserialize(payload string, target any) error { + if !strings.HasPrefix(payload, "converter:") { + return fmt.Errorf("missing converter prefix") + } + return json.Unmarshal([]byte(strings.TrimPrefix(payload, "converter:")), target) +} + +var _ scheduledTaskBackend = (*scheduleClientBackend)(nil) +var _ durabletaskclient.TaskHubGrpcWorkerOption = WithScheduledTasks() + +func TestScheduleRetryPolicyFromPublicAppliesDefaultsWithoutMutation(t *testing.T) { + public := &ScheduleRetryPolicy{InitialRetryInterval: time.Second} + normalized, err := scheduleRetryPolicyFromPublic(public) + require.NoError(t, err) + require.Equal(t, 1, normalized.MaxAttempts) + require.Equal(t, 1.0, normalized.BackoffCoefficient) + require.Equal(t, dotNetSpan(math.MaxInt64), normalized.MaxRetryInterval) + require.Equal(t, dotNetSpan(math.MaxInt64), normalized.RetryTimeout) + require.Zero(t, public.MaxAttempts) + require.Zero(t, public.BackoffCoefficient) + require.Zero(t, public.MaxRetryInterval) + require.Zero(t, public.RetryTimeout) +} diff --git a/durabletaskscheduler/worker.go b/durabletaskscheduler/worker.go new file mode 100644 index 00000000..0889b0b2 --- /dev/null +++ b/durabletaskscheduler/worker.go @@ -0,0 +1,65 @@ +package durabletaskscheduler + +import ( + "context" + "io" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/task" + "google.golang.org/grpc" +) + +// NewWorker creates a DTS worker that owns and recreates its gRPC connections. +// The returned worker connects and performs its fail-fast Hello in Start or Run. +func NewWorker( + options *Options, + registry *task.TaskRegistry, + logger api.Logger, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) (*durabletaskclient.TaskHubGrpcWorker, error) { + prepared, err := prepareOptions(options) + if err != nil { + return nil, err + } + workerID := prepared.WorkerID + if workerID == "" { + workerID = defaultWorkerID() + } + + factory := func(ctx context.Context) (grpc.ClientConnInterface, io.Closer, error) { + connection, err := connect(&prepared, workerRole, workerID) + if err != nil { + return nil, nil, err + } + return connection, connection, nil + } + configuredOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithWorkerHelloTimeout(prepared.HelloTimeout), + durabletaskclient.WithWorkerDataConverter(prepared.DataConverter), + durabletaskclient.WithMaximumTimerInterval(prepared.MaximumTimerInterval), + durabletaskclient.WithMaxOrchestratorCompletionBytes(min( + prepared.MaxSendMessageSize, + durabletaskclient.DefaultMaxOrchestratorCompletionBytes, + )), + } + if prepared.LargePayloads != nil { + configuredOptions = append( + configuredOptions, + durabletaskclient.WithWorkerLargePayloads(prepared.LargePayloads), + ) + } + if prepared.Versioning != nil { + configuredOptions = append( + configuredOptions, + durabletaskclient.WithTaskVersioning(*prepared.Versioning), + ) + } + configuredOptions = append(configuredOptions, workerOptions...) + return durabletaskclient.NewTaskHubGrpcWorkerWithConnectionFactory( + factory, + registry, + logger, + configuredOptions..., + ) +} diff --git a/exporthistory/README.md b/exporthistory/README.md new file mode 100644 index 00000000..d76b5faa --- /dev/null +++ b/exporthistory/README.md @@ -0,0 +1,239 @@ +# History export (preview) + +`exporthistory` exports terminal orchestration histories out of a task hub and +into Azure Blob Storage. It is the Go counterpart of the .NET +`Microsoft.DurableTask.ExportHistory` preview package, but its export-job +lifecycle protocol is Go-specific. Mixed .NET or older Go export-history workers +are not supported in the same task hub. + +> **Maturity: preview.** The exported API, the serialized shape of the +> `ExportJob` entity state, and the names of the system entity, orchestrators, +> and activities may change in a future release without a major version bump. +> Drain and delete export jobs before upgrading. Legacy export-job state is not +> supported or migrated. + +## Model + +An export job is a durable entity keyed by job ID. + +| Component | Name | Role | +| --- | --- | --- | +| Entity | `ExportJob` | Owns lifecycle, configuration, checkpoint, and progress | +| Orchestrator | `ExecuteExportJobOperationOrchestrator` | Runs one entity operation on behalf of a client | +| Orchestrator | `ExportJobOrchestrator` | Lists, exports, and checkpoints; one per run generation | +| Activity | `ListTerminalInstancesActivity` | Returns one page of terminal instance IDs | +| Activity | `ExportInstanceHistoryActivity` | Exports one instance's history | + +Creating a job moves the entity to `Active`, mints a run token, and reserves the +generation-specific instance ID `ExportJob--` before signaling +`Run`. `Run` starts `ExportJobOrchestrator` using that stored ID. Read +`description.OrchestratorInstanceID` to identify the current generation; a job +ID alone no longer identifies an orchestration. The orchestration repeatedly +lists terminal instances matching the job's filter, +exports each instance's history in bounded parallel windows, and commits a +checkpoint back to the entity. A batch job completes when the task hub reports +no more pages; a continuous job idles for a minute and lists again. + +### Lifecycle + +| Operation | From | To | +| --- | --- | --- | +| `Create` | `Pending`, `Failed`, `Completed` | `Active` | +| `MarkAsCompleted` | `Active` | `Completed` | +| `MarkAsFailed` | `Active` | `Failed` | + +Every other transition raises an `InvalidTransitionError`, including recreating +a job that is already `Active`. Recreating a `Failed` or `Completed` job resets +its progress counters and checkpoint but preserves its original creation time. +Each successful `Create` assigns a fresh orchestration ID, so concurrent creates +are resolved entirely by the entity's lifecycle transition: only one generation +can be `Active` at a time, and creation performs no destructive cleanup. Previous generations' +orchestration histories are retained, not purged on recreation. + +A page whose exports keep failing commits without a checkpoint and with the +collected failures, which implicitly moves the job to `Failed` and leaves the +cursor on the failing page so a fix can resume from it. That implicit failure +goes through the same lifecycle transition as an explicit `MarkAsFailed`. + +The entity's `Delete` atomically captures its current orchestration ID, clears +the state, and returns that ID as a string (an empty string if the job is absent). +The client then terminates and purges only that captured generation. State +deletion and orchestration cleanup are not atomic, but a `Create` between them +is safe: cleanup never looks up or targets the replacement. A result decoding +failure is reported without attempting cleanup. + +Deleting an absent job is a no-op. A separate `Delete` that executes after +recreation legitimately removes the new generation. If cleanup fails, the +removed entity state stays deleted. + +### Run fencing + +Each `Create` mints a random run token that is stored on the job and carried by +the run it starts. Every orchestration-originated mutation — `CommitCheckpoint`, +`MarkAsCompleted`, `MarkAsFailed`, and the `Run` signal — carries that token, and +the entity requires a matching **nonempty** token. A run left over from a +job that was deleted and recreated, or from a prior in-place recreate, therefore +cannot checkpoint, complete, or fail the new job. The run also stops itself as +soon as it reads a job whose token no longer matches. + +Missing tokens never match, even when both the request and stored token are +empty. Runs do not adopt another generation's token. A delayed `Run` signal is +dropped after deletion or recreation, and an already-emitted delayed Start stops +when it reads the missing or replacement job instead of exporting its data. + +There is one **current state-owning generation**, not a guarantee of zero +overlap. Previous orchestrations and already-scheduled activities can briefly +remain in flight, but cannot mutate the replacement's job state. Recreation +does not purge their histories; `Delete` cleans only the generation it removes. + +## Output layout + +Each exported instance becomes one object named +`.`: + +| Format | Extension | Content type | Content-Encoding | +| --- | --- | --- | --- | +| `ExportFormatJSONL` (default) | `jsonl.gz` | `application/gzip` | not set | +| `ExportFormatJSON` | `json` | `application/json` | not set | + +A JSONL object is gzip-compressed and stored as an opaque gzip file: the name and +content type agree and no content coding is declared, so every reader downloads +exactly the gzip stream the name promises. Declaring the compression as +`Content-Encoding: gzip` instead would let some clients transparently decompress +the download while others would not, leaving a reader unable to tell what it +received. + +JSONL objects carry one `api.HistoryEvent` per line. Every object carries +`instanceId` and `schemaVersion` metadata, plus `executionId` when the task hub +returns one. The name is derived deterministically, so re-exporting an instance +overwrites its object instead of duplicating it. + +When no destination is supplied, a job writes to the client's configured +container under the prefix `-/`. + +## Worker setup + +```go +store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ + ConnectionString: storageConnectionString, + ContainerName: "history-exports", +}) +err = exporthistory.Register(registry, exporthistory.WorkerOptions{ + Source: taskHubClient, + Store: store, +}) +worker, err := durabletaskscheduler.NewWorker(options, registry, logger, + durabletaskclient.WithAutoWorkItemFilters(), + exporthistory.WithExportHistory(), +) +``` + +`Source` supplies the three management reads the export performs: +`ListInstanceIDs`, `FetchOrchestrationMetadata`, and `GetOrchestrationHistory`. +`*client.TaskHubGrpcClient` and the Durable Task Scheduler client satisfy it. + +`Store` is a narrow interface with a single `Write` method. `AzureBlobHistoryStore` is +the production implementation; supply your own to export elsewhere. It is +deliberately separate from `payload.AzureBlobStore`, whose large-payload +contract assigns random object names inside a single container. Its endpoint +validation is at least as strict as that store's: an `AccountURL` carrying +userinfo, a query string, or a fragment is rejected outright, and plaintext HTTP +is confined to loopback endpoints behind `AllowInsecureHTTP`. + +### Versioning + +Every system task is registered unversioned so it stays reachable when an +application enables default versioning. Because a strict worker advertises its +own version for unversioned registrations, `WithExportHistory()` allow-lists +both the system orchestrators and their activities, so the derived work-item +filters keep advertising them unversioned. Without it, a strict-version worker +constructs successfully but the service never dispatches export work to it. + +## Client + +```go +exportClient, err := exporthistory.NewClient(taskHubClient, exporthistory.ClientOptions{ + ContainerName: "history-exports", +}) + +job, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ + Mode: exporthistory.ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + MaxInstancesPerBatch: 200, +}) +description, err := job.Describe(ctx) +page, err := exportClient.ListJobs(ctx, exporthistory.ExportJobQuery{JobIDPrefix: "nightly-"}) +err = job.Delete(ctx) +``` + +### Validation + +`JobCreationOptions.Normalize` applies the same rules as .NET: + +- Batch mode requires `CompletedTimeFrom` and `CompletedTimeTo`, requires + `CompletedTimeTo` to be strictly greater than `CompletedTimeFrom`, and rejects + an upper bound in the future. +- Continuous mode rejects `CompletedTimeTo` and defaults `CompletedTimeFrom` to + now. +- `MaxInstancesPerBatch` must be between 1 and 1000; it defaults to 100. +- `RuntimeStatus` accepts only `COMPLETED`, `FAILED`, and `TERMINATED`; an empty + filter selects all three. +- A missing job ID is generated as a 32-character GUID. + +The `ExportJob` entity re-validates creation options on the worker, whose clock +is independent of the client's. Everything is checked identically except the +"upper bound is not in the future" rule, which the entity relaxes by +`MaxCreationClockSkew` (5 minutes) so a worker running slightly behind does not +reject a window the client accepted. Clients stay strict, and the tolerance never +shifts the window a continuous job starts from. + +### Errors + +| Error | Sentinel | Raised when | +| --- | --- | --- | +| `ValidationError` | `ErrValidation` | Invalid options, destination, or client configuration | +| `NotFoundError` | `ErrJobNotFound` | Reading a job that does not exist | +| `InvalidTransitionError` | `ErrJobInvalidTransition` | An operation the lifecycle does not allow | +| `OperationError` | `ErrJobOperationFailed` | An operation orchestration failed for another reason | + +Errors raised inside the entity carry stable cross-language error types, so a +client reconstructs the typed error across the orchestration boundary and +`errors.Is`/`errors.As` keep working. + +## Service behavior and limitations + +- **Extended sessions are not supported.** +- **Pagination.** DTS signals the end of the stream by omitting the continuation + token. An empty page that still carries a token advances the cursor and the job + keeps listing. If a non-empty final page omits its token, a continuous job + re-scans that page on each idle cycle; deterministic blob names make the + writes idempotent, but scanned and exported counters include the re-scan. +- **List visibility lag.** The service's instance-ID index can lag orchestration + completion. A batch job whose first page is empty legitimately completes with + nothing exported, so schedule a job after the window's instances are listable. +- **Retry semantics.** A per-instance export is attempted up to three times + (retry delays 15s, then 30s) for transient failures such as a storage write + error. Conditions retrying cannot fix, such as a missing or non-terminal + instance, are collected immediately. A page whose instances still fail is + attempted three times in total, waiting 1 minute before the second attempt and + 2 minutes before the third; the third attempt fails the page instead of + waiting again. +- **Delete cleanup is separate** from entity deletion and targets only the + captured generation. Older generation histories are retained after recreation + and must be purged separately if no longer needed. + +## Tests + +Unit and orchestration-replay tests run with no external services. Live tests +skip unless their environment is configured: + +```bash +# Azure Blob write path against Azurite +AZURITE_CONNECTION_STRING="..." go test ./exporthistory/ + +# End-to-end against a live Durable Task Scheduler plus Azurite +DTS_EMULATOR_ENDPOINT="http://127.0.0.1:8080" DTS_TASK_HUB=default \ + AZURITE_CONNECTION_STRING="..." \ + go test ./tests/durabletaskscheduler/ -run TestDTSExportHistory +``` diff --git a/exporthistory/activities.go b/exporthistory/activities.go new file mode 100644 index 00000000..131f467e --- /dev/null +++ b/exporthistory/activities.go @@ -0,0 +1,274 @@ +package exporthistory + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/task" +) + +// HistorySource supplies the management reads an export job needs. It is +// implemented by [github.com/microsoft/durabletask-go/client.TaskHubGrpcClient] +// and by the Durable Task Scheduler client. +type HistorySource interface { + // ListInstanceIDs returns one page of instance IDs matching the query. + ListInstanceIDs(ctx context.Context, query api.InstanceIDQuery) (*api.InstanceIDQueryResult, error) + // FetchOrchestrationMetadata returns the instance's current metadata. + FetchOrchestrationMetadata( + ctx context.Context, + id api.InstanceID, + opts ...api.FetchOrchestrationMetadataOptions, + ) (*api.OrchestrationMetadata, error) + // GetOrchestrationHistory returns the instance's durable history. + GetOrchestrationHistory( + ctx context.Context, + id api.InstanceID, + query api.HistoryQuery, + ) (*api.OrchestrationHistory, error) +} + +// exportRuntime carries the worker-side dependencies shared by the export +// activities. +type exportRuntime struct { + source HistorySource + store Store + historyPage api.HistoryQuery +} + +// listTerminalInstancesActivity returns one page of terminal instance IDs for +// the job's filter, starting from the job's durable checkpoint. +func (r *exportRuntime) listTerminalInstancesActivity(ctx task.ActivityContext) (any, error) { + var input ListTerminalInstancesRequest + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if r.source == nil { + return nil, errors.New("export history worker has no configured history source") + } + statuses := input.RuntimeStatus + if len(statuses) == 0 { + statuses = TerminalStatuses() + } + if err := validateTerminalStatuses(statuses); err != nil { + return nil, err + } + pageSize := input.MaxInstancesPerBatch + if pageSize <= 0 { + pageSize = DefaultMaxInstancesPerBatch + } + var completedTimeTo time.Time + if input.CompletedTimeTo != nil { + completedTimeTo = *input.CompletedTimeTo + } + page, err := r.source.ListInstanceIDs(ctx.Context(), api.InstanceIDQuery{ + RuntimeStatus: statuses, + CompletedTimeFrom: input.CompletedTimeFrom, + CompletedTimeTo: completedTimeTo, + PageSize: pageSize, + ContinuationToken: input.LastInstanceKey, + }) + if err != nil { + return nil, fmt.Errorf("failed to list terminal orchestration instances: %w", err) + } + if page == nil { + return nil, errors.New("failed to list terminal orchestration instances: no result") + } + result := InstancePage{InstanceIDs: make([]string, 0, len(page.InstanceIDs))} + // A task hub reports the end of the stream by omitting the continuation + // token. Leaving the checkpoint nil preserves the last opaque backend cursor; + // a continuous job may re-scan that final page, but deterministic blob names + // make the repeated writes idempotent. + if page.ContinuationToken != "" { + result.NextCheckpoint = &ExportCheckpoint{LastInstanceKey: page.ContinuationToken} + } + for _, id := range page.InstanceIDs { + result.InstanceIDs = append(result.InstanceIDs, string(id)) + } + return result, nil +} + +// permanentExportError marks a per-instance condition that retrying cannot fix, +// such as an instance that does not exist or has not reached a terminal state. +// Transient failures are returned as activity errors instead, so the activity's +// retry policy can recover before the instance is recorded as failed. +type permanentExportError struct{ message string } + +func (e *permanentExportError) Error() string { return e.message } + +func permanentExportFailure(format string, args ...any) error { + return &permanentExportError{message: fmt.Sprintf(format, args...)} +} + +// exportInstanceHistoryActivity exports one instance's history to the +// destination. +// +// A condition that retrying cannot fix is returned as an unsuccessful +// [ExportResult] so the orchestration can collect every failing instance in a +// page. A transient failure is returned as an activity error so the activity's +// retry policy applies first; the orchestration collects it as a failure only +// after every attempt is exhausted. A malformed request always fails the +// activity. +func (r *exportRuntime) exportInstanceHistoryActivity(ctx task.ActivityContext) (any, error) { + var input ExportRequest + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if input.InstanceID == "" { + return nil, &ValidationError{Message: "export request instance ID is required"} + } + if err := input.Destination.Validate(); err != nil { + return nil, err + } + if !input.Format.Kind.IsValid() { + return nil, &ValidationError{ + Message: fmt.Sprintf("invalid export format kind %d", int(input.Format.Kind)), + } + } + if r.source == nil { + return nil, errors.New("export history worker has no configured history source") + } + if r.store == nil { + return nil, errors.New("export history worker has no configured store") + } + + result, err := r.exportInstance(ctx.Context(), input) + if err != nil { + var permanent *permanentExportError + if errors.As(err, &permanent) { + return ExportResult{InstanceID: input.InstanceID, Success: false, Error: permanent.Error()}, nil + } + return nil, err + } + return result, nil +} + +func (r *exportRuntime) exportInstance(ctx context.Context, input ExportRequest) (ExportResult, error) { + instanceID := api.InstanceID(input.InstanceID) + metadata, err := r.source.FetchOrchestrationMetadata(ctx, instanceID) + if err != nil { + if errors.Is(err, api.ErrInstanceNotFound) { + return ExportResult{}, permanentExportFailure("instance %s not found", input.InstanceID) + } + return ExportResult{}, fmt.Errorf("failed to read instance %s metadata: %w", input.InstanceID, err) + } + if metadata == nil { + return ExportResult{}, permanentExportFailure("instance %s not found", input.InstanceID) + } + if !isTerminalStatus(metadata.RuntimeStatus) { + return ExportResult{}, permanentExportFailure( + "instance %s is not in a completed state", input.InstanceID) + } + if metadata.ExecutionID == "" { + return ExportResult{}, fmt.Errorf("instance %s metadata is missing an execution ID", input.InstanceID) + } + + query := r.historyPage + query.ExecutionID = metadata.ExecutionID + history, err := r.source.GetOrchestrationHistory(ctx, instanceID, query) + if err != nil { + return ExportResult{}, fmt.Errorf("failed to read instance %s history: %w", input.InstanceID, err) + } + if history == nil { + return ExportResult{}, fmt.Errorf("instance %s returned no history", input.InstanceID) + } + if history.ExecutionID != metadata.ExecutionID { + return ExportResult{}, fmt.Errorf("instance %s history execution ID %q does not match metadata execution %q", + input.InstanceID, history.ExecutionID, metadata.ExecutionID) + } + + content, contentType, err := serializeHistory(history.Events, input.Format) + if err != nil { + return ExportResult{}, err + } + + completedAt := completionTimestamp(metadata) + name := blobObjectName(completedAt, input.InstanceID, input.Format) + path := input.Destination.BlobPath(name) + object := ExportObject{ + Container: input.Destination.Container, + Name: path, + Content: content, + ContentType: contentType, + Metadata: map[string]string{ + "instanceId": input.InstanceID, + "executionId": history.ExecutionID, + "schemaVersion": input.Format.SchemaVersion, + }, + } + if err := r.store.Write(ctx, object); err != nil { + return ExportResult{}, err + } + return ExportResult{ + InstanceID: input.InstanceID, + Success: true, + BlobPath: path, + EventCount: len(history.Events), + }, nil +} + +// completionTimestamp picks the instant that identifies a terminal instance. +// The service reports a dedicated completion time for most instances; fall back +// to the last update so the object name stays stable and collision-resistant. +func completionTimestamp(metadata *api.OrchestrationMetadata) time.Time { + if !metadata.CompletedAt.IsZero() { + return metadata.CompletedAt + } + return metadata.LastUpdatedAt +} + +// blobObjectName derives a deterministic, collision-resistant object name from +// the instance's completion time and ID, so re-exporting the same instance +// overwrites its object instead of duplicating it. +func blobObjectName(completedAt time.Time, instanceID string, format ExportFormat) string { + digest := sha256.Sum256([]byte(completedAt.UTC().Format(time.RFC3339Nano) + "|" + instanceID)) + return hex.EncodeToString(digest[:]) + "." + format.FileExtension() +} + +// serializeHistory renders events in the requested format and returns the object +// body with the content type to store it under. +// +// A JSONL object is gzip-compressed and stored as an opaque gzip file: its name +// ends in .jsonl.gz and its content type is application/gzip, with no +// Content-Encoding. Declaring the compression as a content coding instead would +// make some clients transparently decompress the download while the object name +// still promises gzip bytes, so readers could not tell what they received. +func serializeHistory( + events []*api.HistoryEvent, + format ExportFormat, +) (content []byte, contentType string, err error) { + if format.Kind == ExportFormatJSON { + if events == nil { + events = []*api.HistoryEvent{} + } + payload, err := json.Marshal(events) + if err != nil { + return nil, "", fmt.Errorf("failed to serialize orchestration history: %w", err) + } + return payload, format.ContentType(), nil + } + + var builder strings.Builder + for i, event := range events { + if event == nil { + continue + } + line, err := json.Marshal(event) + if err != nil { + return nil, "", fmt.Errorf("failed to serialize orchestration history event %d: %w", i, err) + } + builder.Write(line) + builder.WriteByte('\n') + } + compressed, err := gzipContent([]byte(builder.String())) + if err != nil { + return nil, "", err + } + return compressed, format.ContentType(), nil +} diff --git a/exporthistory/activities_test.go b/exporthistory/activities_test.go new file mode 100644 index 00000000..c85141d4 --- /dev/null +++ b/exporthistory/activities_test.go @@ -0,0 +1,883 @@ +package exporthistory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "reflect" + "strings" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// fakeSource serves scripted instance pages, metadata, and histories to the +// export activities. +type fakeSource struct { + mu sync.Mutex + + pages []api.InstanceIDQueryResult + pageIndex int + listQuery []api.InstanceIDQuery + listErr error + + metadata map[string]*api.OrchestrationMetadata + metadataErr map[string]error + + history map[string]*api.OrchestrationHistory + historyErr map[string]error + historyQuery api.HistoryQuery +} + +func newFakeSource() *fakeSource { + return &fakeSource{ + metadata: map[string]*api.OrchestrationMetadata{}, + metadataErr: map[string]error{}, + history: map[string]*api.OrchestrationHistory{}, + historyErr: map[string]error{}, + } +} + +func (s *fakeSource) ListInstanceIDs( + _ context.Context, + query api.InstanceIDQuery, +) (*api.InstanceIDQueryResult, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.listQuery = append(s.listQuery, query) + if s.listErr != nil { + return nil, s.listErr + } + if s.pageIndex >= len(s.pages) { + return &api.InstanceIDQueryResult{}, nil + } + page := s.pages[s.pageIndex] + s.pageIndex++ + return &page, nil +} + +func (s *fakeSource) FetchOrchestrationMetadata( + _ context.Context, + id api.InstanceID, + _ ...api.FetchOrchestrationMetadataOptions, +) (*api.OrchestrationMetadata, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err, ok := s.metadataErr[string(id)]; ok { + return nil, err + } + metadata, ok := s.metadata[string(id)] + if !ok { + return nil, api.ErrInstanceNotFound + } + return metadata, nil +} + +func (s *fakeSource) GetOrchestrationHistory( + _ context.Context, + id api.InstanceID, + query api.HistoryQuery, +) (*api.OrchestrationHistory, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.historyQuery = query + if err, ok := s.historyErr[string(id)]; ok { + return nil, err + } + history, ok := s.history[string(id)] + if !ok { + return nil, api.ErrInstanceNotFound + } + return history, nil +} + +func (s *fakeSource) addInstance(instanceID string, status api.OrchestrationStatus, events int) { + s.mu.Lock() + defer s.mu.Unlock() + completedAt := time.Date(2024, time.April, 1, 12, 0, 0, 0, time.UTC) + s.metadata[instanceID] = &api.OrchestrationMetadata{ + InstanceID: api.InstanceID(instanceID), + ExecutionID: instanceID + "-execution", + Name: "TestOrchestration", + RuntimeStatus: status, + CompletedAt: completedAt, + LastUpdatedAt: completedAt, + } + history := &api.OrchestrationHistory{ + InstanceID: api.InstanceID(instanceID), + ExecutionID: instanceID + "-execution", + } + for i := 0; i < events; i++ { + history.Events = append(history.Events, &api.HistoryEvent{ + Type: api.HistoryEventOrchestratorStarted, + EventID: int32(i), + Timestamp: completedAt, + }) + } + s.history[instanceID] = history +} + +var _ HistorySource = (*fakeSource)(nil) + +func newTestRuntime(source HistorySource, store Store) *exportRuntime { + query, err := api.NormalizeHistoryQuery(api.HistoryQuery{}) + if err != nil { + panic(err) + } + return &exportRuntime{source: source, store: store, historyPage: query} +} + +// activityContext is a minimal [task.ActivityContext] that carries a +// pre-serialized input, so activity bodies can be driven directly. +type activityContext struct { + ctx context.Context + input []byte + // decoded stands in for a worker configured with a custom + // api.DataConverter, whose decoding does not go through encoding/json and so + // can produce values the JSON decoder would have rejected. + decoded any +} + +func newActivityContext(t *testing.T, input any) *activityContext { + t.Helper() + payload, err := json.Marshal(input) + require.NoError(t, err) + return &activityContext{ctx: context.Background(), input: payload} +} + +func newDecodedActivityContext(input any) *activityContext { + return &activityContext{ctx: context.Background(), decoded: input} +} + +func (c *activityContext) GetInput(target any) error { + if c.decoded != nil { + reflect.ValueOf(target).Elem().Set(reflect.ValueOf(c.decoded)) + return nil + } + return json.Unmarshal(c.input, target) +} + +func (c *activityContext) Context() context.Context { return c.ctx } + +var _ task.ActivityContext = (*activityContext)(nil) + +func TestListTerminalInstancesActivity(t *testing.T) { + t.Run("passes the filter and checkpoint to the management query", func(t *testing.T) { + source := newFakeSource() + source.pages = []api.InstanceIDQueryResult{{ + InstanceIDs: []api.InstanceID{"a", "b"}, + ContinuationToken: "next-cursor", + }} + runtime := newTestRuntime(source, newMemoryStore()) + + from := time.Date(2024, time.March, 1, 0, 0, 0, 0, time.UTC) + to := from.Add(time.Hour) + result, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{ + CompletedTimeFrom: from, + CompletedTimeTo: &to, + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + LastInstanceKey: "cursor", + MaxInstancesPerBatch: 7, + })) + require.NoError(t, err) + + page, ok := result.(InstancePage) + require.True(t, ok) + assert.Equal(t, []string{"a", "b"}, page.InstanceIDs) + require.NotNil(t, page.NextCheckpoint) + assert.Equal(t, "next-cursor", page.NextCheckpoint.LastInstanceKey) + + require.Len(t, source.listQuery, 1) + query := source.listQuery[0] + assert.Equal(t, []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, query.RuntimeStatus) + assert.Equal(t, from, query.CompletedTimeFrom) + assert.Equal(t, to, query.CompletedTimeTo) + assert.Equal(t, 7, query.PageSize) + assert.Equal(t, "cursor", query.ContinuationToken) + }) + + t.Run("omits the backend cursor at the end of the stream", func(t *testing.T) { + source := newFakeSource() + // A task hub reports the last page by omitting the continuation token. + source.pages = []api.InstanceIDQueryResult{{InstanceIDs: []api.InstanceID{"a"}}} + runtime := newTestRuntime(source, newMemoryStore()) + result, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{})) + require.NoError(t, err) + page := result.(InstancePage) + assert.Equal(t, []string{"a"}, page.InstanceIDs) + assert.Nil(t, page.NextCheckpoint) + }) + + t.Run("defaults the status filter and page size", func(t *testing.T) { + source := newFakeSource() + source.pages = []api.InstanceIDQueryResult{{}} + runtime := newTestRuntime(source, newMemoryStore()) + _, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{})) + require.NoError(t, err) + require.Len(t, source.listQuery, 1) + assert.Equal(t, TerminalStatuses(), source.listQuery[0].RuntimeStatus) + assert.Equal(t, DefaultMaxInstancesPerBatch, source.listQuery[0].PageSize) + }) + + t.Run("rejects non-terminal statuses", func(t *testing.T) { + runtime := newTestRuntime(newFakeSource(), newMemoryStore()) + _, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{ + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING}, + })) + require.ErrorIs(t, err, ErrValidation) + }) + + t.Run("surfaces management failures", func(t *testing.T) { + failure := errors.New("list failed") + source := newFakeSource() + source.listErr = failure + runtime := newTestRuntime(source, newMemoryStore()) + _, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{})) + require.ErrorIs(t, err, failure) + }) + + t.Run("requires a configured source", func(t *testing.T) { + runtime := &exportRuntime{store: newMemoryStore()} + _, err := runtime.listTerminalInstancesActivity(newActivityContext(t, ListTerminalInstancesRequest{})) + require.Error(t, err) + assert.Contains(t, err.Error(), "no configured history source") + }) +} + +func TestExportInstanceHistoryActivity(t *testing.T) { + destination := ExportDestination{Container: "container", Prefix: "batch-job/"} + + t.Run("writes a gzip-compressed JSONL object", func(t *testing.T) { + source := newFakeSource() + source.addInstance("instance-1", api.RUNTIME_STATUS_COMPLETED, 3) + store := newMemoryStore() + runtime := newTestRuntime(source, store) + + result, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: "instance-1", + Destination: destination, + Format: DefaultExportFormat(), + })) + require.NoError(t, err) + exported, ok := result.(ExportResult) + require.True(t, ok) + require.True(t, exported.Success, exported.Error) + assert.Equal(t, "instance-1", exported.InstanceID) + assert.Equal(t, 3, exported.EventCount) + assert.True(t, strings.HasPrefix(exported.BlobPath, "batch-job/")) + assert.True(t, strings.HasSuffix(exported.BlobPath, ".jsonl.gz")) + + objects := store.snapshot() + require.Len(t, objects, 1) + object := objects["container/"+exported.BlobPath] + // A gzip-compressed JSONL object is stored as an opaque gzip file, so it + // carries no content coding a reader could transparently apply. + assert.Equal(t, "application/gzip", object.ContentType) + assert.Equal(t, "instance-1", object.Metadata["instanceId"]) + assert.Equal(t, "instance-1-execution", object.Metadata["executionId"]) + assert.Equal(t, DefaultSchemaVersion, object.Metadata["schemaVersion"]) + + decompressed := decompressGzip(t, object.Content) + lines := strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") + require.Len(t, lines, 3) + for _, line := range lines { + var event api.HistoryEvent + require.NoError(t, json.Unmarshal([]byte(line), &event)) + assert.Equal(t, api.HistoryEventOrchestratorStarted, event.Type) + } + }) + + t.Run("writes an uncompressed JSON array", func(t *testing.T) { + source := newFakeSource() + source.addInstance("instance-1", api.RUNTIME_STATUS_FAILED, 2) + store := newMemoryStore() + runtime := newTestRuntime(source, store) + + result, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: "instance-1", + Destination: destination, + Format: ExportFormat{Kind: ExportFormatJSON, SchemaVersion: "1.0"}, + })) + require.NoError(t, err) + exported := result.(ExportResult) + require.True(t, exported.Success, exported.Error) + assert.True(t, strings.HasSuffix(exported.BlobPath, ".json")) + + object := store.snapshot()["container/"+exported.BlobPath] + assert.Equal(t, "application/json", object.ContentType) + var events []api.HistoryEvent + require.NoError(t, json.Unmarshal(object.Content, &events)) + require.Len(t, events, 2) + }) + + t.Run("object names are deterministic and instance-specific", func(t *testing.T) { + source := newFakeSource() + source.addInstance("instance-1", api.RUNTIME_STATUS_COMPLETED, 1) + source.addInstance("instance-2", api.RUNTIME_STATUS_COMPLETED, 1) + store := newMemoryStore() + runtime := newTestRuntime(source, store) + + request := ExportRequest{ + InstanceID: "instance-1", + Destination: destination, + Format: DefaultExportFormat(), + } + first, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, request)) + require.NoError(t, err) + second, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, request)) + require.NoError(t, err) + assert.Equal(t, first.(ExportResult).BlobPath, second.(ExportResult).BlobPath) + // Re-exporting overwrites rather than duplicating. + assert.Equal(t, 1, store.count()) + assert.Equal(t, 2, store.writeCount()) + + request.InstanceID = "instance-2" + other, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, request)) + require.NoError(t, err) + assert.NotEqual(t, first.(ExportResult).BlobPath, other.(ExportResult).BlobPath) + assert.Equal(t, 2, store.count()) + }) + + t.Run("collects permanent per-instance failures without retrying", func(t *testing.T) { + source := newFakeSource() + source.addInstance("running", api.RUNTIME_STATUS_RUNNING, 1) + runtime := newTestRuntime(source, newMemoryStore()) + + tests := []struct { + instanceID string + message string + }{ + {"missing", "not found"}, + {"running", "not in a completed state"}, + } + for _, test := range tests { + result, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: test.instanceID, + Destination: destination, + Format: DefaultExportFormat(), + })) + require.NoError(t, err, test.instanceID) + exported := result.(ExportResult) + assert.False(t, exported.Success, test.instanceID) + assert.Equal(t, test.instanceID, exported.InstanceID) + assert.Contains(t, exported.Error, test.message) + } + }) + + // Transient failures must fail the activity so its retry policy applies; + // the orchestration collects them only after every attempt is exhausted. + t.Run("fails the activity for transient failures", func(t *testing.T) { + source := newFakeSource() + source.addInstance("history-error", api.RUNTIME_STATUS_COMPLETED, 1) + source.historyErr["history-error"] = errors.New("history unavailable") + source.metadataErr["metadata-error"] = errors.New("metadata unavailable") + source.addInstance("store-error", api.RUNTIME_STATUS_COMPLETED, 1) + store := newMemoryStore() + store.failInstance("store-error", errors.New("upload rejected")) + runtime := newTestRuntime(source, store) + + tests := []struct { + instanceID string + message string + }{ + {"history-error", "failed to read instance history-error history"}, + {"metadata-error", "failed to read instance metadata-error metadata"}, + {"store-error", "upload rejected"}, + } + for _, test := range tests { + _, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: test.instanceID, + Destination: destination, + Format: DefaultExportFormat(), + })) + require.Error(t, err, test.instanceID) + assert.Contains(t, err.Error(), test.message) + } + }) + + t.Run("rejects malformed requests", func(t *testing.T) { + runtime := newTestRuntime(newFakeSource(), newMemoryStore()) + tests := []ExportRequest{ + {Destination: destination, Format: DefaultExportFormat()}, + {InstanceID: "i", Format: DefaultExportFormat()}, + {InstanceID: "i", Destination: ExportDestination{Container: "BAD"}, Format: DefaultExportFormat()}, + } + for i, request := range tests { + _, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, request)) + require.ErrorIs(t, err, ErrValidation, "case %d", i) + } + + // An out-of-range format kind cannot be produced by the JSON converter, + // but a custom api.DataConverter can, so the activity still checks it. + _, err := runtime.exportInstanceHistoryActivity(newDecodedActivityContext(ExportRequest{ + InstanceID: "i", + Destination: destination, + Format: ExportFormat{Kind: ExportFormatKind(9)}, + })) + require.ErrorIs(t, err, ErrValidation) + }) + + t.Run("requires configured dependencies", func(t *testing.T) { + request := newActivityContext(t, ExportRequest{ + InstanceID: "i", + Destination: destination, + Format: DefaultExportFormat(), + }) + _, err := (&exportRuntime{store: newMemoryStore()}).exportInstanceHistoryActivity(request) + require.Error(t, err) + assert.Contains(t, err.Error(), "no configured history source") + + _, err = (&exportRuntime{source: newFakeSource()}).exportInstanceHistoryActivity(request) + require.Error(t, err) + assert.Contains(t, err.Error(), "no configured store") + }) + + t.Run("falls back to LastUpdatedAt when CompletedAt is unset", func(t *testing.T) { + lastUpdated := time.Date(2024, time.February, 2, 2, 2, 2, 0, time.UTC) + metadata := &api.OrchestrationMetadata{LastUpdatedAt: lastUpdated} + assert.Equal(t, lastUpdated, completionTimestamp(metadata)) + metadata.CompletedAt = lastUpdated.Add(time.Hour) + assert.Equal(t, lastUpdated.Add(time.Hour), completionTimestamp(metadata)) + }) +} + +func TestSerializeHistory(t *testing.T) { + t.Run("empty JSONL history produces an empty gzip stream", func(t *testing.T) { + content, contentType, err := serializeHistory(nil, DefaultExportFormat()) + require.NoError(t, err) + assert.Equal(t, "application/gzip", contentType) + decompressed := decompressGzip(t, content) + assert.Empty(t, decompressed) + }) + + t.Run("empty JSON history produces an empty array", func(t *testing.T) { + content, contentType, err := serializeHistory(nil, ExportFormat{Kind: ExportFormatJSON}) + require.NoError(t, err) + assert.Equal(t, "application/json", contentType) + assert.JSONEq(t, `[]`, string(content)) + }) + + t.Run("nil events are skipped rather than serialized as null", func(t *testing.T) { + events := []*api.HistoryEvent{ + {Type: api.HistoryEventExecutionStarted}, + nil, + {Type: api.HistoryEventExecutionCompleted}, + } + content, _, err := serializeHistory(events, DefaultExportFormat()) + require.NoError(t, err) + decompressed := decompressGzip(t, content) + lines := strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") + require.Len(t, lines, 2) + assert.NotContains(t, string(decompressed), "null") + }) +} + +func TestBlobObjectName(t *testing.T) { + completedAt := time.Date(2024, time.January, 1, 0, 0, 0, 0, time.UTC) + name := blobObjectName(completedAt, "instance", DefaultExportFormat()) + assert.True(t, strings.HasSuffix(name, ".jsonl.gz")) + assert.Len(t, strings.TrimSuffix(name, ".jsonl.gz"), 64) + assert.Equal(t, name, blobObjectName(completedAt.In(time.FixedZone("x", 3600)), "instance", DefaultExportFormat())) + assert.NotEqual(t, name, blobObjectName(completedAt.Add(time.Nanosecond), "instance", DefaultExportFormat())) + assert.NotEqual(t, name, blobObjectName(completedAt, "other", DefaultExportFormat())) + + jsonName := blobObjectName(completedAt, "instance", ExportFormat{Kind: ExportFormatJSON}) + assert.True(t, strings.HasSuffix(jsonName, ".json")) + assert.Equal(t, + strings.TrimSuffix(name, ".jsonl.gz"), + strings.TrimSuffix(jsonName, ".json"), + "the digest must not depend on the format") +} + +// TestBatchRetryBackoff pins the delays the whole-page retry ladder actually +// schedules. Only attempts 1 and 2 create a timer, because the third and final +// attempt fails the page instead of waiting again. +func TestBatchRetryBackoff(t *testing.T) { + assert.Equal(t, minBatchRetryBackoff, batchRetryBackoff(1)) + assert.Equal(t, 2*time.Minute, batchRetryBackoff(2)) + // The reachable schedule ends here: maxBatchRetryAttempts is 3, and the + // final attempt returns the collected failures without a backoff. + assert.Equal(t, 3, maxBatchRetryAttempts) +} + +// TestProcessBatchWithRetryHoldsTheCursorWhenNoAttemptRuns covers the defensive +// exit from the retry loop: a batch that was never attempted must be reported as +// failed so the caller keeps the cursor on the page instead of committing a +// checkpoint that skips it. +func TestProcessBatchWithRetryHoldsTheCursorWhenNoAttemptRuns(t *testing.T) { + for _, attempts := range []int{0, -1} { + result, err := processBatchWithRetry( + &task.OrchestrationContext{}, "job-1", []string{"i1", "i2"}, ExportJobConfiguration{}, attempts) + require.NoError(t, err) + assert.Zero(t, result.exportedCount) + require.Len(t, result.failures, 1) + assert.Contains(t, result.failures[0].Reason, "never attempted") + } +} + +func TestExportActivityRetryPolicy(t *testing.T) { + policy := exportActivityRetryPolicy() + require.NoError(t, policy.Validate()) + assert.Equal(t, 3, policy.MaxAttempts) + assert.Equal(t, 15*time.Second, policy.InitialRetryInterval) + assert.Equal(t, 2.0, policy.BackoffCoefficient) + assert.Equal(t, time.Minute, policy.MaxRetryInterval) +} + +func TestRegisterValidation(t *testing.T) { + source := newFakeSource() + store := newMemoryStore() + + require.ErrorIs(t, Register(nil, WorkerOptions{Source: source, Store: store}), ErrValidation) + require.ErrorIs(t, Register(task.NewTaskRegistry(), WorkerOptions{Store: store}), ErrValidation) + require.ErrorIs(t, Register(task.NewTaskRegistry(), WorkerOptions{Source: source}), ErrValidation) + + err := Register(task.NewTaskRegistry(), WorkerOptions{ + Source: source, + Store: store, + HistoryQuery: api.HistoryQuery{MaxEvents: -1}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid export history query") +} + +// TestRegisterAddsEverySystemTask keeps the registry snapshot, and therefore the +// worker's derived work-item filters, complete. +func TestRegisterAddsEverySystemTask(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, Register(registry, WorkerOptions{Source: newFakeSource(), Store: newMemoryStore()})) + + snapshot := registry.Snapshot() + orchestrators := map[string]string{} + for _, registration := range snapshot.Orchestrators { + orchestrators[registration.Name] = registration.Version + } + assert.Contains(t, orchestrators, ExportJobOrchestratorName) + assert.Contains(t, orchestrators, ExecuteExportJobOperationOrchestratorName) + // System tasks stay unversioned so default versioning cannot hide them. + assert.Equal(t, task.UnversionedTaskVersion, orchestrators[ExportJobOrchestratorName]) + assert.Equal(t, task.UnversionedTaskVersion, orchestrators[ExecuteExportJobOperationOrchestratorName]) + + activities := map[string]string{} + for _, registration := range snapshot.Activities { + activities[registration.Name] = registration.Version + } + assert.Contains(t, activities, ListTerminalInstancesActivityName) + assert.Contains(t, activities, ExportInstanceHistoryActivityName) + assert.Equal(t, task.UnversionedTaskVersion, activities[ListTerminalInstancesActivityName]) + assert.Equal(t, task.UnversionedTaskVersion, activities[ExportInstanceHistoryActivityName]) + + assert.Contains(t, snapshot.Entities, strings.ToLower(ExportJobEntityName)) +} + +func TestRegisterRejectsDuplicateRegistration(t *testing.T) { + registry := task.NewTaskRegistry() + options := WorkerOptions{Source: newFakeSource(), Store: newMemoryStore()} + require.NoError(t, Register(registry, options)) + require.Error(t, Register(registry, options)) +} + +func TestSystemNamesMatchDotNet(t *testing.T) { + // These names are part of the cross-SDK contract: a .NET client creating a + // job schedules these exact orchestrator and entity names. + assert.Equal(t, "ExportJob", ExportJobEntityName) + assert.Equal(t, "ExportJobOrchestrator", ExportJobOrchestratorName) + assert.Equal(t, "ExecuteExportJobOperationOrchestrator", ExecuteExportJobOperationOrchestratorName) + assert.Equal(t, "ListTerminalInstancesActivity", ListTerminalInstancesActivityName) + assert.Equal(t, "ExportInstanceHistoryActivity", ExportInstanceHistoryActivityName) + assert.Equal(t, "ExportJob-", OrchestratorInstanceIDPrefix) + + assert.Equal(t, "Create", createOperation) + assert.Equal(t, "Get", getOperation) + assert.Equal(t, "Run", runOperation) + assert.Equal(t, "CommitCheckpoint", commitCheckpointOperation) + assert.Equal(t, "MarkAsCompleted", markAsCompletedOperation) + assert.Equal(t, "MarkAsFailed", markAsFailedOperation) + assert.Equal(t, "Delete", deleteOperation) +} + +func TestExportRuntimeUsesTheConfiguredHistoryQuery(t *testing.T) { + source := newFakeSource() + source.addInstance("instance-1", api.RUNTIME_STATUS_COMPLETED, 1) + source.addInstance("instance-2", api.RUNTIME_STATUS_COMPLETED, 1) + query := api.HistoryQuery{ExecutionID: "shared-default", MaxEvents: 11, MaxBytes: 2048} + runtime := &exportRuntime{ + source: source, + store: newMemoryStore(), + historyPage: query, + } + for _, instanceID := range []string{"instance-1", "instance-2"} { + _, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: instanceID, + Destination: ExportDestination{Container: "container"}, + Format: DefaultExportFormat(), + })) + require.NoError(t, err) + assert.Equal(t, instanceID+"-execution", source.historyQuery.ExecutionID) + assert.Equal(t, query.MaxEvents, source.historyQuery.MaxEvents) + assert.Equal(t, query.MaxBytes, source.historyQuery.MaxBytes) + assert.Equal(t, query, runtime.historyPage, "per-instance pinning must not mutate the shared query") + } +} + +func TestExportInstanceHistoryRejectsExecutionIdentity(t *testing.T) { + for _, test := range []struct { + name string + metadataID string + historyID string + readHistory bool + }{ + {"different execution", "A", "B", true}, + {"missing history execution", "A", "", true}, + {"missing metadata execution", "", "A", false}, + } { + t.Run(test.name, func(t *testing.T) { + source := newFakeSource() + source.addInstance("instance-1", api.RUNTIME_STATUS_COMPLETED, 1) + source.metadata["instance-1"].ExecutionID = test.metadataID + source.history["instance-1"].ExecutionID = test.historyID + // Identity validation must run before attempting to serialize content. + details := &api.FailureDetails{} + details.InnerFailure = details + source.history["instance-1"].Events = []*api.HistoryEvent{{ + TaskFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}, + }} + store := newMemoryStore() + runtime := newTestRuntime(source, store) + _, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + InstanceID: "instance-1", + Destination: ExportDestination{Container: "container"}, + Format: DefaultExportFormat(), + })) + assert.ErrorContains(t, err, "execution") + assert.Zero(t, store.writeCount()) + assert.Empty(t, store.snapshot()) + if test.readHistory { + assert.Equal(t, test.metadataID, source.historyQuery.ExecutionID) + } else { + assert.Empty(t, source.historyQuery, "metadata without an identity must not trigger a history read") + } + }) + } +} + +func TestExportInstanceHistoryPinsConcurrentReads(t *testing.T) { + source := newFakeSource() + for _, id := range []string{"instance-1", "instance-2"} { + source.addInstance(id, api.RUNTIME_STATUS_COMPLETED, 1) + } + store := newMemoryStore() + runtime := newTestRuntime(source, store) + query := runtime.historyPage + results := make(chan error, 2) + for _, id := range []string{"instance-1", "instance-2"} { + go func() { + _, err := runtime.exportInstance(context.Background(), ExportRequest{ + InstanceID: id, Destination: ExportDestination{Container: "container"}, Format: DefaultExportFormat(), + }) + results <- err + }() + } + for range 2 { + require.NoError(t, <-results) + } + require.Equal(t, query, runtime.historyPage) + require.Equal(t, 2, store.writeCount()) +} + +type exportHistoryIdentityServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + state *protos.OrchestrationState + events []*protos.HistoryEvent + requests chan *protos.StreamInstanceHistoryRequest +} + +func (s *exportHistoryIdentityServer) GetInstance(context.Context, *protos.GetInstanceRequest) (*protos.GetInstanceResponse, error) { + return &protos.GetInstanceResponse{Exists: true, OrchestrationState: s.state}, nil +} + +func (s *exportHistoryIdentityServer) StreamInstanceHistory( + request *protos.StreamInstanceHistoryRequest, + stream protos.TaskHubSidecarService_StreamInstanceHistoryServer, +) error { + s.requests <- request + return stream.Send(&protos.HistoryChunk{Events: s.events}) +} + +func TestExportInstanceHistoryPinsTheClientCollector(t *testing.T) { + for _, test := range []struct { + name string + executionIDs []string + wantError bool + }{ + {"matching execution", []string{"A"}, false}, + {"restarted execution", []string{"B"}, true}, + {"mixed executions", []string{"A", "B"}, true}, + {"missing execution", []string{""}, true}, + {"empty history", nil, true}, + } { + t.Run(test.name, func(t *testing.T) { + serverImpl := &exportHistoryIdentityServer{ + state: &protos.OrchestrationState{ + InstanceId: "instance-1", + ExecutionId: wrapperspb.String("A"), + OrchestrationStatus: api.RUNTIME_STATUS_COMPLETED, + CompletedTimestamp: timestamppb.New(time.Date(2024, time.April, 1, 12, 0, 0, 0, time.UTC)), + }, + requests: make(chan *protos.StreamInstanceHistoryRequest, 1), + } + for _, executionID := range test.executionIDs { + serverImpl.events = append(serverImpl.events, &protos.HistoryEvent{ + EventType: &protos.HistoryEvent_ExecutionStarted{ExecutionStarted: &protos.ExecutionStartedEvent{ + OrchestrationInstance: &protos.OrchestrationInstance{ + InstanceId: "instance-1", ExecutionId: wrapperspb.String(executionID), + }, + }}, + }) + } + server := grpc.NewServer() + protos.RegisterTaskHubSidecarServiceServer(server, serverImpl) + listener := bufconn.Listen(1024 * 1024) + go func() { _ = server.Serve(listener) }() + t.Cleanup(func() { + server.Stop() + require.NoError(t, listener.Close()) + }) + connection, err := grpc.NewClient( + "passthrough:///export-history-identity", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { + return listener.DialContext(ctx) + }), + ) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, connection.Close()) }) + + store := newMemoryStore() + runtime := newTestRuntime(durabletaskclient.NewTaskHubGrpcClient(connection, api.DefaultLogger()), store) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + input := newActivityContext(t, ExportRequest{ + InstanceID: "instance-1", Destination: ExportDestination{Container: "container"}, Format: DefaultExportFormat(), + }) + input.ctx = ctx + result, err := runtime.exportInstanceHistoryActivity(input) + if test.wantError { + assert.ErrorContains(t, err, "execution") + assert.Nil(t, result) + assert.Zero(t, store.writeCount()) + } else { + require.NoError(t, err) + require.True(t, result.(ExportResult).Success) + require.Equal(t, 1, store.writeCount()) + for _, object := range store.snapshot() { + require.Equal(t, "A", object.Metadata["executionId"]) + } + } + select { + case request := <-serverImpl.requests: + assert.Equal(t, "A", request.GetExecutionId().GetValue()) + case <-ctx.Done(): + t.Fatal("history was not requested") + } + assert.Empty(t, runtime.historyPage.ExecutionID) + }) + } +} + +func TestExportJobRunRequestJSON(t *testing.T) { + request := ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + ProcessedCycles: 3, + RunToken: "run-a", + ContinuedExecution: true, + } + encoded, err := json.Marshal(request) + require.NoError(t, err) + assert.Contains(t, string(encoded), `"JobEntityId"`) + var decoded ExportJobRunRequest + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, request, decoded) +} + +func TestExportJobOperationRequestJSON(t *testing.T) { + request := ExportJobOperationRequest{ + EntityID: EntityID("job-1"), + OperationName: createOperation, + Input: map[string]any{"Mode": float64(1)}, + } + encoded, err := json.Marshal(request) + require.NoError(t, err) + assert.Contains(t, string(encoded), `"EntityId"`) + assert.Contains(t, string(encoded), `"OperationName"`) + var decoded ExportJobOperationRequest + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, request.EntityID, decoded.EntityID) + assert.Equal(t, request.OperationName, decoded.OperationName) + + // A nil input is omitted rather than serialized as null. + encoded, err = json.Marshal(ExportJobOperationRequest{ + EntityID: EntityID("job-1"), + OperationName: getOperation, + }) + require.NoError(t, err) + assert.NotContains(t, string(encoded), `"Input"`) +} + +func TestInstancePageJSON(t *testing.T) { + page := InstancePage{ + InstanceIDs: []string{"a", "b"}, + NextCheckpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + } + encoded, err := json.Marshal(page) + require.NoError(t, err) + var decoded InstancePage + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, page.InstanceIDs, decoded.InstanceIDs) + require.NotNil(t, decoded.NextCheckpoint) + assert.Equal(t, "cursor", decoded.NextCheckpoint.LastInstanceKey) +} + +func TestExportResultJSON(t *testing.T) { + result := ExportResult{InstanceID: "i", Success: true, BlobPath: "p/o", EventCount: 4} + encoded, err := json.Marshal(result) + require.NoError(t, err) + var decoded ExportResult + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, result, decoded) + + failed, err := json.Marshal(ExportResult{InstanceID: "i", Error: "boom"}) + require.NoError(t, err) + assert.Contains(t, string(failed), `"Error":"boom"`) + assert.NotContains(t, string(failed), `"BlobPath"`) +} + +func TestFakeSourceHelpersProduceDistinctInstances(t *testing.T) { + source := newFakeSource() + for i := 0; i < 3; i++ { + source.addInstance(fmt.Sprintf("instance-%d", i), api.RUNTIME_STATUS_COMPLETED, i+1) + } + for i := 0; i < 3; i++ { + history, err := source.GetOrchestrationHistory( + context.Background(), + api.InstanceID(fmt.Sprintf("instance-%d", i)), + api.HistoryQuery{}, + ) + require.NoError(t, err) + assert.Len(t, history.Events, i+1) + } +} diff --git a/exporthistory/client.go b/exporthistory/client.go new file mode 100644 index 00000000..3fd23080 --- /dev/null +++ b/exporthistory/client.go @@ -0,0 +1,438 @@ +package exporthistory + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" +) + +// ClientOptions configures [Client]. ContainerName and Prefix supply the +// destination for jobs created without an explicit [ExportDestination]. +type ClientOptions struct { + // ContainerName is the default destination container. + ContainerName string + // Prefix is the default destination path prefix. When empty each job gets + // the mode-and-ID prefix "-/", matching .NET. + Prefix string +} + +// taskHubClient is the narrow task hub surface the export client uses. +type taskHubClient interface { + ScheduleNewOrchestration( + ctx context.Context, + orchestrator string, + opts ...api.NewOrchestrationOptions, + ) (api.InstanceID, error) + WaitForOrchestrationCompletion( + ctx context.Context, + id api.InstanceID, + opts ...api.FetchOrchestrationMetadataOptions, + ) (*api.OrchestrationMetadata, error) + TerminateOrchestration(ctx context.Context, id api.InstanceID, opts ...api.TerminateOptions) error + PurgeOrchestrationState(ctx context.Context, id api.InstanceID, opts ...api.PurgeOptions) error + GetEntity(ctx context.Context, entityID api.EntityID, options ...api.GetEntityOptions) (*api.EntityMetadata, error) + QueryEntities(ctx context.Context, query api.EntityQuery) (*api.EntityQueryResults, error) +} + +// Client creates and inspects export jobs in a task hub. +type Client struct { + hub taskHubClient + container string + prefix string +} + +// NewClient returns an export history client backed by client. +func NewClient(client *durabletaskclient.TaskHubGrpcClient, options ClientOptions) (*Client, error) { + if client == nil { + return nil, &ValidationError{Message: "task hub client is required"} + } + return newClient(client, options) +} + +// newClient builds a client over the narrow task hub surface so tests can +// supply their own implementation. +func newClient(hub taskHubClient, options ClientOptions) (*Client, error) { + if hub == nil { + return nil, &ValidationError{Message: "task hub client is required"} + } + container := strings.TrimSpace(options.ContainerName) + if container != "" && !validBlobContainerName(container) { + return nil, &ValidationError{ + Message: fmt.Sprintf("default export container %q is not a valid Azure Blob container name", container), + } + } + if err := validateBlobPrefix(options.Prefix); err != nil { + return nil, err + } + return &Client{ + hub: hub, + container: container, + prefix: options.Prefix, + }, nil +} + +// JobClient returns a handle for jobID without contacting the service. +func (c *Client) JobClient(jobID string) (*JobClient, error) { + if c == nil || c.hub == nil { + return nil, &ValidationError{Message: "export history client is required"} + } + if err := validateJobID(jobID); err != nil { + return nil, err + } + return &JobClient{ + hub: c.hub, + container: c.container, + prefix: c.prefix, + jobID: jobID, + }, nil +} + +// CreateJob normalizes options, resolves the destination, creates the job, and +// returns a handle for it. +func (c *Client) CreateJob(ctx context.Context, options JobCreationOptions) (*JobClient, error) { + if c == nil || c.hub == nil { + return nil, &ValidationError{Message: "export history client is required"} + } + normalized, err := options.Normalize() + if err != nil { + return nil, err + } + job, err := c.JobClient(normalized.JobID) + if err != nil { + return nil, err + } + if err := job.Create(ctx, normalized); err != nil { + return nil, err + } + return job, nil +} + +// GetJob returns the current description of jobID, or a [NotFoundError] when +// the job does not exist. +func (c *Client) GetJob(ctx context.Context, jobID string) (*ExportJobDescription, error) { + job, err := c.JobClient(jobID) + if err != nil { + return nil, err + } + return job.Describe(ctx) +} + +// ListJobs returns one page of export jobs matching query. Status and +// creation-time filters are applied after the service returns the entity page, +// so a page can contain fewer than PageSize items and an empty page can still +// carry a continuation token. +func (c *Client) ListJobs(ctx context.Context, query ExportJobQuery) (*ExportJobQueryResult, error) { + if c == nil || c.hub == nil { + return nil, &ValidationError{Message: "export history client is required"} + } + pageSize := query.PageSize + switch { + case pageSize < 0: + return nil, &ValidationError{Message: "page size cannot be negative"} + case pageSize == 0: + pageSize = DefaultJobQueryPageSize + case pageSize > api.MaxInstanceQueryPageSize: + return nil, &ValidationError{ + Message: fmt.Sprintf("page size cannot exceed %d", api.MaxInstanceQueryPageSize), + } + } + if strings.Contains(query.JobIDPrefix, "@") { + return nil, &ValidationError{Message: "export job ID prefix must not contain '@'"} + } + if query.Status != nil && !query.Status.IsValid() { + return nil, &ValidationError{ + Message: fmt.Sprintf("invalid export job status %d", int(*query.Status)), + } + } + entities, err := c.hub.QueryEntities(ctx, api.EntityQuery{ + InstanceIDStartsWith: strings.ToLower("@"+ExportJobEntityName+"@") + query.JobIDPrefix, + PageSize: pageSize, + ContinuationToken: query.ContinuationToken, + }) + if err != nil { + return nil, err + } + if entities == nil { + return nil, errors.New("export job query returned no result") + } + result := &ExportJobQueryResult{ContinuationToken: entities.ContinuationToken} + for _, entity := range entities.Entities { + description, err := jobDescription(entity) + if err != nil { + return nil, err + } + if description == nil || !query.matches(description) { + continue + } + result.Jobs = append(result.Jobs, description) + } + return result, nil +} + +// JobClient manages one export job. +type JobClient struct { + hub taskHubClient + container string + prefix string + jobID string +} + +// ID returns the job identifier managed by this handle. +func (c *JobClient) ID() string { + if c == nil { + return "" + } + return c.jobID +} + +// Create creates the job, or recreates it in place when it previously failed or +// completed. Creating a job that is already Active fails with an +// [InvalidTransitionError]. +// +// Each successful Create reserves a new generation-specific orchestration ID. +// Previous generations' orchestration histories are not terminated or purged. +func (c *JobClient) Create(ctx context.Context, options JobCreationOptions) error { + if c == nil || c.hub == nil { + return &ValidationError{Message: "export job client is required"} + } + if options.JobID != "" && options.JobID != c.jobID { + return &ValidationError{ + JobID: c.jobID, + Message: fmt.Sprintf("creation job ID %q does not match export job client %q", options.JobID, c.jobID), + } + } + options.JobID = c.jobID + normalized, err := options.Normalize() + if err != nil { + return err + } + destination, err := c.resolveDestination(normalized) + if err != nil { + return err + } + normalized.Destination = destination + return c.operate(ctx, createOperation, normalized, nil) +} + +// resolveDestination fills in the client's configured container and prefix when +// the caller did not supply a destination. +func (c *JobClient) resolveDestination(options JobCreationOptions) (*ExportDestination, error) { + destination := ExportDestination{} + if options.Destination != nil { + destination = *options.Destination + } + if destination.Container == "" { + destination.Container = c.container + } + if destination.Prefix == "" { + destination.Prefix = c.prefix + } + if destination.Prefix == "" { + destination.Prefix = defaultPrefix(options.Mode, c.jobID) + } + if destination.Container == "" { + return nil, &ValidationError{ + JobID: c.jobID, + Message: "export destination container is required because the client has no default container; " + + "set ClientOptions.ContainerName or JobCreationOptions.Destination", + } + } + if err := destination.Validate(); err != nil { + return nil, withJobID(err, c.jobID) + } + return &destination, nil +} + +// Describe returns the job's current description, or a [NotFoundError]. +func (c *JobClient) Describe(ctx context.Context) (*ExportJobDescription, error) { + if c == nil || c.hub == nil { + return nil, &ValidationError{Message: "export job client is required"} + } + entity, err := c.hub.GetEntity(ctx, EntityID(c.jobID)) + if err != nil { + return nil, err + } + if entity == nil { + return nil, &NotFoundError{JobID: c.jobID} + } + description, err := jobDescription(entity) + if err != nil { + return nil, err + } + if description == nil { + return nil, &NotFoundError{JobID: c.jobID} + } + description.JobID = c.jobID + return description, nil +} + +// Delete removes the job's entity state and stops the generation it removed. +// +// The entity delete and the orchestration cleanup are not atomic: the entity is +// deleted first and returns its captured orchestration ID. Cleanup targets only +// that ID, so a concurrent Create cannot have its new generation terminated or +// purged by this Delete. A missing job or orchestration is not an error. +func (c *JobClient) Delete(ctx context.Context) error { + if c == nil || c.hub == nil { + return &ValidationError{Message: "export job client is required"} + } + var instanceID string + if err := c.operate(ctx, deleteOperation, nil, &instanceID); err != nil { + return err + } + return c.terminateAndPurgeOrchestration(ctx, api.InstanceID(instanceID)) +} + +func (c *JobClient) terminateAndPurgeOrchestration(ctx context.Context, instanceID api.InstanceID) error { + if instanceID == "" { + return nil + } + err := c.hub.TerminateOrchestration(ctx, instanceID, api.WithOutput("Export job deleted")) + if err != nil { + if isInstanceMissing(err) { + return nil + } + return fmt.Errorf("failed to terminate export orchestration %s: %w", instanceID, err) + } + if _, err := c.hub.WaitForOrchestrationCompletion(ctx, instanceID); err != nil { + if isInstanceMissing(err) { + return nil + } + return fmt.Errorf("failed to wait for export orchestration %s to terminate: %w", instanceID, err) + } + if err := c.hub.PurgeOrchestrationState(ctx, instanceID); err != nil { + if isInstanceMissing(err) { + return nil + } + return fmt.Errorf("failed to purge export orchestration %s: %w", instanceID, err) + } + return nil +} + +// operate runs one entity operation through the system operation orchestrator +// and translates a failed orchestration back into a typed error. Results are +// decoded through the metadata's data converter into the caller's target. +func (c *JobClient) operate(ctx context.Context, operation string, input, output any) error { + instanceID, err := c.hub.ScheduleNewOrchestration( + ctx, + ExecuteExportJobOperationOrchestratorName, + api.WithInput(ExportJobOperationRequest{ + EntityID: EntityID(c.jobID), + OperationName: operation, + Input: input, + }), + // System operations must not inherit an application default version. + api.WithVersion(""), + ) + if err != nil { + return err + } + metadata, err := c.hub.WaitForOrchestrationCompletion(ctx, instanceID) + if err != nil { + return err + } + if metadata == nil || metadata.RuntimeStatus != api.RUNTIME_STATUS_COMPLETED { + return operationFailure(c.jobID, operation, metadata) + } + if output != nil { + if metadata.SerializedOutput == "" { + return fmt.Errorf("export job %q %s returned no output", c.jobID, operation) + } + if err := metadata.ReadOutput(output); err != nil { + return fmt.Errorf("failed to decode %s result for export job %q: %w", operation, c.jobID, err) + } + } + return nil +} + +func defaultPrefix(mode ExportMode, jobID string) string { + return strings.ToLower(mode.String()) + "-" + jobID + "/" +} + +func isInstanceMissing(err error) bool { + return errors.Is(err, api.ErrInstanceNotFound) +} + +func jobDescription(metadata *api.EntityMetadata) (*ExportJobDescription, error) { + if metadata == nil || metadata.SerializedState == "" { + return nil, nil + } + var state ExportJobState + if err := json.Unmarshal([]byte(metadata.SerializedState), &state); err != nil { + return nil, fmt.Errorf("failed to deserialize export job state: %w", err) + } + return state.description(metadata.InstanceID.Key), nil +} + +// operationFailure reconstructs the typed error the entity raised, so a caller +// can match an invalid transition or a validation failure with errors.Is and +// errors.As across the orchestration boundary. +func operationFailure( + jobID string, + operation string, + metadata *api.OrchestrationMetadata, +) error { + if metadata == nil { + return &OperationError{JobID: jobID, Operation: operation} + } + if details := findFailure(metadata.FailureDetails, invalidTransitionErrorType); details != nil { + return &InvalidTransitionError{ + JobID: stringProperty(details.Properties, "jobId", jobID), + From: ExportJobStatus(intProperty(details.Properties, "from")), + To: ExportJobStatus(intProperty(details.Properties, "to")), + Operation: stringProperty(details.Properties, "operation", operation), + } + } + if details := findFailure(metadata.FailureDetails, notFoundErrorType); details != nil { + return &NotFoundError{JobID: stringProperty(details.Properties, "jobId", jobID)} + } + if details := findFailure(metadata.FailureDetails, validationErrorType); details != nil { + return &ValidationError{ + JobID: stringProperty(details.Properties, "jobId", jobID), + Message: stringProperty(details.Properties, "message", details.ErrorMessage), + } + } + return &OperationError{ + JobID: jobID, + Operation: operation, + RuntimeStatus: metadata.RuntimeStatus, + FailureDetails: metadata.FailureDetails, + } +} + +func findFailure(details *api.FailureDetails, errorType api.ErrorType) *api.FailureDetails { + for current := details; current != nil; current = current.InnerFailure { + if current.ErrorType == errorType || + strings.HasSuffix(string(current.ErrorType), "."+string(errorType)) { + return current + } + } + return nil +} + +func stringProperty(properties map[string]any, name, fallback string) string { + if value, ok := properties[name].(string); ok && value != "" { + return value + } + return fallback +} + +func intProperty(properties map[string]any, name string) int { + switch value := properties[name].(type) { + case int: + return value + case int32: + return int(value) + case int64: + return int(value) + case float64: + return int(value) + default: + return 0 + } +} diff --git a/exporthistory/client_test.go b/exporthistory/client_test.go new file mode 100644 index 00000000..36fa1c0c --- /dev/null +++ b/exporthistory/client_test.go @@ -0,0 +1,788 @@ +package exporthistory + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The export client and activities are written against the task hub client's +// public surface, so the concrete gRPC client must satisfy both contracts. +var ( + _ taskHubClient = (*durabletaskclient.TaskHubGrpcClient)(nil) + _ HistorySource = (*durabletaskclient.TaskHubGrpcClient)(nil) +) + +// fakeBackend records the calls the export client makes and replays scripted +// responses, standing in for a task hub. +type fakeBackend struct { + mu sync.Mutex + + scheduled []scheduledCall + terminated []api.InstanceID + purged []api.InstanceID + waited []api.InstanceID + entityReads []api.EntityID + + scheduleErr error + scheduleResult api.InstanceID + completion *api.OrchestrationMetadata + completionErr error + // completionErrByInstance overrides completionErr for one instance, so a + // test can fail the orchestration cleanup without failing the entity + // operation that runs first. + completionErrByInstance map[api.InstanceID]error + terminateErr error + purgeErr error + + entity *api.EntityMetadata + entityErr error + queryPages *api.EntityQueryResults + queryErr error + lastQuery api.EntityQuery +} + +type scheduledCall struct { + name string + request ExportJobOperationRequest + version string +} + +func newFakeBackend() *fakeBackend { + return &fakeBackend{ + scheduleResult: "test-instance", + completion: &api.OrchestrationMetadata{RuntimeStatus: api.RUNTIME_STATUS_COMPLETED}, + } +} + +func (f *fakeBackend) ScheduleNewOrchestration( + _ context.Context, + orchestrator string, + opts ...api.NewOrchestrationOptions, +) (api.InstanceID, error) { + f.mu.Lock() + defer f.mu.Unlock() + if f.scheduleErr != nil { + return "", f.scheduleErr + } + // Replay the functional options through the real request type so the test + // observes exactly the input and version the client sent. + request := &protos.CreateInstanceRequest{} + for _, configure := range opts { + if err := configure(request, api.NormalizeDataConverter(nil)); err != nil { + return "", err + } + } + var decoded ExportJobOperationRequest + if input := request.GetInput().GetValue(); input != "" { + if err := json.Unmarshal([]byte(input), &decoded); err != nil { + return "", err + } + } + f.scheduled = append(f.scheduled, scheduledCall{ + name: orchestrator, + request: decoded, + version: request.GetVersion().GetValue(), + }) + return f.scheduleResult, nil +} + +func (f *fakeBackend) WaitForOrchestrationCompletion( + _ context.Context, + id api.InstanceID, + _ ...api.FetchOrchestrationMetadataOptions, +) (*api.OrchestrationMetadata, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.waited = append(f.waited, id) + if err, ok := f.completionErrByInstance[id]; ok { + return nil, err + } + if f.completionErr != nil { + return nil, f.completionErr + } + return f.completion, nil +} + +func (f *fakeBackend) TerminateOrchestration( + _ context.Context, + id api.InstanceID, + _ ...api.TerminateOptions, +) error { + f.mu.Lock() + defer f.mu.Unlock() + f.terminated = append(f.terminated, id) + return f.terminateErr +} + +func (f *fakeBackend) PurgeOrchestrationState( + _ context.Context, + id api.InstanceID, + _ ...api.PurgeOptions, +) error { + f.mu.Lock() + defer f.mu.Unlock() + f.purged = append(f.purged, id) + return f.purgeErr +} + +func (f *fakeBackend) GetEntity( + _ context.Context, + id api.EntityID, + _ ...api.GetEntityOptions, +) (*api.EntityMetadata, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.entityReads = append(f.entityReads, id) + if f.entityErr != nil { + return nil, f.entityErr + } + return f.entity, nil +} + +func (f *fakeBackend) QueryEntities( + _ context.Context, + query api.EntityQuery, +) (*api.EntityQueryResults, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.lastQuery = query + if f.queryErr != nil { + return nil, f.queryErr + } + return f.queryPages, nil +} + +func entityMetadata(t *testing.T, jobID string, state ExportJobState) *api.EntityMetadata { + t.Helper() + payload, err := json.Marshal(state) + require.NoError(t, err) + return &api.EntityMetadata{ + InstanceID: EntityID(jobID), + StateIncluded: true, + HasState: true, + SerializedState: string(payload), + } +} + +func newTestClient(t *testing.T, hub taskHubClient, options ClientOptions) *Client { + t.Helper() + client, err := newClient(hub, options) + require.NoError(t, err) + return client +} + +func TestNewClientValidation(t *testing.T) { + _, err := NewClient(nil, ClientOptions{}) + require.ErrorIs(t, err, ErrValidation) + + _, err = newClient(nil, ClientOptions{}) + require.ErrorIs(t, err, ErrValidation) + + _, err = newClient(newFakeBackend(), ClientOptions{ContainerName: "Bad_Container"}) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "not a valid Azure Blob container name") + + _, err = newClient(newFakeBackend(), ClientOptions{ContainerName: "container", Prefix: "/bad"}) + require.ErrorIs(t, err, ErrValidation) + + client, err := newClient(newFakeBackend(), ClientOptions{}) + require.NoError(t, err) + assert.NotNil(t, client) +} + +func TestJobClientValidation(t *testing.T) { + client := newTestClient(t, newFakeBackend(), ClientOptions{ContainerName: "container"}) + for _, jobID := range []string{"", " ", "bad@id"} { + _, err := client.JobClient(jobID) + require.ErrorIs(t, err, ErrValidation, "job ID %q", jobID) + } + job, err := client.JobClient("job-1") + require.NoError(t, err) + assert.Equal(t, "job-1", job.ID()) + + var nilClient *Client + _, err = nilClient.JobClient("job") + require.ErrorIs(t, err, ErrValidation) + + var nilJob *JobClient + assert.Empty(t, nilJob.ID()) + require.ErrorIs(t, nilJob.Create(context.Background(), JobCreationOptions{}), ErrValidation) + _, describeErr := nilJob.Describe(context.Background()) + require.ErrorIs(t, describeErr, ErrValidation) + require.ErrorIs(t, nilJob.Delete(context.Background()), ErrValidation) +} + +func TestCreateJobSchedulesTheOperationOrchestrator(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + + job, err := client.CreateJob(context.Background(), JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.NoError(t, err) + require.NotNil(t, job) + assert.NotEmpty(t, job.ID()) + + require.Len(t, hub.scheduled, 1) + call := hub.scheduled[0] + assert.Equal(t, ExecuteExportJobOperationOrchestratorName, call.name) + assert.Equal(t, createOperation, call.request.OperationName) + assert.Equal(t, EntityID(job.ID()), call.request.EntityID) + // System operations must not inherit an application default version. + assert.Equal(t, "", call.version) + require.Equal(t, []api.InstanceID{"test-instance"}, hub.waited) +} + +func TestCreateJobResolvesTheDestination(t *testing.T) { + tests := []struct { + name string + clientOptions ClientOptions + destination *ExportDestination + mode ExportMode + expectedPrefix string + expectedName string + }{ + { + name: "default prefix derives from mode and job ID", + clientOptions: ClientOptions{ContainerName: "container"}, + mode: ExportModeBatch, + expectedPrefix: "batch-job-1/", + expectedName: "container", + }, + { + name: "continuous jobs get their own prefix", + clientOptions: ClientOptions{ContainerName: "container"}, + mode: ExportModeContinuous, + expectedPrefix: "continuous-job-1/", + expectedName: "container", + }, + { + name: "client prefix wins over the derived default", + clientOptions: ClientOptions{ContainerName: "container", Prefix: "client-prefix/"}, + mode: ExportModeBatch, + expectedPrefix: "client-prefix/", + expectedName: "container", + }, + { + name: "explicit destination wins over the client defaults", + clientOptions: ClientOptions{ContainerName: "container", Prefix: "client-prefix/"}, + destination: &ExportDestination{Container: "other", Prefix: "explicit/"}, + mode: ExportModeBatch, + expectedPrefix: "explicit/", + expectedName: "other", + }, + { + name: "an explicit container still gets the derived prefix", + clientOptions: ClientOptions{ContainerName: "container"}, + destination: &ExportDestination{Container: "other"}, + mode: ExportModeBatch, + expectedPrefix: "batch-job-1/", + expectedName: "other", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, test.clientOptions) + job, err := client.JobClient("job-1") + require.NoError(t, err) + + options := JobCreationOptions{Mode: test.mode, Destination: test.destination} + if test.mode == ExportModeBatch { + options.CompletedTimeFrom = time.Now().UTC().Add(-time.Hour) + options.CompletedTimeTo = time.Now().UTC().Add(-time.Minute) + } + require.NoError(t, job.Create(context.Background(), options)) + + require.Len(t, hub.scheduled, 1) + created, ok := hub.scheduled[0].request.Input.(map[string]any) + require.True(t, ok) + destination, ok := created["Destination"].(map[string]any) + require.True(t, ok) + assert.Equal(t, test.expectedName, destination["Container"]) + assert.Equal(t, test.expectedPrefix, destination["Prefix"]) + }) + } +} + +func TestCreateJobRequiresAContainer(t *testing.T) { + client := newTestClient(t, newFakeBackend(), ClientOptions{}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + err = job.Create(context.Background(), JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "ClientOptions.ContainerName") +} + +func TestCreateJobRejectsMismatchedJobID(t *testing.T) { + client := newTestClient(t, newFakeBackend(), ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + err = job.Create(context.Background(), JobCreationOptions{ + JobID: "other-job", + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "does not match export job client") +} + +func TestCreateJobPropagatesValidationBeforeScheduling(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.CreateJob(context.Background(), JobCreationOptions{Mode: ExportModeBatch}) + require.ErrorIs(t, err, ErrValidation) + assert.Empty(t, hub.scheduled) +} + +func TestCreateJobReportsOrchestrationFailure(t *testing.T) { + hub := newFakeBackend() + hub.completion = &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: invalidTransitionErrorType, + Properties: map[string]any{"from": float64(1), "to": float64(1), "operation": createOperation}, + }, + } + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.CreateJob(context.Background(), JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.ErrorIs(t, err, ErrJobInvalidTransition) + + hub.completion = &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorType: "Contoso.Boom", ErrorMessage: "boom"}, + } + _, err = client.CreateJob(context.Background(), JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.ErrorIs(t, err, ErrJobOperationFailed) + assert.Contains(t, err.Error(), "boom") +} + +func TestCreateJobPropagatesTransportErrors(t *testing.T) { + scheduleFailure := errors.New("schedule failed") + hub := newFakeBackend() + hub.scheduleErr = scheduleFailure + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.CreateJob(context.Background(), JobCreationOptions{Mode: ExportModeContinuous}) + require.ErrorIs(t, err, scheduleFailure) + + waitFailure := errors.New("wait failed") + hub = newFakeBackend() + hub.completionErr = waitFailure + client = newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err = client.CreateJob(context.Background(), JobCreationOptions{Mode: ExportModeContinuous}) + require.ErrorIs(t, err, waitFailure) +} + +func TestGetJob(t *testing.T) { + t.Run("returns the description when the job exists", func(t *testing.T) { + now := time.Now().UTC().Truncate(time.Millisecond) + hub := newFakeBackend() + hub.entity = entityMetadata(t, "job-1", ExportJobState{ + Status: ExportJobStatusActive, + CreatedAt: &now, + LastModifiedAt: &now, + ScannedInstances: 3, + ExportedInstances: 2, + Config: &ExportJobConfiguration{ + Mode: ExportModeBatch, + Destination: ExportDestination{Container: "container"}, + Format: DefaultExportFormat(), + }, + }) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + description, err := client.GetJob(context.Background(), "job-1") + require.NoError(t, err) + assert.Equal(t, "job-1", description.JobID) + assert.Equal(t, ExportJobStatusActive, description.Status) + assert.Equal(t, int64(3), description.ScannedInstances) + assert.Equal(t, int64(2), description.ExportedInstances) + require.NotNil(t, description.Config) + assert.Equal(t, ExportModeBatch, description.Config.Mode) + }) + + t.Run("reports a typed not-found error", func(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.GetJob(context.Background(), "job-1") + require.ErrorIs(t, err, ErrJobNotFound) + var notFound *NotFoundError + require.ErrorAs(t, err, ¬Found) + assert.Equal(t, "job-1", notFound.JobID) + }) + + t.Run("maps a missing instance to not found", func(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.GetJob(context.Background(), "job-1") + require.ErrorIs(t, err, ErrJobNotFound) + }) + + t.Run("a deleted entity with empty state is not found", func(t *testing.T) { + hub := newFakeBackend() + hub.entity = &api.EntityMetadata{ + InstanceID: EntityID("job-1"), + StateIncluded: true, + } + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.GetJob(context.Background(), "job-1") + require.ErrorIs(t, err, ErrJobNotFound) + }) + + t.Run("corrupt state surfaces a deserialization error", func(t *testing.T) { + hub := newFakeBackend() + hub.entity = &api.EntityMetadata{ + InstanceID: EntityID("job-1"), + StateIncluded: true, + HasState: true, + SerializedState: "not json", + } + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.GetJob(context.Background(), "job-1") + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to deserialize export job state") + }) + + t.Run("propagates transport errors", func(t *testing.T) { + failure := errors.New("fetch failed") + hub := newFakeBackend() + hub.entityErr = failure + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.GetJob(context.Background(), "job-1") + require.ErrorIs(t, err, failure) + }) +} + +func TestListJobs(t *testing.T) { + older := time.Now().UTC().Add(-24 * time.Hour) + newer := time.Now().UTC() + pages := func(t *testing.T) *api.EntityQueryResults { + t.Helper() + return &api.EntityQueryResults{ + Entities: []*api.EntityMetadata{ + entityMetadata(t, "job-1", ExportJobState{Status: ExportJobStatusActive, CreatedAt: &older}), + entityMetadata(t, "job-2", ExportJobState{Status: ExportJobStatusCompleted, CreatedAt: &newer}), + {InstanceID: EntityID("job-3")}, + }, + ContinuationToken: "next", + } + } + + t.Run("returns every job when unfiltered", func(t *testing.T) { + hub := newFakeBackend() + hub.queryPages = pages(t) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + result, err := client.ListJobs(context.Background(), ExportJobQuery{}) + require.NoError(t, err) + require.Len(t, result.Jobs, 2) + assert.Equal(t, "job-1", result.Jobs[0].JobID) + assert.Equal(t, "job-2", result.Jobs[1].JobID) + assert.Equal(t, "next", result.ContinuationToken) + assert.Equal(t, "@exportjob@", hub.lastQuery.InstanceIDStartsWith) + assert.False(t, hub.lastQuery.ExcludeState) + assert.Equal(t, int32(DefaultJobQueryPageSize), hub.lastQuery.PageSize) + }) + + t.Run("filters by status", func(t *testing.T) { + hub := newFakeBackend() + hub.queryPages = pages(t) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + active := ExportJobStatusActive + result, err := client.ListJobs(context.Background(), ExportJobQuery{Status: &active}) + require.NoError(t, err) + require.Len(t, result.Jobs, 1) + assert.Equal(t, "job-1", result.Jobs[0].JobID) + }) + + t.Run("rejects an invalid status", func(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + invalid := ExportJobStatus(99) + _, err := client.ListJobs(context.Background(), ExportJobQuery{Status: &invalid}) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "invalid export job status") + assert.Zero(t, hub.lastQuery) + }) + + t.Run("filters by creation time", func(t *testing.T) { + hub := newFakeBackend() + hub.queryPages = pages(t) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + result, err := client.ListJobs(context.Background(), ExportJobQuery{ + CreatedFrom: older.Add(time.Hour), + }) + require.NoError(t, err) + require.Len(t, result.Jobs, 1) + assert.Equal(t, "job-2", result.Jobs[0].JobID) + }) + + t.Run("applies the job ID prefix to the entity query", func(t *testing.T) { + hub := newFakeBackend() + hub.queryPages = &api.EntityQueryResults{} + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.ListJobs(context.Background(), ExportJobQuery{ + JobIDPrefix: "nightly-", + PageSize: 25, + ContinuationToken: "token", + }) + require.NoError(t, err) + assert.Equal(t, "@exportjob@nightly-", hub.lastQuery.InstanceIDStartsWith) + assert.Equal(t, int32(25), hub.lastQuery.PageSize) + assert.Equal(t, "token", hub.lastQuery.ContinuationToken) + }) + + t.Run("validates the page size and prefix", func(t *testing.T) { + client := newTestClient(t, newFakeBackend(), ClientOptions{ContainerName: "container"}) + _, err := client.ListJobs(context.Background(), ExportJobQuery{PageSize: -1}) + require.ErrorIs(t, err, ErrValidation) + _, err = client.ListJobs(context.Background(), ExportJobQuery{PageSize: api.MaxInstanceQueryPageSize + 1}) + require.ErrorIs(t, err, ErrValidation) + _, err = client.ListJobs(context.Background(), ExportJobQuery{JobIDPrefix: "bad@prefix"}) + require.ErrorIs(t, err, ErrValidation) + }) + + t.Run("surfaces query errors", func(t *testing.T) { + failure := errors.New("query failed") + hub := newFakeBackend() + hub.queryErr = failure + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err := client.ListJobs(context.Background(), ExportJobQuery{}) + require.ErrorIs(t, err, failure) + + hub = newFakeBackend() + client = newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + _, err = client.ListJobs(context.Background(), ExportJobQuery{}) + require.Error(t, err) + assert.Contains(t, err.Error(), "returned no result") + }) + + t.Run("rejects a nil client", func(t *testing.T) { + var client *Client + _, err := client.ListJobs(context.Background(), ExportJobQuery{}) + require.ErrorIs(t, err, ErrValidation) + _, err = client.CreateJob(context.Background(), JobCreationOptions{}) + require.ErrorIs(t, err, ErrValidation) + }) +} + +func TestDeleteJob(t *testing.T) { + instanceID := api.InstanceID("ExportJob-job-1-run-removed") + newBackend := func() *fakeBackend { + hub := newFakeBackend() + payload, err := json.Marshal(string(instanceID)) + require.NoError(t, err) + hub.completion.SerializedOutput = string(payload) + return hub + } + + t.Run("deletes the entity then terminates and purges the orchestration", func(t *testing.T) { + hub := newBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Delete(context.Background())) + + require.Len(t, hub.scheduled, 1) + assert.Equal(t, deleteOperation, hub.scheduled[0].request.OperationName) + assert.Equal(t, []api.InstanceID{instanceID}, hub.terminated) + assert.Equal(t, []api.InstanceID{instanceID}, hub.purged) + assert.Equal(t, []api.InstanceID{"test-instance", instanceID}, hub.waited) + assert.Empty(t, hub.entityReads) + }) + + t.Run("an absent job returns an empty string and needs no cleanup", func(t *testing.T) { + hub := newBackend() + hub.completion.SerializedOutput = `""` + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Delete(context.Background())) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + assert.Empty(t, hub.entityReads) + assert.Equal(t, []api.InstanceID{"test-instance"}, hub.waited) + }) + + t.Run("tolerates a missing orchestration", func(t *testing.T) { + for _, name := range []string{"terminate", "wait", "purge"} { + hub := newBackend() + switch name { + case "terminate": + hub.terminateErr = api.ErrInstanceNotFound + case "wait": + hub.completionErrByInstance = map[api.InstanceID]error{ + instanceID: api.ErrInstanceNotFound, + } + case "purge": + hub.purgeErr = api.ErrInstanceNotFound + } + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Delete(context.Background()), name) + } + }) + + t.Run("reports other cleanup failures", func(t *testing.T) { + for _, step := range []string{"terminate", "wait", "purge"} { + t.Run(step, func(t *testing.T) { + failure := errors.New(step + " failed") + hub := newBackend() + switch step { + case "terminate": + hub.terminateErr = failure + case "wait": + hub.completionErrByInstance = map[api.InstanceID]error{instanceID: failure} + case "purge": + hub.purgeErr = failure + } + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + require.ErrorIs(t, job.Delete(context.Background()), failure) + assert.Empty(t, hub.entityReads) + if step != "purge" { + assert.Empty(t, hub.purged) + } + }) + } + }) + + t.Run("does not clean up when the entity delete fails", func(t *testing.T) { + hub := newFakeBackend() + hub.completion = &api.OrchestrationMetadata{RuntimeStatus: api.RUNTIME_STATUS_FAILED} + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.Error(t, job.Delete(context.Background())) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + }) + + t.Run("an unreadable delete result never triggers cleanup or Describe", func(t *testing.T) { + for _, output := range []string{"", "not-json", `{"InstanceID":"wrong-shape"}`, `123`} { + hub := newBackend() + hub.completion.SerializedOutput = output + hub.entity = entityMetadata(t, "job-1", ExportJobState{ + Status: ExportJobStatusActive, OrchestratorInstanceID: "replacement-run", + }) + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + err = job.Delete(context.Background()) + require.Error(t, err, "output %q", output) + if output == "" { + assert.Contains(t, err.Error(), "Delete returned no output") + } else { + assert.Contains(t, err.Error(), "failed to decode Delete result") + } + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + assert.Empty(t, hub.entityReads) + } + }) + + t.Run("missing operation metadata reports an error without cleanup", func(t *testing.T) { + hub := newBackend() + hub.completion = nil + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + require.ErrorIs(t, job.Delete(context.Background()), ErrJobOperationFailed) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + }) +} + +func TestCreateDoesNotCleanUpPreviousRuns(t *testing.T) { + options := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + } + for _, status := range []ExportJobStatus{ExportJobStatusCompleted, ExportJobStatusFailed, ExportJobStatusActive} { + t.Run(status.String(), func(t *testing.T) { + hub := newFakeBackend() + hub.entity = entityMetadata(t, "job-1", ExportJobState{ + Status: status, + OrchestratorInstanceID: "previous-run", + }) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Create(context.Background(), options)) + + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + assert.Empty(t, hub.entityReads) + require.Len(t, hub.scheduled, 1) + assert.Equal(t, createOperation, hub.scheduled[0].request.OperationName) + }) + } + + t.Run("a job that does not exist needs no cleanup", func(t *testing.T) { + hub := newFakeBackend() + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Create(context.Background(), options)) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + require.Len(t, hub.scheduled, 1) + }) + + t.Run("creation does not depend on management reads or cleanup", func(t *testing.T) { + hub := newFakeBackend() + hub.entityErr = errors.New("read failed") + hub.terminateErr = errors.New("terminate failed") + hub.purgeErr = errors.New("purge failed") + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Create(context.Background(), options)) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + assert.Empty(t, hub.entityReads) + require.Len(t, hub.scheduled, 1) + }) +} + +func TestDescribeUsesTheClientJobID(t *testing.T) { + hub := newFakeBackend() + // The service reports the lowercased entity key; the handle's own ID wins so + // a caller always sees the ID it asked for. + hub.entity = entityMetadata(t, "JOB-1", ExportJobState{Status: ExportJobStatusActive}) + client := newTestClient(t, hub, ClientOptions{ContainerName: "container"}) + job, err := client.JobClient("job-1") + require.NoError(t, err) + description, err := job.Describe(context.Background()) + require.NoError(t, err) + assert.Equal(t, "job-1", description.JobID) +} + +func TestDefaultPrefix(t *testing.T) { + assert.Equal(t, "batch-job/", defaultPrefix(ExportModeBatch, "job")) + assert.Equal(t, "continuous-job/", defaultPrefix(ExportModeContinuous, "job")) + assert.True(t, strings.HasSuffix(defaultPrefix(ExportModeBatch, "job"), "/")) +} diff --git a/exporthistory/doc.go b/exporthistory/doc.go new file mode 100644 index 00000000..c442f070 --- /dev/null +++ b/exporthistory/doc.go @@ -0,0 +1,77 @@ +// Package exporthistory exports terminal orchestration histories to Azure Blob +// Storage using durable entities, orchestrations, and activities. +// +// # Maturity: preview +// +// This package is a preview API. Its exported types, function signatures, the +// serialized shape of the ExportJob entity state, and the names of the system +// entity, orchestrators, and activities may change in a future release without a +// major version bump. Do not depend on it for production workloads that cannot +// tolerate a breaking change. Drain and delete export jobs before upgrading; +// legacy export-job state is not supported or migrated. Do not mix .NET or +// older Go export-history workers with these system tasks in the same task hub. +// +// # Model +// +// An export job is a durable entity keyed by job ID. Creating a job transitions +// the entity to [ExportJobStatusActive], mints a run token, and reserves an +// orchestration ID derived from the job ID and token before signaling Run. +// [ExportJobDescription.OrchestratorInstanceID] identifies that generation. +// Only the current generation can mutate job state; older orchestrations and +// already-scheduled activities can briefly remain in flight. The orchestration +// repeatedly lists terminal orchestration instances that match the job's filter, +// exports each instance's history, and commits a checkpoint back to the entity. +// A batch job completes when the task hub reports no further pages; an empty page +// with a continuation cursor advances the cursor and keeps going. A continuous +// job idles and lists again. +// +// Recreation assigns a fresh orchestration ID and retains previous generations' +// orchestration histories. Delete atomically captures the current ID and removes +// entity state, then the client terminates and purges only that captured +// generation. A concurrent recreation is not affected by that cleanup. Every +// run and orchestration-originated mutation requires a matching nonempty token. +// +// # Usage +// +// Workers register the system tasks and advertise them to the service: +// +// registry := task.NewTaskRegistry() +// store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ +// ConnectionString: connectionString, +// ContainerName: "history-exports", +// }) +// if err != nil { +// return err +// } +// if err := exporthistory.Register(registry, exporthistory.WorkerOptions{ +// Source: taskHubClient, +// Store: store, +// }); err != nil { +// return err +// } +// worker, err := durabletaskscheduler.NewWorker( +// options, registry, logger, exporthistory.WithExportHistory()) +// +// Clients create and inspect jobs: +// +// exportClient, err := exporthistory.NewClient(taskHubClient, exporthistory.ClientOptions{ +// ContainerName: "history-exports", +// }) +// if err != nil { +// return err +// } +// job, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ +// Mode: exporthistory.ExportModeBatch, +// CompletedTimeFrom: from, +// CompletedTimeTo: to, +// }) +// if err != nil { +// return err +// } +// description, err := job.Describe(ctx) +// +// # Limitations +// +// Export requires a task hub that implements the instance-ID listing and history +// streaming management APIs. Extended sessions are not supported. +package exporthistory diff --git a/exporthistory/entity.go b/exporthistory/entity.go new file mode 100644 index 00000000..9699528b --- /dev/null +++ b/exporthistory/entity.go @@ -0,0 +1,364 @@ +package exporthistory + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/microsoft/durabletask-go/task" +) + +// exportJobEntity implements the ExportJob system entity. It owns the job's +// lifecycle, configuration, checkpoint cursor, and progress counters. +func exportJobEntity(ctx *task.EntityContext) (any, error) { + var state ExportJobState + if rawState, ok := ctx.GetRawState(); ok && rawState != "" { + if err := json.Unmarshal([]byte(rawState), &state); err != nil { + return nil, fmt.Errorf("failed to deserialize export job state: %w", err) + } + } + jobID := ctx.ID.Key + + switch { + case strings.EqualFold(ctx.Operation, getOperation): + // Get never mutates, so it must not rewrite state and resurrect a + // deleted entity. + if !ctx.HasState() { + return nil, nil + } + return state, nil + case strings.EqualFold(ctx.Operation, deleteOperation): + instanceID := state.OrchestratorInstanceID + ctx.DeleteState() + return instanceID, nil + case strings.EqualFold(ctx.Operation, createOperation): + if !ctx.HasInput() { + return nil, &ValidationError{JobID: jobID, Message: "export job creation options are required"} + } + var options JobCreationOptions + if err := ctx.GetInput(&options); err != nil { + return nil, &ValidationError{JobID: jobID, Message: "export job creation options could not be deserialized"} + } + if options.isZero() { + return nil, &ValidationError{JobID: jobID, Message: "export job creation options are required"} + } + if err := createExportJob(ctx, &state, options); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, runOperation): + var request RunJobRequest + if ctx.HasInput() { + if err := ctx.GetInput(&request); err != nil { + return nil, &ValidationError{JobID: jobID, Message: "run request could not be deserialized"} + } + } + if dropsStaleMutation(ctx, &state, "a run signal", request.RunToken) { + return nil, nil + } + if err := runExportJob(ctx, &state); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, commitCheckpointOperation): + // A checkpoint that arrives after the job was deleted must not + // resurrect the entity with a configuration-less state, which would + // leave a permanently Pending job behind. The orchestration is + // terminated separately, so dropping the commit is the correct outcome. + if !ctx.HasState() { + ctx.Logger().Warn("dropping a checkpoint for a deleted export job", "jobId", jobID) + return nil, nil + } + var request CommitCheckpointRequest + if err := ctx.GetInput(&request); err != nil { + return nil, &ValidationError{JobID: jobID, Message: "checkpoint commit request is required"} + } + if dropsStaleMutation(ctx, &state, "a checkpoint", request.RunToken) { + return nil, nil + } + if err := commitCheckpoint(ctx, &state, request); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, markAsCompletedOperation): + var request MarkAsCompletedRequest + if ctx.HasInput() { + if err := ctx.GetInput(&request); err != nil { + return nil, &ValidationError{JobID: jobID, Message: "completion request could not be deserialized"} + } + } + if dropsStaleMutation(ctx, &state, "a completion", request.RunToken) { + return nil, nil + } + if err := markExportJobCompleted(ctx, &state); err != nil { + return nil, err + } + case strings.EqualFold(ctx.Operation, markAsFailedOperation): + var request MarkAsFailedRequest + if ctx.HasInput() { + if err := ctx.GetInput(&request); err != nil { + return nil, &ValidationError{JobID: jobID, Message: "failure request could not be deserialized"} + } + } + if dropsStaleMutation(ctx, &state, "a failure", request.RunToken) { + return nil, nil + } + if err := markExportJobFailed(ctx, &state, request.Error); err != nil { + return nil, err + } + default: + return nil, &ValidationError{ + JobID: jobID, + Message: fmt.Sprintf("export job does not support operation %q", ctx.Operation), + } + } + + payload, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("failed to serialize export job state: %w", err) + } + ctx.SetRawState(string(payload)) + return nil, nil +} + +// newRunToken mints the fencing token of a new job generation. +// +// The token is random rather than derived, because the one property it must +// guarantee is that a generation never reuses a predecessor's token, and a +// delete clears the state a derived token could chain from. Randomness is safe +// here because an entity operation is not replayed: its state write and its +// emitted actions are produced by a single execution, so the token the job +// stores and the token its run signal carries always agree, and a redelivered +// Create is rejected by the lifecycle transition instead of minting a second +// token for a job that is already Active. +func newRunToken() string { + return newCompactUUID() +} + +// runTokenMatches reports whether a mutation carrying token belongs to the job's +// current run generation. Both tokens must be nonempty. +func runTokenMatches(state *ExportJobState, token string) bool { + return token != "" && state.RunToken == token +} + +// dropsStaleMutation reports whether a mutation carrying token belongs to a +// superseded run generation, logging the drop when it does. Every +// orchestration-originated mutation is fenced this way, so a run left over from +// a deleted-and-recreated job cannot alter the new generation. The mutation +// argument names the dropped signal in the log entry, for example "a run +// signal". +func dropsStaleMutation(ctx *task.EntityContext, state *ExportJobState, mutation, token string) bool { + if runTokenMatches(state, token) { + return false + } + ctx.Logger().Warn( + "dropping "+mutation+" from a stale export job generation", + "jobId", ctx.ID.Key, + "requestToken", token, + ) + return true +} + +func createExportJob(ctx *task.EntityContext, state *ExportJobState, options JobCreationOptions) error { + jobID := ctx.ID.Key + if !isValidTransition(createOperation, state.Status, ExportJobStatusActive) { + return &InvalidTransitionError{ + JobID: jobID, + From: state.Status, + To: ExportJobStatusActive, + Operation: createOperation, + } + } + // The entity validates against its own clock, which is independent of the + // client's, so the upper bound is allowed a bounded skew. Clients stay + // strict, and [MaxCreationClockSkew] documents the tolerance. + normalized, err := options.normalize(entityNow(ctx), MaxCreationClockSkew) + if err != nil { + return withJobID(err, jobID) + } + if normalized.JobID != jobID { + return &ValidationError{ + JobID: jobID, + Message: fmt.Sprintf("creation job ID %q does not match entity key %q", normalized.JobID, jobID), + } + } + config, err := normalized.configuration() + if err != nil { + return withJobID(err, jobID) + } + + now := entityNow(ctx) + createdAt := now + if state.CreatedAt != nil { + createdAt = *state.CreatedAt + } + // Recreating a terminal job resets progress and the cursor so the new run + // starts from the beginning of its window, and mints a new run token so a + // run left over from the previous generation cannot commit into it. + runToken := newRunToken() + *state = ExportJobState{ + Status: ExportJobStatusActive, + Config: config, + CreatedAt: &createdAt, + LastModifiedAt: &now, + OrchestratorInstanceID: OrchestratorInstanceIDPrefix + jobID + "-" + runToken, + RunToken: runToken, + } + + ctx.Logger().Info("created export job", "jobId", jobID, "mode", config.Mode.String()) + return ctx.SignalEntity(ctx.ID, runOperation, RunJobRequest{RunToken: runToken}) +} + +func runExportJob(ctx *task.EntityContext, state *ExportJobState) error { + jobID := ctx.ID.Key + if state.Config == nil { + return &ValidationError{JobID: jobID, Message: "export job configuration is missing"} + } + if state.Status != ExportJobStatusActive { + return &ValidationError{JobID: jobID, Message: "export job must be in Active status to run"} + } + if state.OrchestratorInstanceID == "" { + return &ValidationError{JobID: jobID, Message: "export job orchestration instance ID is missing"} + } + return startExportOrchestration(ctx, state) +} + +// startExportOrchestration starts the job's dedicated orchestration. A failure +// to schedule is recorded on the job instead of failing the operation, matching +// the upstream behavior of leaving the entity readable with a diagnosable error. +func startExportOrchestration(ctx *task.EntityContext, state *ExportJobState) error { + jobID := ctx.ID.Key + request := ExportJobRunRequest{JobEntityID: ctx.ID, RunToken: state.RunToken} + payload, err := ctx.SerializeInput(request) + if err != nil { + return fmt.Errorf("failed to serialize export job run request: %w", err) + } + now := entityNow(ctx) + startErr := ctx.StartNewOrchestration( + ExportJobOrchestratorName, + task.WithEntityStartOrchestrationInstanceID(state.OrchestratorInstanceID), + task.WithRawEntityStartOrchestrationInput(payload), + // System orchestrations must not inherit an application default version. + task.WithEntityStartOrchestrationVersion(task.UnversionedTaskVersion), + ) + if startErr != nil { + state.Status = ExportJobStatusFailed + state.LastError = startErr.Error() + state.LastModifiedAt = &now + ctx.Logger().Error("failed to start export orchestration", "jobId", jobID, "error", startErr) + return nil + } + state.LastModifiedAt = &now + return nil +} + +// commitCheckpoint records one batch's progress. A nil checkpoint keeps the +// cursor so the same page is retried; combined with failures it implicitly +// fails the job, which is how a batch that exhausted its retries stops the run. +func commitCheckpoint(ctx *task.EntityContext, state *ExportJobState, request CommitCheckpointRequest) error { + if request.ScannedInstances < 0 || request.ExportedInstances < 0 { + return &ValidationError{ + JobID: ctx.ID.Key, + Message: "checkpoint progress counts must not be negative", + } + } + if request.Checkpoint != nil && len(request.Failures) > 0 { + return &ValidationError{ + JobID: ctx.ID.Key, + Message: "checkpoint and failures cannot be committed together", + } + } + state.ScannedInstances += request.ScannedInstances + state.ExportedInstances += request.ExportedInstances + if request.Checkpoint != nil { + checkpoint := *request.Checkpoint + state.Checkpoint = &checkpoint + } + now := entityNow(ctx) + state.LastCheckpointTime = &now + state.LastModifiedAt = &now + + if request.Checkpoint == nil && len(request.Failures) > 0 { + // The implicit failure goes through the same transition helper as an + // explicit MarkAsFailed, so the lifecycle rules stay in one place. A + // job that already left Active keeps its terminal state; the progress + // this commit carried is still recorded above. + message := "Batch export failed after retries. Failures: " + summarizeFailures(request.Failures) + if err := markExportJobFailed(ctx, state, message); err != nil { + ctx.Logger().Warn("a failing checkpoint could not fail the export job", + "jobId", ctx.ID.Key, "status", state.Status.String(), "error", err) + } + } + return nil +} + +func markExportJobCompleted(ctx *task.EntityContext, state *ExportJobState) error { + jobID := ctx.ID.Key + if !isValidTransition(markAsCompletedOperation, state.Status, ExportJobStatusCompleted) { + return &InvalidTransitionError{ + JobID: jobID, + From: state.Status, + To: ExportJobStatusCompleted, + Operation: markAsCompletedOperation, + } + } + now := entityNow(ctx) + state.Status = ExportJobStatusCompleted + state.LastError = "" + state.LastModifiedAt = &now + ctx.Logger().Info("export job completed", "jobId", jobID) + return nil +} + +func markExportJobFailed(ctx *task.EntityContext, state *ExportJobState, errorMessage string) error { + jobID := ctx.ID.Key + if !isValidTransition(markAsFailedOperation, state.Status, ExportJobStatusFailed) { + return &InvalidTransitionError{ + JobID: jobID, + From: state.Status, + To: ExportJobStatusFailed, + Operation: markAsFailedOperation, + } + } + now := entityNow(ctx) + state.Status = ExportJobStatusFailed + state.LastError = errorMessage + state.LastModifiedAt = &now + ctx.Logger().Warn("export job failed", "jobId", jobID, "error", errorMessage) + return nil +} + +// maxSummarizedFailures bounds the failure detail written to entity state and +// reported in the orchestration's terminal error, so a large failing batch +// cannot grow either without limit. +const maxSummarizedFailures = 10 + +// summarizeFailures renders a bounded, human-readable summary of the instances +// a batch could not export. It is the single summarizer the entity and the +// orchestration share, so both report the same detail under the same limit. +func summarizeFailures(failures []ExportFailure) string { + if len(failures) == 0 { + return "no failure details available" + } + var builder strings.Builder + limit := min(len(failures), maxSummarizedFailures) + for i := 0; i < limit; i++ { + if i > 0 { + builder.WriteString("; ") + } + fmt.Fprintf(&builder, "InstanceId: %s, Reason: %s", failures[i].InstanceID, failures[i].Reason) + } + if remaining := len(failures) - limit; remaining > 0 { + fmt.Fprintf(&builder, " ... and %d more failures", remaining) + } + return builder.String() +} + +// entityNow returns the timestamp the entity operation should treat as the +// current time. The service stamps each operation with a durable timestamp, so +// two workers processing the same job agree on the instant it recorded; the wall +// clock is only a fallback for transports that do not supply one. +func entityNow(ctx *task.EntityContext) time.Time { + if now := ctx.CurrentTimeUTC(); !now.IsZero() { + return now.UTC() + } + return time.Now().UTC() +} diff --git a/exporthistory/entity_test.go b/exporthistory/entity_test.go new file mode 100644 index 00000000..d959968b --- /dev/null +++ b/exporthistory/entity_test.go @@ -0,0 +1,847 @@ +package exporthistory + +import ( + "context" + "encoding/json" + "fmt" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// entityHarness drives the ExportJob entity through the production entity +// executor, so operation dispatch, state persistence, and the emitted signal and +// start-orchestration actions are exercised exactly as the worker runs them. +type entityHarness struct { + t *testing.T + executor task.EntityExecutor + converter api.DataConverter + jobID string + state *wrapperspb.StringValue + actions []*protos.OperationAction +} + +func newEntityHarness(t *testing.T, jobID string, converters ...api.DataConverter) *entityHarness { + t.Helper() + converter := api.DefaultDataConverter() + if len(converters) > 0 { + converter = api.NormalizeDataConverter(converters[0]) + } + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN(ExportJobEntityName, exportJobEntity)) + executor, ok := task.NewTaskExecutor(registry, task.WithDataConverter(converter)).(task.EntityExecutor) + require.True(t, ok) + return &entityHarness{t: t, executor: executor, converter: converter, jobID: jobID} +} + +// call runs one operation and returns its serialized result. It fails the test +// when the operation reports a failure. +func (h *entityHarness) call(operation string, input any) string { + h.t.Helper() + result, err := h.tryCall(operation, input) + require.NoError(h.t, err) + return result +} + +// tryCall runs one operation and returns the entity's failure as a Go error. +func (h *entityHarness) tryCall(operation string, input any) (string, error) { + h.t.Helper() + var rawInput *wrapperspb.StringValue + if input != nil { + payload, err := api.SerializeData(h.converter, input) + require.NoError(h.t, err) + rawInput = wrapperspb.String(payload) + } + return h.tryCallRaw(operation, rawInput) +} + +func (h *entityHarness) tryCallRaw(operation string, input *wrapperspb.StringValue) (string, error) { + h.t.Helper() + request := &protos.EntityBatchRequest{ + InstanceId: EntityID(h.jobID).String(), + EntityState: h.state, + Operations: []*protos.OperationRequest{{ + Operation: operation, + RequestId: operation + "-request", + Input: input, + }}, + } + result, err := h.executor.ExecuteEntity(context.Background(), request) + require.NoError(h.t, err) + require.Len(h.t, result.Results, 1) + h.state = result.EntityState + h.actions = result.Actions + + if failure := result.Results[0].GetFailure(); failure != nil { + return "", entityFailure{details: failure.GetFailureDetails()} + } + return result.Results[0].GetSuccess().GetResult().GetValue(), nil +} + +// entityFailure adapts a protobuf failure into an error so tests can assert on +// the error type the entity produced. +type entityFailure struct{ details *protos.TaskFailureDetails } + +func (e entityFailure) Error() string { + if e.details == nil { + return "entity operation failed" + } + return e.details.GetErrorType() + ": " + e.details.GetErrorMessage() +} + +// assertEntityErrorType asserts the durable error type the entity reported. +func assertEntityErrorType(t *testing.T, err error, expected api.ErrorType) { + t.Helper() + var failure entityFailure + require.ErrorAs(t, err, &failure) + assert.Equal(t, string(expected), failure.details.GetErrorType()) +} + +func (h *entityHarness) jobState() *ExportJobState { + h.t.Helper() + if h.state == nil { + return nil + } + var state ExportJobState + require.NoError(h.t, json.Unmarshal([]byte(h.state.GetValue()), &state)) + return &state +} + +func (h *entityHarness) hasState() bool { return h.state != nil } + +func (h *entityHarness) runToken() string { + h.t.Helper() + state := h.jobState() + require.NotNil(h.t, state) + return state.RunToken +} + +func batchOptions(jobID string) JobCreationOptions { + options, err := JobCreationOptions{ + JobID: jobID, + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-24 * time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + Destination: &ExportDestination{Container: "test-container"}, + }.Normalize() + if err != nil { + panic(err) + } + return options +} + +// TestEntityCreate ports the upstream ExportJobTests create scenarios. +func TestEntityCreate(t *testing.T) { + t.Run("valid options activate the job and start the orchestration", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + + state := harness.jobState() + require.NotNil(t, state) + assert.Equal(t, ExportJobStatusActive, state.Status) + require.NotNil(t, state.Config) + assert.Equal(t, ExportModeBatch, state.Config.Mode) + assert.Equal(t, "test-container", state.Config.Destination.Container) + require.NotNil(t, state.CreatedAt) + require.NotNil(t, state.LastModifiedAt) + assert.Empty(t, state.LastError) + assert.Zero(t, state.ScannedInstances) + assert.Zero(t, state.ExportedInstances) + assert.Nil(t, state.Checkpoint) + require.NotEmpty(t, state.RunToken) + assert.Equal(t, OrchestratorInstanceIDPrefix+"job-1-"+state.RunToken, state.OrchestratorInstanceID) + + // Create signals Run rather than starting the orchestration directly, so + // the start action only appears after the signal is delivered. + require.Len(t, harness.actions, 1) + signal := harness.actions[0].GetSendSignal() + require.NotNil(t, signal) + assert.Equal(t, runOperation, signal.GetName()) + assert.Equal(t, EntityID("job-1").String(), signal.GetInstanceId()) + }) + + t.Run("missing options are rejected", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + _, err := harness.tryCall(createOperation, nil) + require.Error(t, err) + assertEntityErrorType(t, err, validationErrorType) + assert.Contains(t, err.Error(), "creation options are required") + assert.False(t, harness.hasState()) + }) + + t.Run("mismatched job ID is rejected", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + _, err := harness.tryCall(createOperation, batchOptions("other-job")) + require.Error(t, err) + assertEntityErrorType(t, err, validationErrorType) + assert.Contains(t, err.Error(), "does not match entity key") + }) + + t.Run("invalid options are rejected inside the entity", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + _, err := harness.tryCall(createOperation, JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeBatch, + Destination: &ExportDestination{Container: "test-container"}, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "CompletedTimeFrom is required") + }) + + t.Run("a destination is required", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + _, err := harness.tryCall(createOperation, JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC().Add(-time.Minute), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "export destination is required") + }) + + t.Run("recreating an active job is rejected", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + _, err := harness.tryCall(createOperation, batchOptions("job-1")) + require.Error(t, err) + assertEntityErrorType(t, err, invalidTransitionErrorType) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + }) + + t.Run("recreating a failed job resets progress and keeps CreatedAt", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 5, + ExportedInstances: 5, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + RunToken: harness.runToken(), + }) + harness.call(markAsFailedOperation, MarkAsFailedRequest{ + RunToken: harness.runToken(), + Error: "test error", + }) + originalCreatedAt := *harness.jobState().CreatedAt + + harness.call(createOperation, batchOptions("job-1")) + state := harness.jobState() + assert.Equal(t, ExportJobStatusActive, state.Status) + assert.Zero(t, state.ScannedInstances) + assert.Zero(t, state.ExportedInstances) + assert.Nil(t, state.Checkpoint) + assert.Nil(t, state.LastCheckpointTime) + assert.Empty(t, state.LastError) + require.NotNil(t, state.CreatedAt) + assert.True(t, state.CreatedAt.Equal(originalCreatedAt)) + }) + + t.Run("recreating a completed job is allowed", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: harness.runToken()}) + harness.call(createOperation, batchOptions("job-1")) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + }) +} + +func TestEntityGet(t *testing.T) { + t.Run("returns the current state", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + result := harness.call(getOperation, nil) + + var state ExportJobState + require.NoError(t, json.Unmarshal([]byte(result), &state)) + assert.Equal(t, ExportJobStatusActive, state.Status) + require.NotNil(t, state.Config) + }) + + t.Run("does not resurrect a deleted entity", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(deleteOperation, nil) + require.False(t, harness.hasState()) + + result := harness.call(getOperation, nil) + assert.Empty(t, result) + assert.False(t, harness.hasState()) + }) + + t.Run("reads emit no actions", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(getOperation, nil) + assert.Empty(t, harness.actions) + }) +} + +func TestEntityRun(t *testing.T) { + t.Run("starts the orchestration with the ID persisted by Create", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + instanceID := harness.jobState().OrchestratorInstanceID + require.NotEmpty(t, instanceID) + harness.call(runOperation, RunJobRequest{RunToken: harness.runToken()}) + + require.Len(t, harness.actions, 1) + start := harness.actions[0].GetStartNewOrchestration() + require.NotNil(t, start) + assert.Equal(t, ExportJobOrchestratorName, start.GetName()) + assert.Equal(t, instanceID, start.GetInstanceId()) + // System orchestrations are started explicitly unversioned. + assert.Equal(t, task.UnversionedTaskVersion, start.GetVersion().GetValue()) + + var request ExportJobRunRequest + require.NoError(t, json.Unmarshal([]byte(start.GetInput().GetValue()), &request)) + assert.Equal(t, EntityID("job-1"), request.JobEntityID) + assert.Zero(t, request.ProcessedCycles) + + assert.Equal(t, instanceID, harness.jobState().OrchestratorInstanceID) + + harness.call(runOperation, RunJobRequest{RunToken: harness.runToken()}) + require.Len(t, harness.actions, 1) + assert.Equal(t, instanceID, harness.actions[0].GetStartNewOrchestration().GetInstanceId(), + "a duplicate Run must target the same generation") + }) + + t.Run("rejects a job without configuration", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.state = wrapperspb.String(`{"Status":1,"RunToken":"run-a"}`) + _, err := harness.tryCall(runOperation, RunJobRequest{RunToken: "run-a"}) + require.Error(t, err) + assert.Contains(t, err.Error(), "configuration is missing") + }) + + t.Run("rejects a missing stored orchestration ID without deriving a replacement", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + state := harness.jobState() + state.OrchestratorInstanceID = "" + payload, err := json.Marshal(state) + require.NoError(t, err) + harness.state = wrapperspb.String(string(payload)) + + _, err = harness.tryCall(runOperation, RunJobRequest{RunToken: state.RunToken}) + require.ErrorContains(t, err, "orchestration instance ID is missing") + assert.Empty(t, harness.actions) + }) + + t.Run("uses the stored ID rather than reconstructing it from the token", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + state := harness.jobState() + state.OrchestratorInstanceID = "persisted-generation-id" + payload, err := json.Marshal(state) + require.NoError(t, err) + harness.state = wrapperspb.String(string(payload)) + + harness.call(runOperation, RunJobRequest{RunToken: state.RunToken}) + require.Len(t, harness.actions, 1) + assert.Equal(t, state.OrchestratorInstanceID, harness.actions[0].GetStartNewOrchestration().GetInstanceId()) + }) + + t.Run("rejects a job that is not active", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: harness.runToken()}) + _, err := harness.tryCall(runOperation, RunJobRequest{RunToken: harness.runToken()}) + require.Error(t, err) + assert.Contains(t, err.Error(), "must be in Active status to run") + }) +} + +// TestEntityCommitCheckpoint covers durable checkpointing and the implicit +// failure a failed batch produces. +func TestEntityCommitCheckpoint(t *testing.T) { + t.Run("advances the cursor and accumulates progress", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 100, + ExportedInstances: 95, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "last-key"}, + RunToken: harness.runToken(), + }) + state := harness.jobState() + assert.Equal(t, int64(100), state.ScannedInstances) + assert.Equal(t, int64(95), state.ExportedInstances) + require.NotNil(t, state.Checkpoint) + assert.Equal(t, "last-key", state.Checkpoint.LastInstanceKey) + require.NotNil(t, state.LastCheckpointTime) + assert.Equal(t, ExportJobStatusActive, state.Status) + + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 10, + ExportedInstances: 10, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "next-key"}, + RunToken: harness.runToken(), + }) + state = harness.jobState() + assert.Equal(t, int64(110), state.ScannedInstances) + assert.Equal(t, int64(105), state.ExportedInstances) + assert.Equal(t, "next-key", state.Checkpoint.LastInstanceKey) + }) + + t.Run("a nil checkpoint keeps the cursor", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 1, + ExportedInstances: 1, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "keep-me"}, + RunToken: harness.runToken(), + }) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{RunToken: harness.runToken()}) + state := harness.jobState() + require.NotNil(t, state.Checkpoint) + assert.Equal(t, "keep-me", state.Checkpoint.LastInstanceKey) + assert.Equal(t, ExportJobStatusActive, state.Status) + }) + + t.Run("failures without a checkpoint implicitly fail the job", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + Failures: []ExportFailure{ + {InstanceID: "instance-1", Reason: "error1", AttemptCount: 1, LastAttempt: time.Now().UTC()}, + {InstanceID: "instance-2", Reason: "error2", AttemptCount: 2, LastAttempt: time.Now().UTC()}, + }, + RunToken: harness.runToken(), + }) + state := harness.jobState() + assert.Equal(t, ExportJobStatusFailed, state.Status) + assert.Contains(t, state.LastError, "Batch export failed after retries") + assert.Contains(t, state.LastError, "InstanceId: instance-1, Reason: error1") + assert.Contains(t, state.LastError, "InstanceId: instance-2, Reason: error2") + }) + + t.Run("failures cannot be silently discarded by a checkpoint", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + before := harness.jobState() + _, err := harness.tryCall(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 1, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + Failures: []ExportFailure{{InstanceID: "i", Reason: "r"}}, + RunToken: harness.runToken(), + }) + require.ErrorContains(t, err, "checkpoint and failures cannot be committed together") + after := harness.jobState() + assert.Equal(t, before.Status, after.Status) + assert.Equal(t, before.ScannedInstances, after.ScannedInstances) + assert.Equal(t, before.ExportedInstances, after.ExportedInstances) + assert.Equal(t, before.Checkpoint, after.Checkpoint) + }) + + t.Run("a checkpoint for a deleted job does not resurrect it", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(deleteOperation, nil) + require.False(t, harness.hasState()) + + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 5, + ExportedInstances: 5, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + }) + assert.False(t, harness.hasState(), "a deleted export job must stay deleted") + }) + + t.Run("negative progress is rejected", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + _, err := harness.tryCall(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: -1, + RunToken: harness.runToken(), + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must not be negative") + }) + + t.Run("the persisted failure summary is bounded", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + failures := make([]ExportFailure, 0, 25) + for i := 0; i < 25; i++ { + failures = append(failures, ExportFailure{InstanceID: "instance", Reason: "boom"}) + } + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + Failures: failures, + RunToken: harness.runToken(), + }) + assert.Contains(t, harness.jobState().LastError, "and 15 more failures") + }) +} + +func TestEntityMarkAsCompletedAndFailed(t *testing.T) { + t.Run("active jobs complete", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: harness.runToken()}) + state := harness.jobState() + assert.Equal(t, ExportJobStatusCompleted, state.Status) + assert.Empty(t, state.LastError) + }) + + t.Run("completing a non-active job is an invalid transition", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsFailedOperation, MarkAsFailedRequest{ + RunToken: harness.runToken(), + Error: "test error", + }) + _, err := harness.tryCall( + markAsCompletedOperation, + MarkAsCompletedRequest{RunToken: harness.runToken()}, + ) + require.Error(t, err) + assertEntityErrorType(t, err, invalidTransitionErrorType) + }) + + t.Run("active jobs fail with a message", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsFailedOperation, MarkAsFailedRequest{ + RunToken: harness.runToken(), + Error: "Test error", + }) + state := harness.jobState() + assert.Equal(t, ExportJobStatusFailed, state.Status) + assert.Equal(t, "Test error", state.LastError) + }) + + t.Run("failing without a message is allowed", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsFailedOperation, MarkAsFailedRequest{RunToken: harness.runToken()}) + assert.Equal(t, ExportJobStatusFailed, harness.jobState().Status) + }) + + t.Run("failing a non-active job is an invalid transition", func(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: harness.runToken()}) + _, err := harness.tryCall(markAsFailedOperation, MarkAsFailedRequest{ + RunToken: harness.runToken(), + Error: "boom", + }) + require.Error(t, err) + assertEntityErrorType(t, err, invalidTransitionErrorType) + }) +} + +// TestEntityRunGenerationFencing covers run fencing on the entity side: every +// orchestration-originated mutation carrying a stale generation token is dropped +// so a run left over from a deleted-and-recreated job cannot alter the new one. +func TestEntityRunGenerationFencing(t *testing.T) { + // newGeneration deletes and recreates the job, returning the token of the + // previous generation and of the new one. + newGeneration := func(t *testing.T) (*entityHarness, string, string) { + t.Helper() + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + stale := harness.jobState().RunToken + require.NotEmpty(t, stale) + + harness.call(deleteOperation, nil) + harness.call(createOperation, batchOptions("job-1")) + current := harness.jobState().RunToken + require.NotEmpty(t, current) + require.NotEqual(t, stale, current, + "a delete-and-recreate must mint a new run generation") + return harness, stale, current + } + + t.Run("a stale checkpoint is dropped", func(t *testing.T) { + harness, stale, current := newGeneration(t) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 5, + ExportedInstances: 5, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "stale-cursor"}, + RunToken: stale, + }) + state := harness.jobState() + assert.Zero(t, state.ScannedInstances) + assert.Zero(t, state.ExportedInstances) + assert.Nil(t, state.Checkpoint) + assert.Equal(t, current, state.RunToken) + + // The current generation's checkpoint still applies. + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + ScannedInstances: 2, + ExportedInstances: 2, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "fresh-cursor"}, + RunToken: current, + }) + state = harness.jobState() + assert.Equal(t, int64(2), state.ScannedInstances) + require.NotNil(t, state.Checkpoint) + assert.Equal(t, "fresh-cursor", state.Checkpoint.LastInstanceKey) + }) + + t.Run("a stale implicit failure is dropped", func(t *testing.T) { + harness, stale, _ := newGeneration(t) + harness.call(commitCheckpointOperation, CommitCheckpointRequest{ + Failures: []ExportFailure{{InstanceID: "i1", Reason: "boom"}}, + RunToken: stale, + }) + state := harness.jobState() + assert.Equal(t, ExportJobStatusActive, state.Status) + assert.Empty(t, state.LastError) + }) + + t.Run("a stale completion is dropped", func(t *testing.T) { + harness, stale, current := newGeneration(t) + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: stale}) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + + harness.call(markAsCompletedOperation, MarkAsCompletedRequest{RunToken: current}) + assert.Equal(t, ExportJobStatusCompleted, harness.jobState().Status) + }) + + t.Run("a stale failure is dropped", func(t *testing.T) { + harness, stale, current := newGeneration(t) + harness.call(markAsFailedOperation, MarkAsFailedRequest{RunToken: stale, Error: "stale boom"}) + state := harness.jobState() + assert.Equal(t, ExportJobStatusActive, state.Status) + assert.Empty(t, state.LastError) + + harness.call(markAsFailedOperation, MarkAsFailedRequest{RunToken: current, Error: "fresh boom"}) + state = harness.jobState() + assert.Equal(t, ExportJobStatusFailed, state.Status) + assert.Equal(t, "fresh boom", state.LastError) + }) + + t.Run("a stale run signal starts no orchestration", func(t *testing.T) { + harness, stale, current := newGeneration(t) + harness.call(runOperation, RunJobRequest{RunToken: stale}) + assert.Empty(t, harness.actions, "a stale run signal must not start an orchestration") + + harness.call(runOperation, RunJobRequest{RunToken: current}) + require.Len(t, harness.actions, 1) + require.NotNil(t, harness.actions[0].GetStartNewOrchestration()) + }) + + t.Run("missing request tokens reject every mutation", func(t *testing.T) { + harness, _, _ := newGeneration(t) + before := harness.state.GetValue() + for _, request := range []struct { + operation string + input any + }{ + {runOperation, nil}, + {runOperation, RunJobRequest{}}, + {commitCheckpointOperation, CommitCheckpointRequest{ScannedInstances: 1}}, + {markAsCompletedOperation, nil}, + {markAsCompletedOperation, MarkAsCompletedRequest{}}, + {markAsFailedOperation, nil}, + {markAsFailedOperation, MarkAsFailedRequest{Error: "untokenized failure"}}, + } { + harness.call(request.operation, request.input) + assert.Equal(t, before, harness.state.GetValue(), request.operation) + assert.Empty(t, harness.actions, request.operation) + } + }) + + t.Run("missing state tokens reject every mutation", func(t *testing.T) { + for _, token := range []string{"", "any-token"} { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + state := harness.jobState() + state.RunToken = "" + payload, err := json.Marshal(state) + require.NoError(t, err) + harness.state = wrapperspb.String(string(payload)) + + for operation, input := range map[string]any{ + runOperation: RunJobRequest{RunToken: token}, + commitCheckpointOperation: CommitCheckpointRequest{ScannedInstances: 3, RunToken: token}, + markAsCompletedOperation: MarkAsCompletedRequest{RunToken: token}, + markAsFailedOperation: MarkAsFailedRequest{RunToken: token, Error: "stale failure"}, + } { + harness.call(operation, input) + assert.Equal(t, string(payload), harness.state.GetValue(), operation) + assert.Empty(t, harness.actions, operation) + } + } + }) +} + +func TestEntityDeletedJobDropsMutations(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + token := harness.runToken() + harness.call(deleteOperation, nil) + for operation, input := range map[string]any{ + runOperation: RunJobRequest{RunToken: token}, + commitCheckpointOperation: CommitCheckpointRequest{ScannedInstances: 3, RunToken: token}, + markAsCompletedOperation: MarkAsCompletedRequest{RunToken: token}, + markAsFailedOperation: MarkAsFailedRequest{RunToken: token, Error: "delayed failure"}, + } { + harness.call(operation, input) + assert.False(t, harness.hasState(), operation) + assert.Empty(t, harness.actions, operation) + } +} + +func TestEntityRejectsMalformedMutationRequests(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + token := harness.runToken() + before := harness.state.GetValue() + for operation, payload := range map[string]string{ + runOperation: fmt.Sprintf(`{"RunToken":%q,"RunToken":42}`, token), + commitCheckpointOperation: fmt.Sprintf(`{"RunToken":%q,"ScannedInstances":"bad"}`, token), + markAsCompletedOperation: fmt.Sprintf(`{"RunToken":%q,"RunToken":42}`, token), + markAsFailedOperation: fmt.Sprintf(`{"RunToken":%q,"Error":42}`, token), + } { + _, err := harness.tryCallRaw(operation, wrapperspb.String(payload)) + assertEntityErrorType(t, err, validationErrorType) + assert.Equal(t, before, harness.state.GetValue(), operation) + assert.Empty(t, harness.actions, operation) + } +} + +// TestEntityRunTokenTravelsToTheOrchestration keeps the generation the entity +// minted and the generation the orchestration runs under in sync. +func TestEntityRunTokenTravelsToTheOrchestration(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + token := harness.jobState().RunToken + require.NotEmpty(t, token) + + require.Len(t, harness.actions, 1) + signal := harness.actions[0].GetSendSignal() + require.NotNil(t, signal) + var runRequest RunJobRequest + require.NoError(t, json.Unmarshal([]byte(signal.GetInput().GetValue()), &runRequest)) + assert.Equal(t, token, runRequest.RunToken) + + harness.call(runOperation, runRequest) + require.Len(t, harness.actions, 1) + start := harness.actions[0].GetStartNewOrchestration() + require.NotNil(t, start) + var request ExportJobRunRequest + require.NoError(t, json.Unmarshal([]byte(start.GetInput().GetValue()), &request)) + assert.Equal(t, token, request.RunToken) + assert.False(t, request.ContinuedExecution) +} + +// TestEntityCreationToleratesBoundedClockSkew pins the documented tolerance the +// entity applies to a batch window's upper bound. A client validates strictly +// against its own clock, so a worker running slightly behind must not reject a +// window that client accepted. +func TestEntityCreationToleratesBoundedClockSkew(t *testing.T) { + windowEnd := time.Now().UTC().Add(MaxCreationClockSkew / 2) + options := JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: windowEnd, + Destination: &ExportDestination{Container: "test-container"}, + } + + // The client is strict: an upper bound ahead of its clock is rejected. + require.ErrorIs(t, options.Validate(), ErrValidation) + + // The entity absorbs the skew so a job the client accepted on a slightly + // faster clock still activates. + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, options) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + + t.Run("beyond the documented skew it is still rejected", func(t *testing.T) { + beyond := options + beyond.CompletedTimeTo = time.Now().UTC().Add(2 * MaxCreationClockSkew) + rejected := newEntityHarness(t, "job-1") + _, err := rejected.tryCall(createOperation, beyond) + require.Error(t, err) + assertEntityErrorType(t, err, validationErrorType) + assert.Contains(t, err.Error(), "cannot be in the future") + }) +} + +func TestEntityDelete(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + harness.call(runOperation, RunJobRequest{RunToken: harness.runToken()}) + require.True(t, harness.hasState()) + instanceID := harness.jobState().OrchestratorInstanceID + require.NotEmpty(t, instanceID) + + var deletedID string + require.NoError(t, json.Unmarshal([]byte(harness.call(deleteOperation, nil)), &deletedID)) + assert.Equal(t, instanceID, deletedID) + assert.False(t, harness.hasState()) + + // Deleting an already-deleted job is a no-op rather than an error. + require.NoError(t, json.Unmarshal([]byte(harness.call(deleteOperation, nil)), &deletedID)) + assert.Empty(t, deletedID) + assert.False(t, harness.hasState()) + + // A deleted job can be created again from scratch. + harness.call(createOperation, batchOptions("job-1")) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + replacementID := harness.jobState().OrchestratorInstanceID + require.NotEmpty(t, replacementID) + assert.NotEqual(t, instanceID, replacementID) + // A new Delete after recreation removes that new generation, not the old one. + require.NoError(t, json.Unmarshal([]byte(harness.call(deleteOperation, nil)), &deletedID)) + assert.Equal(t, replacementID, deletedID) + assert.False(t, harness.hasState()) +} + +func TestEntityDeleteBeforeRunDropsDelayedSignal(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call(createOperation, batchOptions("job-1")) + var delayed RunJobRequest + require.NoError(t, json.Unmarshal([]byte(harness.actions[0].GetSendSignal().GetInput().GetValue()), &delayed)) + harness.call(deleteOperation, nil) + + harness.call(runOperation, delayed) + assert.False(t, harness.hasState()) + assert.Empty(t, harness.actions) + + harness.call(createOperation, batchOptions("job-1")) + current := harness.state.GetValue() + harness.call(runOperation, delayed) + assert.Equal(t, current, harness.state.GetValue()) + assert.Empty(t, harness.actions) +} + +func TestEntityUnknownOperation(t *testing.T) { + harness := newEntityHarness(t, "job-1") + _, err := harness.tryCall("NotAnOperation", nil) + require.Error(t, err) + assertEntityErrorType(t, err, validationErrorType) + assert.Contains(t, err.Error(), `does not support operation "NotAnOperation"`) +} + +// TestEntityOperationNamesAreCaseInsensitive keeps the entity reachable from +// SDKs that normalize operation names differently. +func TestEntityOperationNamesAreCaseInsensitive(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.call("create", batchOptions("job-1")) + assert.Equal(t, ExportJobStatusActive, harness.jobState().Status) + harness.call("MARKASCOMPLETED", MarkAsCompletedRequest{RunToken: harness.runToken()}) + assert.Equal(t, ExportJobStatusCompleted, harness.jobState().Status) + harness.call("DELETE", nil) + assert.False(t, harness.hasState()) +} + +// TestEntityRejectsCorruptState surfaces unreadable state instead of silently +// starting a second export from a blank slate. +func TestEntityRejectsCorruptState(t *testing.T) { + harness := newEntityHarness(t, "job-1") + harness.state = wrapperspb.String("not json") + _, err := harness.tryCall(getOperation, nil) + require.Error(t, err) + assert.Contains(t, err.Error(), "failed to deserialize export job state") +} diff --git a/exporthistory/errors.go b/exporthistory/errors.go new file mode 100644 index 00000000..28f01a8f --- /dev/null +++ b/exporthistory/errors.go @@ -0,0 +1,139 @@ +package exporthistory + +import ( + "errors" + "fmt" + + "github.com/microsoft/durabletask-go/api" +) + +var ( + // ErrJobNotFound identifies attempts to read a missing export job. + ErrJobNotFound = errors.New("export job not found") + // ErrJobInvalidTransition identifies an invalid export job state change. + ErrJobInvalidTransition = errors.New("invalid export job state transition") + // ErrValidation identifies invalid export job configuration or client input. + ErrValidation = errors.New("invalid export job configuration") + // ErrJobOperationFailed identifies a failed export job mutation. + ErrJobOperationFailed = errors.New("export job operation failed") +) + +// Stable cross-language error types so a failure raised inside the entity keeps +// its identity when a client reconstructs it from orchestration failure details. +const ( + validationErrorType api.ErrorType = "ExportJobClientValidationException" + invalidTransitionErrorType api.ErrorType = "ExportJobInvalidTransitionException" + notFoundErrorType api.ErrorType = "ExportJobNotFoundException" + operationFailedErrorType api.ErrorType = "ExportJobOperationFailed" +) + +// ValidationError reports invalid export job options or configuration. +type ValidationError struct { + JobID string + Message string +} + +func (e *ValidationError) Error() string { + if e.JobID == "" { + return "export job validation failed: " + e.Message + } + return fmt.Sprintf("validation failed for export job %q: %s", e.JobID, e.Message) +} + +func (e *ValidationError) Unwrap() error { return ErrValidation } + +func (*ValidationError) DurableTaskErrorType() api.ErrorType { return validationErrorType } + +func (e *ValidationError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"jobId": e.JobID, "message": e.Message} +} + +// NotFoundError reports an export job that does not exist. +type NotFoundError struct{ JobID string } + +func (e *NotFoundError) Error() string { + return fmt.Sprintf("export history job with ID %q was not found", e.JobID) +} + +func (e *NotFoundError) Unwrap() error { return ErrJobNotFound } + +func (*NotFoundError) DurableTaskErrorType() api.ErrorType { return notFoundErrorType } + +func (e *NotFoundError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"jobId": e.JobID} +} + +// InvalidTransitionError reports a lifecycle transition the job does not allow. +type InvalidTransitionError struct { + JobID string + From ExportJobStatus + To ExportJobStatus + Operation string +} + +func (e *InvalidTransitionError) Error() string { + return fmt.Sprintf( + "invalid state transition attempted for export job %q: cannot transition from %s to %s during %s operation", + e.JobID, e.From, e.To, e.Operation) +} + +func (e *InvalidTransitionError) Unwrap() error { return ErrJobInvalidTransition } + +func (*InvalidTransitionError) DurableTaskErrorType() api.ErrorType { + return invalidTransitionErrorType +} + +func (e *InvalidTransitionError) DurableTaskErrorProperties() map[string]any { + return map[string]any{ + "jobId": e.JobID, + "from": int(e.From), + "to": int(e.To), + "operation": e.Operation, + } +} + +// OperationError reports an export job mutation orchestration that did not +// complete successfully and whose failure could not be mapped to a more +// specific typed error. +type OperationError struct { + JobID string + Operation string + RuntimeStatus api.OrchestrationStatus + FailureDetails *api.FailureDetails +} + +func (e *OperationError) Error() string { + message := "" + if e.FailureDetails != nil { + message = e.FailureDetails.ErrorMessage + } + if message == "" { + return fmt.Sprintf("export job %q operation %s failed with status %v", + e.JobID, e.Operation, e.RuntimeStatus) + } + return fmt.Sprintf("export job %q operation %s failed: %s", e.JobID, e.Operation, message) +} + +func (e *OperationError) Unwrap() error { return ErrJobOperationFailed } + +func (*OperationError) DurableTaskErrorType() api.ErrorType { return operationFailedErrorType } + +// isValidTransition reports whether operation may move a job from `from` to `to`. +// Create is deliberately permitted from every terminal status so a failed or +// completed job can be recreated in place, but not from Active, which would +// start a second orchestration for a job that is still running. +func isValidTransition(operation string, from, to ExportJobStatus) bool { + switch operation { + case createOperation: + return to == ExportJobStatusActive && + (from == ExportJobStatusPending || + from == ExportJobStatusFailed || + from == ExportJobStatusCompleted) + case markAsCompletedOperation: + return from == ExportJobStatusActive && to == ExportJobStatusCompleted + case markAsFailedOperation: + return from == ExportJobStatusActive && to == ExportJobStatusFailed + default: + return false + } +} diff --git a/exporthistory/errors_test.go b/exporthistory/errors_test.go new file mode 100644 index 00000000..1376b3fd --- /dev/null +++ b/exporthistory/errors_test.go @@ -0,0 +1,263 @@ +package exporthistory + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestIsValidTransition ports the upstream ExportJobTransitionsTests matrix. +func TestIsValidTransition(t *testing.T) { + tests := []struct { + operation string + from ExportJobStatus + to ExportJobStatus + expected bool + }{ + {createOperation, ExportJobStatusPending, ExportJobStatusActive, true}, + {createOperation, ExportJobStatusFailed, ExportJobStatusActive, true}, + {createOperation, ExportJobStatusCompleted, ExportJobStatusActive, true}, + {createOperation, ExportJobStatusActive, ExportJobStatusActive, false}, + {createOperation, ExportJobStatusActive, ExportJobStatusFailed, false}, + {createOperation, ExportJobStatusActive, ExportJobStatusCompleted, false}, + {createOperation, ExportJobStatusPending, ExportJobStatusCompleted, false}, + {createOperation, ExportJobStatusPending, ExportJobStatusFailed, false}, + + {markAsCompletedOperation, ExportJobStatusActive, ExportJobStatusCompleted, true}, + {markAsCompletedOperation, ExportJobStatusPending, ExportJobStatusCompleted, false}, + {markAsCompletedOperation, ExportJobStatusFailed, ExportJobStatusCompleted, false}, + {markAsCompletedOperation, ExportJobStatusCompleted, ExportJobStatusCompleted, false}, + {markAsCompletedOperation, ExportJobStatusActive, ExportJobStatusFailed, false}, + + {markAsFailedOperation, ExportJobStatusActive, ExportJobStatusFailed, true}, + {markAsFailedOperation, ExportJobStatusPending, ExportJobStatusFailed, false}, + {markAsFailedOperation, ExportJobStatusFailed, ExportJobStatusFailed, false}, + {markAsFailedOperation, ExportJobStatusCompleted, ExportJobStatusFailed, false}, + {markAsFailedOperation, ExportJobStatusActive, ExportJobStatusCompleted, false}, + + {"UnknownOperation", ExportJobStatusPending, ExportJobStatusActive, false}, + {getOperation, ExportJobStatusActive, ExportJobStatusActive, false}, + {deleteOperation, ExportJobStatusActive, ExportJobStatusPending, false}, + } + for _, test := range tests { + name := fmt.Sprintf("%s/%s->%s", test.operation, test.from, test.to) + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.expected, isValidTransition(test.operation, test.from, test.to)) + }) + } +} + +func TestValidationError(t *testing.T) { + err := &ValidationError{JobID: "job-1", Message: "bad input"} + assert.Equal(t, `validation failed for export job "job-1": bad input`, err.Error()) + assert.ErrorIs(t, err, ErrValidation) + assert.Equal(t, validationErrorType, err.DurableTaskErrorType()) + assert.Equal(t, map[string]any{"jobId": "job-1", "message": "bad input"}, err.DurableTaskErrorProperties()) + + bare := &ValidationError{Message: "bad input"} + assert.Equal(t, "export job validation failed: bad input", bare.Error()) + + var target *ValidationError + require.True(t, errors.As(fmt.Errorf("wrapped: %w", err), &target)) + assert.Equal(t, "job-1", target.JobID) +} + +func TestNotFoundError(t *testing.T) { + err := &NotFoundError{JobID: "missing"} + assert.Equal(t, `export history job with ID "missing" was not found`, err.Error()) + assert.ErrorIs(t, err, ErrJobNotFound) + assert.Equal(t, notFoundErrorType, err.DurableTaskErrorType()) + assert.Equal(t, map[string]any{"jobId": "missing"}, err.DurableTaskErrorProperties()) +} + +func TestInvalidTransitionError(t *testing.T) { + err := &InvalidTransitionError{ + JobID: "job-1", + From: ExportJobStatusActive, + To: ExportJobStatusActive, + Operation: createOperation, + } + assert.Equal(t, + `invalid state transition attempted for export job "job-1": `+ + `cannot transition from Active to Active during Create operation`, + err.Error()) + assert.ErrorIs(t, err, ErrJobInvalidTransition) + assert.Equal(t, invalidTransitionErrorType, err.DurableTaskErrorType()) + assert.Equal(t, map[string]any{ + "jobId": "job-1", + "from": int(ExportJobStatusActive), + "to": int(ExportJobStatusActive), + "operation": createOperation, + }, err.DurableTaskErrorProperties()) +} + +func TestOperationError(t *testing.T) { + err := &OperationError{ + JobID: "job-1", + Operation: createOperation, + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + } + assert.Contains(t, err.Error(), `export job "job-1" operation Create failed with status`) + assert.ErrorIs(t, err, ErrJobOperationFailed) + assert.Equal(t, operationFailedErrorType, err.DurableTaskErrorType()) + + withDetails := &OperationError{ + JobID: "job-1", + Operation: deleteOperation, + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorMessage: "boom"}, + } + assert.Equal(t, `export job "job-1" operation Delete failed: boom`, withDetails.Error()) +} + +// TestFindFailureMatchesNamespacedErrorTypes keeps typed errors recoverable when +// another SDK reports the fully qualified .NET exception name. +func TestFindFailureMatchesNamespacedErrorTypes(t *testing.T) { + details := &api.FailureDetails{ + ErrorType: "System.InvalidOperationException", + ErrorMessage: "outer", + InnerFailure: &api.FailureDetails{ + ErrorType: "Microsoft.DurableTask.ExportHistory.ExportJobInvalidTransitionException", + ErrorMessage: "inner", + }, + } + found := findFailure(details, invalidTransitionErrorType) + require.NotNil(t, found) + assert.Equal(t, "inner", found.ErrorMessage) + assert.Nil(t, findFailure(details, notFoundErrorType)) + assert.Nil(t, findFailure(nil, notFoundErrorType)) +} + +func TestOperationFailureReconstructsTypedErrors(t *testing.T) { + t.Run("invalid transition", func(t *testing.T) { + err := operationFailure("job-1", createOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: invalidTransitionErrorType, + Properties: map[string]any{ + "jobId": "job-1", + "from": float64(ExportJobStatusActive), + "to": float64(ExportJobStatusActive), + "operation": createOperation, + }, + }, + }) + var transition *InvalidTransitionError + require.ErrorAs(t, err, &transition) + assert.Equal(t, ExportJobStatusActive, transition.From) + assert.Equal(t, ExportJobStatusActive, transition.To) + assert.Equal(t, createOperation, transition.Operation) + }) + + t.Run("validation", func(t *testing.T) { + err := operationFailure("job-1", createOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ + ErrorType: validationErrorType, + ErrorMessage: "fallback message", + Properties: map[string]any{"message": "explicit message"}, + }, + }) + var validation *ValidationError + require.ErrorAs(t, err, &validation) + assert.Equal(t, "explicit message", validation.Message) + }) + + t.Run("validation falls back to the failure message", func(t *testing.T) { + err := operationFailure("job-1", createOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorType: validationErrorType, ErrorMessage: "fallback"}, + }) + var validation *ValidationError + require.ErrorAs(t, err, &validation) + assert.Equal(t, "fallback", validation.Message) + }) + + t.Run("not found", func(t *testing.T) { + err := operationFailure("job-1", getOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &api.FailureDetails{ErrorType: notFoundErrorType}, + }) + var notFound *NotFoundError + require.ErrorAs(t, err, ¬Found) + assert.Equal(t, "job-1", notFound.JobID) + }) + + t.Run("unmapped failures stay generic", func(t *testing.T) { + err := operationFailure("job-1", createOperation, &api.OrchestrationMetadata{ + RuntimeStatus: api.RUNTIME_STATUS_TERMINATED, + FailureDetails: &api.FailureDetails{ErrorType: "Contoso.Boom", ErrorMessage: "boom"}, + }) + var operation *OperationError + require.ErrorAs(t, err, &operation) + assert.Equal(t, api.RUNTIME_STATUS_TERMINATED, operation.RuntimeStatus) + }) + + t.Run("missing metadata", func(t *testing.T) { + err := operationFailure("job-1", createOperation, nil) + var operation *OperationError + require.ErrorAs(t, err, &operation) + assert.Equal(t, "job-1", operation.JobID) + }) +} + +func TestPropertyReaders(t *testing.T) { + properties := map[string]any{ + "int": 7, + "int32": int32(8), + "int64": int64(9), + "float": float64(10), + "string": "value", + "empty": "", + "unknown": []int{1}, + } + assert.Equal(t, 7, intProperty(properties, "int")) + assert.Equal(t, 8, intProperty(properties, "int32")) + assert.Equal(t, 9, intProperty(properties, "int64")) + assert.Equal(t, 10, intProperty(properties, "float")) + assert.Equal(t, 0, intProperty(properties, "unknown")) + assert.Equal(t, 0, intProperty(properties, "absent")) + assert.Equal(t, 0, intProperty(nil, "absent")) + + assert.Equal(t, "value", stringProperty(properties, "string", "fallback")) + assert.Equal(t, "fallback", stringProperty(properties, "empty", "fallback")) + assert.Equal(t, "fallback", stringProperty(properties, "absent", "fallback")) + assert.Equal(t, "fallback", stringProperty(nil, "absent", "fallback")) +} + +func TestWithJobIDOnlyFillsMissingIdentifiers(t *testing.T) { + filled := withJobID(&ValidationError{Message: "bad"}, "job-1") + var validation *ValidationError + require.ErrorAs(t, filled, &validation) + assert.Equal(t, "job-1", validation.JobID) + + preserved := withJobID(&ValidationError{JobID: "original", Message: "bad"}, "job-1") + require.ErrorAs(t, preserved, &validation) + assert.Equal(t, "original", validation.JobID) + + other := errors.New("other") + assert.Equal(t, other, withJobID(other, "job-1")) +} + +// TestSummarizeFailures pins the single bounded summarizer the entity and the +// orchestration share, so both report the same detail under the same limit. +func TestSummarizeFailures(t *testing.T) { + assert.Equal(t, "no failure details available", summarizeFailures(nil)) + assert.Equal(t, "no failure details available", summarizeFailures([]ExportFailure{})) + + failures := make([]ExportFailure, 0, 12) + for i := 0; i < 12; i++ { + failures = append(failures, ExportFailure{InstanceID: fmt.Sprintf("i%d", i), Reason: "boom"}) + } + summary := summarizeFailures(failures) + assert.Contains(t, summary, "InstanceId: i0, Reason: boom") + assert.Contains(t, summary, "InstanceId: i9, Reason: boom") + assert.NotContains(t, summary, "InstanceId: i10") + assert.Contains(t, summary, "and 2 more failures") + assert.Equal(t, maxSummarizedFailures, strings.Count(summary, "InstanceId: ")) +} diff --git a/exporthistory/lifecycle_test.go b/exporthistory/lifecycle_test.go new file mode 100644 index 00000000..f93d941f --- /dev/null +++ b/exporthistory/lifecycle_test.go @@ -0,0 +1,351 @@ +package exporthistory + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/gob" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// lifecycleBackend executes client operations through both production executors. +// Hooks control the interleaving without a scheduler, network, or sleeps. +type lifecycleBackend struct { + *fakeBackend + entity *entityHarness + completions map[api.InstanceID]*api.OrchestrationMetadata + beforeCreate func() + afterOperation func(ExportJobOperationRequest, *api.OrchestrationMetadata) +} + +func newLifecycleBackend(entity *entityHarness) *lifecycleBackend { + return &lifecycleBackend{ + fakeBackend: newFakeBackend(), + entity: entity, + completions: map[api.InstanceID]*api.OrchestrationMetadata{}, + } +} + +func (b *lifecycleBackend) ScheduleNewOrchestration( + _ context.Context, + name string, + options ...api.NewOrchestrationOptions, +) (api.InstanceID, error) { + request := &protos.CreateInstanceRequest{} + for _, option := range options { + if err := option(request, b.entity.converter); err != nil { + return "", err + } + } + var operation ExportJobOperationRequest + if err := b.entity.converter.Deserialize(request.GetInput().GetValue(), &operation); err != nil { + return "", err + } + if strings.EqualFold(operation.OperationName, createOperation) && b.beforeCreate != nil { + b.beforeCreate() + } + + id := api.InstanceID(newCompactUUID()) + b.scheduled = append(b.scheduled, scheduledCall{name: name, request: operation, version: request.GetVersion().GetValue()}) + driver := newOrchestrationDriver(b.entity.t, newExportRegistry(b.entity.t), name, id, operation, b.entity.converter) + driver.turn() + called := driver.pendingEntityCall(operation.OperationName) + output, err := b.entity.tryCallRaw(operation.OperationName, called.GetInput()) + if err != nil { + var entityErr entityFailure + require.ErrorAs(b.entity.t, err, &entityErr) + driver.failEntityCallWithDetails(operation.OperationName, entityErr.details) + } else { + var rawOutput *wrapperspb.StringValue + if output != "" { + rawOutput = wrapperspb.String(output) + } + driver.completeEntityCallOutput(operation.OperationName, rawOutput) + } + driver.nextTurn() + driver.turn() + require.NotNil(b.entity.t, driver.completion) + completion := driver.completion + metadata := &api.OrchestrationMetadata{ + InstanceID: id, + RuntimeStatus: completion.GetOrchestrationStatus(), + SerializedOutput: completion.GetResult().GetValue(), + FailureDetails: failure.FromProto(completion.GetFailureDetails()), + Converter: b.entity.converter, + } + b.completions[id] = metadata + if b.afterOperation != nil { + b.afterOperation(operation, metadata) + } + return id, nil +} + +func (b *lifecycleBackend) WaitForOrchestrationCompletion( + ctx context.Context, + id api.InstanceID, + options ...api.FetchOrchestrationMetadataOptions, +) (*api.OrchestrationMetadata, error) { + metadata, err := b.fakeBackend.WaitForOrchestrationCompletion(ctx, id, options...) + if operation, ok := b.completions[id]; ok { + return operation, nil + } + return metadata, err +} + +func (b *lifecycleBackend) GetEntity( + ctx context.Context, + id api.EntityID, + options ...api.GetEntityOptions, +) (*api.EntityMetadata, error) { + _, err := b.fakeBackend.GetEntity(ctx, id, options...) + if err != nil { + return nil, err + } + var snapshot *api.EntityMetadata + if state := b.entity.jobState(); state != nil { + snapshot = entityMetadata(b.entity.t, b.entity.jobID, *state) + } + // Pause after taking the snapshot if a pre-Create read is reintroduced. + // Otherwise ScheduleNewOrchestration pauses before submitting Create. + if b.beforeCreate != nil { + b.beforeCreate() + } + return snapshot, nil +} + +func TestLifecycleConcurrentCreateDoesNotCleanUpTheWinner(t *testing.T) { + for _, terminal := range []string{markAsCompletedOperation, markAsFailedOperation} { + t.Run(terminal, func(t *testing.T) { + entity := newEntityHarness(t, "job-1") + entity.call(createOperation, batchOptions("job-1")) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + previousID := entity.jobState().OrchestratorInstanceID + entity.call(terminal, MarkAsFailedRequest{RunToken: entity.runToken()}) + + hubA, hubB := newLifecycleBackend(entity), newLifecycleBackend(entity) + jobA, err := newTestClient(t, hubA, ClientOptions{ContainerName: "container"}).JobClient("job-1") + require.NoError(t, err) + jobB, err := newTestClient(t, hubB, ClientOptions{ContainerName: "container"}).JobClient("job-1") + require.NoError(t, err) + + paused, resume := make(chan struct{}), make(chan struct{}) + var pauseOnce, resumeOnce sync.Once + hubB.beforeCreate = func() { + pauseOnce.Do(func() { + close(paused) + <-resume + }) + } + defer resumeOnce.Do(func() { close(resume) }) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + done := make(chan error, 1) + go func() { + defer close(done) + done <- jobB.Create(ctx, batchOptions("job-1")) + }() + select { + case <-paused: + case <-ctx.Done(): + t.Fatal("second Create did not reach its interleaving barrier") + } + + require.NoError(t, jobA.Create(ctx, batchOptions("job-1"))) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + current := *entity.jobState() + currentID := api.InstanceID(current.OrchestratorInstanceID) + require.NotEmpty(t, currentID) + resumeOnce.Do(func() { close(resume) }) + select { + case err := <-done: + require.ErrorIs(t, err, ErrJobInvalidTransition) + case <-ctx.Done(): + t.Fatal("second Create did not finish") + } + + assert.Equal(t, current, *entity.jobState()) + assert.NotEqual(t, previousID, string(currentID)) + assert.NotContains(t, hubB.terminated, currentID, "the losing Create must not terminate the winner") + assert.NotContains(t, hubB.purged, currentID, "the losing Create must not purge the winner") + assert.Empty(t, hubA.terminated) + assert.Empty(t, hubA.purged) + assert.Empty(t, hubB.terminated) + assert.Empty(t, hubB.purged) + }) + } +} + +func TestLifecycleDeleteCleanupUsesOnlyTheCapturedGeneration(t *testing.T) { + entity := newEntityHarness(t, "job-1") + entity.call(createOperation, batchOptions("job-1")) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + removedID := api.InstanceID(entity.jobState().OrchestratorInstanceID) + + deletingHub, creatingHub := newLifecycleBackend(entity), newLifecycleBackend(entity) + deletingJob, err := newTestClient(t, deletingHub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + creatingJob, err := newTestClient(t, creatingHub, ClientOptions{ContainerName: "container"}).JobClient("job-1") + require.NoError(t, err) + var replacement ExportJobState + deletingHub.afterOperation = func(request ExportJobOperationRequest, metadata *api.OrchestrationMetadata) { + require.Equal(t, deleteOperation, request.OperationName) + require.False(t, entity.hasState()) + assert.Empty(t, deletingHub.terminated, "the entity completes before management cleanup begins") + var capturedID string + require.NoError(t, metadata.ReadOutput(&capturedID)) + assert.Equal(t, string(removedID), capturedID) + + require.NoError(t, creatingJob.Create(context.Background(), batchOptions("job-1"))) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + replacement = *entity.jobState() + } + require.NoError(t, deletingJob.Delete(context.Background())) + assert.Equal(t, replacement, *entity.jobState()) + assert.NotEqual(t, string(removedID), replacement.OrchestratorInstanceID) + assert.Equal(t, []api.InstanceID{removedID}, deletingHub.terminated) + assert.Equal(t, []api.InstanceID{removedID}, deletingHub.purged) + assert.Empty(t, deletingHub.entityReads, "Delete must not Describe the replacement") + + // Retrying cleanup of the captured ID after it is gone is harmless. A new + // Delete operation, in contrast, would legitimately remove the replacement. + deletingHub.terminateErr = api.ErrInstanceNotFound + require.NoError(t, deletingJob.terminateAndPurgeOrchestration(context.Background(), removedID)) + assert.Equal(t, replacement, *entity.jobState()) + assert.Equal(t, []api.InstanceID{removedID, removedID}, deletingHub.terminated) + assert.Equal(t, []api.InstanceID{removedID}, deletingHub.purged) +} + +// Gob is non-JSON and type-sensitive: a concrete string cannot be decoded into +// *any. A prefixed JSON converter would not catch an untyped Await target. +type lifecycleGobConverter struct{} + +func (lifecycleGobConverter) Serialize(value any) (string, error) { + var buffer bytes.Buffer + if err := gob.NewEncoder(&buffer).Encode(value); err != nil { + return "", err + } + return base64.RawStdEncoding.EncodeToString(buffer.Bytes()), nil +} + +func (lifecycleGobConverter) Deserialize(payload string, target any) error { + data, err := base64.RawStdEncoding.DecodeString(payload) + if err != nil { + return err + } + return gob.NewDecoder(bytes.NewReader(data)).Decode(target) +} + +func TestLifecycleDeleteResultRoundTripWithTypeSensitiveConverter(t *testing.T) { + converter := lifecycleGobConverter{} + encoded, err := converter.Serialize("a-run-id") + require.NoError(t, err) + var untyped any + require.Error(t, converter.Deserialize(encoded, &untyped), "this converter must reject *any string results") + + entity := newEntityHarness(t, "job-1", converter) + entity.call(createOperation, batchOptions("job-1")) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + removedID := api.InstanceID(entity.jobState().OrchestratorInstanceID) + require.NotEmpty(t, removedID) + hub := newLifecycleBackend(entity) + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Delete(context.Background())) + assert.False(t, entity.hasState()) + assert.Equal(t, []api.InstanceID{removedID}, hub.terminated) + assert.Equal(t, []api.InstanceID{removedID}, hub.purged) + + require.NoError(t, job.Delete(context.Background()), "an absent job still returns a typed empty string") + assert.False(t, entity.hasState()) + assert.Equal(t, []api.InstanceID{removedID}, hub.terminated) + assert.Equal(t, []api.InstanceID{removedID}, hub.purged) + assert.Empty(t, hub.entityReads) +} + +func TestLifecycleDeleteBeforeRunCapturesTheReservedID(t *testing.T) { + entity := newEntityHarness(t, "job-1") + entity.call(createOperation, batchOptions("job-1")) + removedID := api.InstanceID(entity.jobState().OrchestratorInstanceID) + require.NotEmpty(t, removedID, "Create must reserve the ID before Run is delivered") + delayedRun := entity.actions[0].GetSendSignal().GetInput() + hub := newLifecycleBackend(entity) + hub.terminateErr = api.ErrInstanceNotFound + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + + require.NoError(t, job.Delete(context.Background())) + assert.Equal(t, []api.InstanceID{removedID}, hub.terminated) + assert.Empty(t, hub.purged) + _, err = entity.tryCallRaw(runOperation, delayedRun) + require.NoError(t, err) + assert.False(t, entity.hasState()) + assert.Empty(t, entity.actions) + + require.NoError(t, job.Delete(context.Background())) + assert.Equal(t, []api.InstanceID{removedID}, hub.terminated) +} + +func TestLifecycleDelayedStartCannotExportTheReplacement(t *testing.T) { + entity := newEntityHarness(t, "job-1") + entity.call(createOperation, batchOptions("job-1")) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + delayedStart := entity.actions[0].GetStartNewOrchestration() + require.NotNil(t, delayedStart) + var delayedInput ExportJobRunRequest + require.NoError(t, entity.converter.Deserialize(delayedStart.GetInput().GetValue(), &delayedInput)) + + hub := newLifecycleBackend(entity) + hub.terminateErr = api.ErrInstanceNotFound // the emitted Start has not arrived + job, err := newTestClient(t, hub, ClientOptions{ContainerName: "container"}).JobClient("job-1") + require.NoError(t, err) + require.NoError(t, job.Delete(context.Background())) + require.NoError(t, job.Create(context.Background(), batchOptions("job-1"))) + entity.call(runOperation, RunJobRequest{RunToken: entity.runToken()}) + replacement := *entity.jobState() + assert.NotEqual(t, delayedStart.GetInstanceId(), replacement.OrchestratorInstanceID) + + driver := newOrchestrationDriver(t, newExportRegistry(t), delayedStart.GetName(), + api.InstanceID(delayedStart.GetInstanceId()), delayedInput) + driver.turn() + output, err := entity.tryCallRaw(getOperation, driver.pendingEntityCall(getOperation).GetInput()) + require.NoError(t, err) + driver.completeEntityCallOutput(getOperation, wrapperspb.String(output)) + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, api.RUNTIME_STATUS_COMPLETED, driver.completion.GetOrchestrationStatus()) + assert.Empty(t, driver.pendingTasks, "a stale Start must never list or export the replacement's instances") + assert.Empty(t, driver.pendingEntities, "a stale Start must not mutate the replacement") + assert.Equal(t, replacement, *entity.jobState()) +} + +func TestLifecycleDeleteDecodeFailureNeverTargetsTheReplacement(t *testing.T) { + entity := newEntityHarness(t, "job-1") + entity.call(createOperation, batchOptions("job-1")) + hub := newLifecycleBackend(entity) + job, err := newTestClient(t, hub, ClientOptions{}).JobClient("job-1") + require.NoError(t, err) + hub.afterOperation = func(_ ExportJobOperationRequest, metadata *api.OrchestrationMetadata) { + require.False(t, entity.hasState()) + entity.call(createOperation, batchOptions("job-1")) + metadata.SerializedOutput = "invalid-output" + } + err = job.Delete(context.Background()) + require.Error(t, err) + assert.False(t, errors.Is(err, ErrJobNotFound)) + assert.Empty(t, hub.terminated) + assert.Empty(t, hub.purged) + assert.Empty(t, hub.entityReads) + assert.Equal(t, ExportJobStatusActive, entity.jobState().Status) +} diff --git a/exporthistory/models.go b/exporthistory/models.go new file mode 100644 index 00000000..68fc629b --- /dev/null +++ b/exporthistory/models.go @@ -0,0 +1,532 @@ +package exporthistory + +import ( + "encoding/json" + "fmt" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" +) + +const ( + // ExportJobEntityName is the system entity that owns export job state. + ExportJobEntityName = "ExportJob" + // ExportJobOrchestratorName is the system orchestrator that performs the export work. + ExportJobOrchestratorName = "ExportJobOrchestrator" + // ExecuteExportJobOperationOrchestratorName is the system orchestrator that + // invokes a single export job entity operation on behalf of a client. + ExecuteExportJobOperationOrchestratorName = "ExecuteExportJobOperationOrchestrator" + // ListTerminalInstancesActivityName lists a page of terminal instance IDs. + ListTerminalInstancesActivityName = "ListTerminalInstancesActivity" + // ExportInstanceHistoryActivityName exports one instance's history. + ExportInstanceHistoryActivityName = "ExportInstanceHistoryActivity" + + // OrchestratorInstanceIDPrefix prefixes each generation-specific export + // orchestration ID. Read ExportJobDescription.OrchestratorInstanceID for + // the current generation's ID. + OrchestratorInstanceIDPrefix = "ExportJob-" + + createOperation = "Create" + getOperation = "Get" + runOperation = "Run" + commitCheckpointOperation = "CommitCheckpoint" + markAsCompletedOperation = "MarkAsCompleted" + markAsFailedOperation = "MarkAsFailed" + deleteOperation = "Delete" +) + +// Defaults applied when a job creation option is not supplied. +const ( + DefaultMaxInstancesPerBatch = 100 + // MaxInstancesPerBatchLimit is the inclusive upper bound accepted for + // JobCreationOptions.MaxInstancesPerBatch. + MaxInstancesPerBatchLimit = 1000 + // DefaultMaxParallelExports bounds concurrent per-instance export activities. + DefaultMaxParallelExports = 32 + // DefaultSchemaVersion is the schema version stamped on exported objects. + DefaultSchemaVersion = "1.0" + // DefaultJobQueryPageSize is used when ExportJobQuery.PageSize is zero. + DefaultJobQueryPageSize = 100 +) + +const terminalStatusesValidationMessage = "export supports terminal orchestration statuses only; " + + "valid statuses are COMPLETED, FAILED, and TERMINATED" + +// EntityID returns the export job entity ID for jobID. +func EntityID(jobID string) api.EntityID { + return api.NewEntityID(ExportJobEntityName, jobID) +} + +// ExportMode selects how an export job consumes its filter window. +type ExportMode int + +const ( + // ExportModeBatch exports a fixed completion-time window and then completes. + ExportModeBatch ExportMode = 1 + // ExportModeContinuous tails terminal instances indefinitely. + ExportModeContinuous ExportMode = 2 +) + +func (m ExportMode) String() string { + switch m { + case ExportModeBatch: + return "Batch" + case ExportModeContinuous: + return "Continuous" + default: + return fmt.Sprintf("ExportMode(%d)", int(m)) + } +} + +// IsValid reports whether m is a mode the export job supports. +func (m ExportMode) IsValid() bool { + return m == ExportModeBatch || m == ExportModeContinuous +} + +// ExportJobStatus is the persisted lifecycle status of an export job. +type ExportJobStatus int + +const ( + // ExportJobStatusPending is the zero value: the entity has no configuration yet. + ExportJobStatusPending ExportJobStatus = iota + // ExportJobStatusActive means the job is running. + ExportJobStatusActive + // ExportJobStatusFailed means the job stopped because of an error. + ExportJobStatusFailed + // ExportJobStatusCompleted means the job exported its whole window. + ExportJobStatusCompleted +) + +func (s ExportJobStatus) String() string { + switch s { + case ExportJobStatusPending: + return "Pending" + case ExportJobStatusActive: + return "Active" + case ExportJobStatusFailed: + return "Failed" + case ExportJobStatusCompleted: + return "Completed" + default: + return fmt.Sprintf("ExportJobStatus(%d)", int(s)) + } +} + +// IsValid reports whether s is a persisted export job status. +func (s ExportJobStatus) IsValid() bool { + return s >= ExportJobStatusPending && s <= ExportJobStatusCompleted +} + +// ExportFormatKind selects the serialization of an exported history. +type ExportFormatKind int + +const ( + // ExportFormatJSONL writes one history event per line and gzip-compresses the object. + ExportFormatJSONL ExportFormatKind = iota + // ExportFormatJSON writes an uncompressed JSON array of history events. + ExportFormatJSON +) + +func (k ExportFormatKind) String() string { + switch k { + case ExportFormatJSONL: + return "Jsonl" + case ExportFormatJSON: + return "Json" + default: + return fmt.Sprintf("ExportFormatKind(%d)", int(k)) + } +} + +// IsValid reports whether k is a supported format kind. +func (k ExportFormatKind) IsValid() bool { + return k == ExportFormatJSONL || k == ExportFormatJSON +} + +// MarshalJSON writes the .NET-compatible string form of the format kind. +func (k ExportFormatKind) MarshalJSON() ([]byte, error) { + if !k.IsValid() { + return nil, fmt.Errorf("invalid export format kind %d", int(k)) + } + return json.Marshal(k.String()) +} + +// UnmarshalJSON accepts either the .NET string form or the numeric form. +func (k *ExportFormatKind) UnmarshalJSON(data []byte) error { + var name string + if err := json.Unmarshal(data, &name); err == nil { + switch strings.ToLower(name) { + case "jsonl": + *k = ExportFormatJSONL + return nil + case "json": + *k = ExportFormatJSON + return nil + default: + return fmt.Errorf("invalid export format kind %q", name) + } + } + var value int + if err := json.Unmarshal(data, &value); err != nil { + return fmt.Errorf("invalid export format kind: %w", err) + } + kind := ExportFormatKind(value) + if !kind.IsValid() { + return fmt.Errorf("invalid export format kind %d", value) + } + *k = kind + return nil +} + +// ExportFormat describes the serialization and schema of exported objects. +type ExportFormat struct { + Kind ExportFormatKind `json:"Kind"` + SchemaVersion string `json:"SchemaVersion"` +} + +// DefaultExportFormat is gzip-compressed JSONL at the default schema version. +func DefaultExportFormat() ExportFormat { + return ExportFormat{Kind: ExportFormatJSONL, SchemaVersion: DefaultSchemaVersion} +} + +// FileExtension returns the object suffix used for this format. +func (f ExportFormat) FileExtension() string { + if f.Kind == ExportFormatJSON { + return "json" + } + return "jsonl.gz" +} + +// ContentType returns the MIME type exported objects are stored under. +// +// JSONL objects are gzip-compressed and stored as opaque gzip files, so they +// use application/gzip and carry no content coding: the bytes a reader +// downloads are always the gzip stream the object name promises. +func (f ExportFormat) ContentType() string { + if f.Kind == ExportFormatJSON { + return "application/json" + } + return "application/gzip" +} + +// ExportDestination is the blob container and optional path prefix that +// receives exported objects. +type ExportDestination struct { + Container string `json:"Container"` + Prefix string `json:"Prefix,omitempty"` +} + +// Validate reports whether the destination can receive exported objects. +func (d ExportDestination) Validate() error { + if strings.TrimSpace(d.Container) == "" { + return &ValidationError{Message: "export destination container is required"} + } + if !validBlobContainerName(d.Container) { + return &ValidationError{ + Message: fmt.Sprintf("export destination container %q is not a valid Azure Blob container name", d.Container), + } + } + if err := validateBlobPrefix(d.Prefix); err != nil { + return err + } + return nil +} + +// BlobPath joins the destination prefix with name using a single separator. +func (d ExportDestination) BlobPath(name string) string { + prefix := strings.TrimRight(d.Prefix, "/") + if prefix == "" { + return name + } + return prefix + "/" + name +} + +// ExportFilter selects the orchestration instances an export job reads. +// Only terminal runtime statuses are supported. A zero CompletedTimeTo means +// the window has no upper bound. +type ExportFilter struct { + CompletedTimeFrom time.Time + CompletedTimeTo time.Time + RuntimeStatus []api.OrchestrationStatus +} + +// filterJSON is the .NET-compatible wire shape of ExportFilter. An unset upper +// bound must serialize as null rather than as the zero instant. +type filterJSON struct { + CompletedTimeFrom time.Time `json:"CompletedTimeFrom"` + CompletedTimeTo *time.Time `json:"CompletedTimeTo"` + RuntimeStatus []api.OrchestrationStatus `json:"RuntimeStatus,omitempty"` +} + +func (f ExportFilter) MarshalJSON() ([]byte, error) { + return json.Marshal(filterJSON{ + CompletedTimeFrom: f.CompletedTimeFrom, + CompletedTimeTo: optionalTime(f.CompletedTimeTo), + RuntimeStatus: f.RuntimeStatus, + }) +} + +func (f *ExportFilter) UnmarshalJSON(data []byte) error { + var decoded filterJSON + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *f = ExportFilter{ + CompletedTimeFrom: decoded.CompletedTimeFrom, + RuntimeStatus: decoded.RuntimeStatus, + } + if decoded.CompletedTimeTo != nil { + f.CompletedTimeTo = *decoded.CompletedTimeTo + } + return nil +} + +func optionalTime(value time.Time) *time.Time { + if value.IsZero() { + return nil + } + return &value +} + +// ExportCheckpoint records the durable pagination cursor of an export job. +type ExportCheckpoint struct { + LastInstanceKey string `json:"LastInstanceKey,omitempty"` +} + +// ExportFailure describes one instance that could not be exported. +type ExportFailure struct { + InstanceID string `json:"InstanceId"` + Reason string `json:"Reason"` + AttemptCount int `json:"AttemptCount"` + LastAttempt time.Time `json:"LastAttempt"` +} + +// ExportJobConfiguration is the durable configuration of an export job. +type ExportJobConfiguration struct { + Mode ExportMode `json:"Mode"` + Filter ExportFilter `json:"Filter"` + Destination ExportDestination `json:"Destination"` + Format ExportFormat `json:"Format"` + MaxParallelExports int `json:"MaxParallelExports"` + MaxInstancesPerBatch int `json:"MaxInstancesPerBatch"` +} + +// ExportJobState is the persisted entity state of an export job. +type ExportJobState struct { + Status ExportJobStatus `json:"Status"` + Config *ExportJobConfiguration `json:"Config,omitempty"` + Checkpoint *ExportCheckpoint `json:"Checkpoint,omitempty"` + CreatedAt *time.Time `json:"CreatedAt,omitempty"` + LastModifiedAt *time.Time `json:"LastModifiedAt,omitempty"` + LastCheckpointTime *time.Time `json:"LastCheckpointTime,omitempty"` + LastError string `json:"LastError,omitempty"` + ScannedInstances int64 `json:"ScannedInstances"` + ExportedInstances int64 `json:"ExportedInstances"` + OrchestratorInstanceID string `json:"OrchestratorInstanceId,omitempty"` + // RunToken identifies the job's current run generation. Every Create mints + // a new one, so a run that started before the job was deleted and recreated + // carries a stale token and is fenced out of the new generation's state. + // Mutations require a matching, nonempty token. + RunToken string `json:"RunToken,omitempty"` +} + +// ExportJobDescription is the client-facing view of an export job. +type ExportJobDescription struct { + JobID string + Status ExportJobStatus + CreatedAt time.Time + LastModifiedAt time.Time + Config *ExportJobConfiguration + // OrchestratorInstanceID identifies the current generation, reserved during + // Create before its Run signal is delivered. Recreation assigns a new ID. + OrchestratorInstanceID string + ScannedInstances int64 + ExportedInstances int64 + LastError string + Checkpoint *ExportCheckpoint + LastCheckpointTime time.Time +} + +// ExportJobQuery filters a single page of export jobs. Status and creation-time +// filters are applied after the service returns the entity page, so a page can +// contain fewer than PageSize items. +type ExportJobQuery struct { + Status *ExportJobStatus + // JobIDPrefix restricts results to jobs whose ID starts with this value. + JobIDPrefix string + // CreatedFrom is an exclusive lower creation-time bound, matching .NET. + CreatedFrom time.Time + // CreatedTo is an exclusive upper creation-time bound, matching .NET. + CreatedTo time.Time + PageSize int32 + ContinuationToken string +} + +// ExportJobQueryResult is one page of export jobs. +type ExportJobQueryResult struct { + Jobs []*ExportJobDescription + ContinuationToken string +} + +// CommitCheckpointRequest is the entity input that records a batch's progress. +// A nil Checkpoint keeps the current cursor so the same batch can be retried; +// combined with a non-empty Failures list it implicitly fails the job. +type CommitCheckpointRequest struct { + ScannedInstances int64 `json:"ScannedInstances"` + ExportedInstances int64 `json:"ExportedInstances"` + Checkpoint *ExportCheckpoint `json:"Checkpoint,omitempty"` + Failures []ExportFailure `json:"Failures,omitempty"` + // RunToken fences the commit to the run generation that produced it. See + // [ExportJobState.RunToken]. + RunToken string `json:"RunToken,omitempty"` +} + +// RunJobRequest is the entity input of the Run operation, which starts the +// job's export orchestration. +type RunJobRequest struct { + // RunToken fences the run signal to the generation that emitted it. See + // [ExportJobState.RunToken]. + RunToken string `json:"RunToken,omitempty"` +} + +// MarkAsCompletedRequest is the entity input of the MarkAsCompleted operation. +type MarkAsCompletedRequest struct { + // RunToken fences the completion to the run generation that produced it. + // See [ExportJobState.RunToken]. + RunToken string `json:"RunToken,omitempty"` +} + +// MarkAsFailedRequest is the entity input of the MarkAsFailed operation. +type MarkAsFailedRequest struct { + // RunToken fences the failure to the run generation that produced it. See + // [ExportJobState.RunToken]. + RunToken string `json:"RunToken,omitempty"` + // Error is the message recorded on the job. It is optional. + Error string `json:"Error,omitempty"` +} + +// ExportJobOperationRequest is the input consumed by +// ExecuteExportJobOperationOrchestrator. +type ExportJobOperationRequest struct { + EntityID api.EntityID `json:"EntityId"` + OperationName string `json:"OperationName"` + Input any `json:"Input,omitempty"` +} + +// ExportJobRunRequest is the input consumed by ExportJobOrchestrator. +type ExportJobRunRequest struct { + JobEntityID api.EntityID `json:"JobEntityId"` + ProcessedCycles int `json:"ProcessedCycles"` + // RunToken is the job generation this orchestration run belongs to. Every + // entity mutation the run performs carries it, so a run left over from a + // deleted-and-recreated job cannot alter the new one. See + // [ExportJobState.RunToken]. + RunToken string `json:"RunToken,omitempty"` + // ContinuedExecution marks an execution that resumed through ContinueAsNew + // rather than a fresh start. Its zero value identifies the first execution, + // which is the only one that treats a missing job as an error rather than + // as a job that was deleted while it ran. + ContinuedExecution bool `json:"ContinuedExecution,omitempty"` +} + +// ListTerminalInstancesRequest is the input consumed by ListTerminalInstancesActivity. +// A zero CompletedTimeTo means the window has no upper bound. +type ListTerminalInstancesRequest struct { + CompletedTimeFrom time.Time `json:"CompletedTimeFrom"` + CompletedTimeTo *time.Time `json:"CompletedTimeTo"` + RuntimeStatus []api.OrchestrationStatus `json:"RuntimeStatus,omitempty"` + LastInstanceKey string `json:"LastInstanceKey,omitempty"` + MaxInstancesPerBatch int `json:"MaxInstancesPerBatch"` +} + +// InstancePage is one page of terminal instance IDs plus the next cursor. +type InstancePage struct { + InstanceIDs []string `json:"InstanceIds"` + NextCheckpoint *ExportCheckpoint `json:"NextCheckpoint,omitempty"` +} + +// ExportRequest is the input consumed by ExportInstanceHistoryActivity. +type ExportRequest struct { + InstanceID string `json:"InstanceId"` + Destination ExportDestination `json:"Destination"` + Format ExportFormat `json:"Format"` +} + +// ExportResult reports the outcome of exporting one instance. Per-instance +// failures are collected rather than thrown so a batch can report every failing +// instance at once. +type ExportResult struct { + InstanceID string `json:"InstanceId"` + Success bool `json:"Success"` + Error string `json:"Error,omitempty"` + BlobPath string `json:"BlobPath,omitempty"` + EventCount int `json:"EventCount,omitempty"` +} + +// TerminalStatuses returns the orchestration runtime statuses an export job +// accepts, in the .NET declaration order. +func TerminalStatuses() []api.OrchestrationStatus { + return []api.OrchestrationStatus{ + api.RUNTIME_STATUS_COMPLETED, + api.RUNTIME_STATUS_FAILED, + api.RUNTIME_STATUS_TERMINATED, + } +} + +func isTerminalStatus(status api.OrchestrationStatus) bool { + switch status { + case api.RUNTIME_STATUS_COMPLETED, api.RUNTIME_STATUS_FAILED, api.RUNTIME_STATUS_TERMINATED: + return true + default: + return false + } +} + +// validateTerminalStatuses rejects a filter that names any status an export job +// cannot read. It is shared by client-side normalization and the listing +// activity so both reject the same set with the same message. +func validateTerminalStatuses(statuses []api.OrchestrationStatus) error { + for _, status := range statuses { + if !isTerminalStatus(status) { + return &ValidationError{Message: terminalStatusesValidationMessage} + } + } + return nil +} + +func (state *ExportJobState) description(jobID string) *ExportJobDescription { + description := &ExportJobDescription{ + JobID: jobID, + Status: state.Status, + Config: state.Config, + OrchestratorInstanceID: state.OrchestratorInstanceID, + ScannedInstances: state.ScannedInstances, + ExportedInstances: state.ExportedInstances, + LastError: state.LastError, + Checkpoint: state.Checkpoint, + } + if state.CreatedAt != nil { + description.CreatedAt = *state.CreatedAt + } + if state.LastModifiedAt != nil { + description.LastModifiedAt = *state.LastModifiedAt + } + if state.LastCheckpointTime != nil { + description.LastCheckpointTime = *state.LastCheckpointTime + } + return description +} + +func (query ExportJobQuery) matches(description *ExportJobDescription) bool { + if query.Status != nil && description.Status != *query.Status { + return false + } + if !query.CreatedFrom.IsZero() && !description.CreatedAt.After(query.CreatedFrom) { + return false + } + if !query.CreatedTo.IsZero() && !description.CreatedAt.Before(query.CreatedTo) { + return false + } + return true +} diff --git a/exporthistory/models_test.go b/exporthistory/models_test.go new file mode 100644 index 00000000..12bee4af --- /dev/null +++ b/exporthistory/models_test.go @@ -0,0 +1,353 @@ +package exporthistory + +import ( + "encoding/json" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestExportModeAndStatusStrings(t *testing.T) { + assert.Equal(t, "Batch", ExportModeBatch.String()) + assert.Equal(t, "Continuous", ExportModeContinuous.String()) + assert.Equal(t, "ExportMode(0)", ExportMode(0).String()) + assert.True(t, ExportModeBatch.IsValid()) + assert.True(t, ExportModeContinuous.IsValid()) + assert.False(t, ExportMode(0).IsValid()) + assert.False(t, ExportMode(3).IsValid()) + + assert.Equal(t, "Pending", ExportJobStatusPending.String()) + assert.Equal(t, "Active", ExportJobStatusActive.String()) + assert.Equal(t, "Failed", ExportJobStatusFailed.String()) + assert.Equal(t, "Completed", ExportJobStatusCompleted.String()) + assert.Equal(t, "ExportJobStatus(9)", ExportJobStatus(9).String()) + assert.True(t, ExportJobStatusPending.IsValid()) + assert.True(t, ExportJobStatusActive.IsValid()) + assert.True(t, ExportJobStatusFailed.IsValid()) + assert.True(t, ExportJobStatusCompleted.IsValid()) + assert.False(t, ExportJobStatus(9).IsValid()) + + // The persisted numeric values are part of the wire contract with .NET. + assert.Equal(t, 0, int(ExportJobStatusPending)) + assert.Equal(t, 1, int(ExportJobStatusActive)) + assert.Equal(t, 2, int(ExportJobStatusFailed)) + assert.Equal(t, 3, int(ExportJobStatusCompleted)) + assert.Equal(t, 1, int(ExportModeBatch)) + assert.Equal(t, 2, int(ExportModeContinuous)) +} + +func TestExportFormatKindJSON(t *testing.T) { + encoded, err := json.Marshal(ExportFormatJSONL) + require.NoError(t, err) + assert.JSONEq(t, `"Jsonl"`, string(encoded)) + + encoded, err = json.Marshal(ExportFormatJSON) + require.NoError(t, err) + assert.JSONEq(t, `"Json"`, string(encoded)) + + _, err = json.Marshal(ExportFormatKind(9)) + require.Error(t, err) + + for _, input := range []string{`"Jsonl"`, `"jsonl"`, `0`} { + var kind ExportFormatKind + require.NoError(t, json.Unmarshal([]byte(input), &kind), input) + assert.Equal(t, ExportFormatJSONL, kind) + } + for _, input := range []string{`"Json"`, `"JSON"`, `1`} { + var kind ExportFormatKind + require.NoError(t, json.Unmarshal([]byte(input), &kind), input) + assert.Equal(t, ExportFormatJSON, kind) + } + for _, input := range []string{`"csv"`, `7`, `{}`} { + var kind ExportFormatKind + require.Error(t, json.Unmarshal([]byte(input), &kind), input) + } +} + +func TestExportFormatDefaultsAndExtensions(t *testing.T) { + format := DefaultExportFormat() + assert.Equal(t, ExportFormatJSONL, format.Kind) + assert.Equal(t, "1.0", format.SchemaVersion) + assert.Equal(t, "jsonl.gz", format.FileExtension()) + assert.Equal(t, "json", ExportFormat{Kind: ExportFormatJSON}.FileExtension()) + // Unknown kinds fall back to the compressed default rather than producing an + // extensionless object name. + assert.Equal(t, "jsonl.gz", ExportFormat{Kind: ExportFormatKind(9)}.FileExtension()) +} + +// TestExportFormatContentType pins the unambiguous storage contract: a JSONL +// object is an opaque gzip file whose name and content type agree, so no reader +// has to guess whether it received raw or already-decompressed bytes. +func TestExportFormatContentType(t *testing.T) { + assert.Equal(t, "application/gzip", DefaultExportFormat().ContentType()) + assert.Equal(t, "application/json", ExportFormat{Kind: ExportFormatJSON}.ContentType()) + assert.Equal(t, "application/gzip", ExportFormat{Kind: ExportFormatKind(9)}.ContentType()) +} + +func TestExportDestinationValidate(t *testing.T) { + valid := []ExportDestination{ + {Container: "container"}, + {Container: "my-container", Prefix: "prefix"}, + {Container: "abc", Prefix: "a/b/c/"}, + {Container: "a1-b2-c3"}, + } + for _, destination := range valid { + require.NoError(t, destination.Validate(), "%+v", destination) + } + + invalid := []struct { + destination ExportDestination + message string + }{ + {ExportDestination{}, "container is required"}, + {ExportDestination{Container: " "}, "container is required"}, + {ExportDestination{Container: "ab"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "UPPER"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "-lead"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "trail-"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "double--dash"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "under_score"}, "not a valid Azure Blob container name"}, + {ExportDestination{Container: "okc", Prefix: "/leading"}, "must not start with '/'"}, + {ExportDestination{Container: "okc", Prefix: "a//b"}, "must not contain '//'"}, + {ExportDestination{Container: "okc", Prefix: "../escape"}, "relative path segments"}, + {ExportDestination{Container: "okc", Prefix: "a/./b"}, "relative path segments"}, + {ExportDestination{Container: "okc", Prefix: "bad\npath"}, "unsupported character"}, + } + for _, test := range invalid { + err := test.destination.Validate() + require.ErrorIs(t, err, ErrValidation, "%+v", test.destination) + assert.Contains(t, err.Error(), test.message) + } +} + +func TestExportDestinationBlobPath(t *testing.T) { + assert.Equal(t, "object", ExportDestination{Container: "c"}.BlobPath("object")) + assert.Equal(t, "p/object", ExportDestination{Container: "c", Prefix: "p"}.BlobPath("object")) + assert.Equal(t, "p/object", ExportDestination{Container: "c", Prefix: "p/"}.BlobPath("object")) + assert.Equal(t, "a/b/object", ExportDestination{Container: "c", Prefix: "a/b//"}.BlobPath("object")) +} + +func TestExportFilterJSONRoundTrip(t *testing.T) { + from := time.Date(2024, time.January, 2, 3, 4, 5, 0, time.UTC) + to := from.Add(time.Hour) + + bounded := ExportFilter{ + CompletedTimeFrom: from, + CompletedTimeTo: to, + RuntimeStatus: TerminalStatuses(), + } + encoded, err := json.Marshal(bounded) + require.NoError(t, err) + var decoded ExportFilter + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.True(t, decoded.CompletedTimeFrom.Equal(from)) + assert.True(t, decoded.CompletedTimeTo.Equal(to)) + assert.Equal(t, TerminalStatuses(), decoded.RuntimeStatus) + + open := ExportFilter{CompletedTimeFrom: from} + encoded, err = json.Marshal(open) + require.NoError(t, err) + assert.Contains(t, string(encoded), `"CompletedTimeTo":null`) + decoded = ExportFilter{} + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.True(t, decoded.CompletedTimeTo.IsZero()) +} + +func TestExportJobStateJSONRoundTrip(t *testing.T) { + now := time.Date(2024, time.May, 5, 6, 7, 8, 0, time.UTC) + state := ExportJobState{ + Status: ExportJobStatusActive, + Config: &ExportJobConfiguration{ + Mode: ExportModeBatch, + Filter: ExportFilter{CompletedTimeFrom: now.Add(-time.Hour), CompletedTimeTo: now}, + Destination: ExportDestination{Container: "container", Prefix: "batch-job/"}, + Format: DefaultExportFormat(), + MaxParallelExports: DefaultMaxParallelExports, + MaxInstancesPerBatch: DefaultMaxInstancesPerBatch, + }, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + CreatedAt: &now, + LastModifiedAt: &now, + LastCheckpointTime: &now, + LastError: "boom", + ScannedInstances: 12, + ExportedInstances: 11, + OrchestratorInstanceID: "ExportJob-job-run-a", + RunToken: "run-a", + } + encoded, err := json.Marshal(state) + require.NoError(t, err) + + var decoded ExportJobState + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, state.Status, decoded.Status) + require.NotNil(t, decoded.Config) + assert.Equal(t, state.Config.Mode, decoded.Config.Mode) + assert.Equal(t, state.Config.Destination, decoded.Config.Destination) + assert.Equal(t, state.Config.MaxInstancesPerBatch, decoded.Config.MaxInstancesPerBatch) + require.NotNil(t, decoded.Checkpoint) + assert.Equal(t, "cursor", decoded.Checkpoint.LastInstanceKey) + assert.Equal(t, int64(12), decoded.ScannedInstances) + assert.Equal(t, int64(11), decoded.ExportedInstances) + assert.Equal(t, "boom", decoded.LastError) + assert.Equal(t, "ExportJob-job-run-a", decoded.OrchestratorInstanceID) + assert.Equal(t, "run-a", decoded.RunToken) + + empty, err := json.Marshal(ExportJobState{}) + require.NoError(t, err) + assert.JSONEq(t, `{"Status":0,"ScannedInstances":0,"ExportedInstances":0}`, string(empty)) +} + +func TestRunTokenMatchesRequiresNonemptyGeneration(t *testing.T) { + for _, stateToken := range []string{"", "run-a", "run-b"} { + for _, requestToken := range []string{"", "run-a", "run-b"} { + assert.Equal(t, stateToken != "" && stateToken == requestToken, + runTokenMatches(&ExportJobState{RunToken: stateToken}, requestToken), + "state token %q, request token %q", stateToken, requestToken) + } + } +} + +func TestMarkAsFailedRequestJSON(t *testing.T) { + require.Error(t, json.Unmarshal([]byte(`"boom"`), new(MarkAsFailedRequest))) + + var fromObject MarkAsFailedRequest + require.NoError(t, json.Unmarshal([]byte(`{"RunToken":"run-a","Error":"boom"}`), &fromObject)) + assert.Equal(t, "run-a", fromObject.RunToken) + assert.Equal(t, "boom", fromObject.Error) + + var empty MarkAsFailedRequest + require.NoError(t, json.Unmarshal([]byte(`{}`), &empty)) + assert.Empty(t, empty.Error) + + require.Error(t, json.Unmarshal([]byte(`[1,2]`), new(MarkAsFailedRequest))) + + encoded, err := json.Marshal(MarkAsFailedRequest{RunToken: "run-a", Error: "boom"}) + require.NoError(t, err) + assert.JSONEq(t, `{"RunToken":"run-a","Error":"boom"}`, string(encoded)) +} + +func TestExportJobRunRequestRunFencingJSON(t *testing.T) { + request := ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + ContinuedExecution: true, + } + encoded, err := json.Marshal(request) + require.NoError(t, err) + assert.JSONEq(t, `{"JobEntityId":"@exportjob@job-1","ProcessedCycles":0,"RunToken":"run-a","ContinuedExecution":true}`, string(encoded)) + var decoded ExportJobRunRequest + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, request, decoded) +} + +func TestExportJobStateDescription(t *testing.T) { + now := time.Date(2024, time.June, 1, 0, 0, 0, 0, time.UTC) + state := ExportJobState{ + Status: ExportJobStatusCompleted, + CreatedAt: &now, + LastModifiedAt: &now, + LastCheckpointTime: &now, + ScannedInstances: 5, + ExportedInstances: 4, + LastError: "err", + OrchestratorInstanceID: "ExportJob-x-run-a", + Checkpoint: &ExportCheckpoint{LastInstanceKey: "k"}, + } + description := state.description("x") + assert.Equal(t, "x", description.JobID) + assert.Equal(t, ExportJobStatusCompleted, description.Status) + assert.Equal(t, now, description.CreatedAt) + assert.Equal(t, now, description.LastModifiedAt) + assert.Equal(t, now, description.LastCheckpointTime) + assert.Equal(t, int64(5), description.ScannedInstances) + assert.Equal(t, int64(4), description.ExportedInstances) + assert.Equal(t, "err", description.LastError) + assert.Equal(t, "ExportJob-x-run-a", description.OrchestratorInstanceID) + require.NotNil(t, description.Checkpoint) + + // Unset timestamps stay zero rather than panicking on a nil dereference. + bare := (&ExportJobState{}).description("y") + assert.True(t, bare.CreatedAt.IsZero()) + assert.True(t, bare.LastModifiedAt.IsZero()) + assert.True(t, bare.LastCheckpointTime.IsZero()) +} + +func TestExportJobQueryMatches(t *testing.T) { + created := time.Date(2024, time.July, 15, 12, 0, 0, 0, time.UTC) + description := &ExportJobDescription{Status: ExportJobStatusActive, CreatedAt: created} + + active := ExportJobStatusActive + completed := ExportJobStatusCompleted + assert.True(t, ExportJobQuery{}.matches(description)) + assert.True(t, ExportJobQuery{Status: &active}.matches(description)) + assert.False(t, ExportJobQuery{Status: &completed}.matches(description)) + + // Bounds are exclusive, matching .NET. + assert.True(t, ExportJobQuery{CreatedFrom: created.Add(-time.Second)}.matches(description)) + assert.False(t, ExportJobQuery{CreatedFrom: created}.matches(description)) + assert.True(t, ExportJobQuery{CreatedTo: created.Add(time.Second)}.matches(description)) + assert.False(t, ExportJobQuery{CreatedTo: created}.matches(description)) +} + +func TestTerminalStatusHelpers(t *testing.T) { + assert.Equal(t, []api.OrchestrationStatus{ + api.RUNTIME_STATUS_COMPLETED, + api.RUNTIME_STATUS_FAILED, + api.RUNTIME_STATUS_TERMINATED, + }, TerminalStatuses()) + + // The exported slice must not be a shared backing array. + first := TerminalStatuses() + first[0] = api.RUNTIME_STATUS_FAILED + assert.Equal(t, api.RUNTIME_STATUS_COMPLETED, TerminalStatuses()[0]) + + for _, status := range TerminalStatuses() { + assert.True(t, isTerminalStatus(status)) + } + for _, status := range []api.OrchestrationStatus{ + api.RUNTIME_STATUS_RUNNING, + api.RUNTIME_STATUS_PENDING, + api.RUNTIME_STATUS_SUSPENDED, + api.RUNTIME_STATUS_CANCELED, + api.RUNTIME_STATUS_CONTINUED_AS_NEW, + } { + assert.False(t, isTerminalStatus(status), status) + } +} + +func TestSystemIdentifiers(t *testing.T) { + assert.Equal(t, "@exportjob@abc", EntityID("abc").String()) + assert.Equal(t, "exportjob", EntityID("abc").Name) + assert.Equal(t, "abc", EntityID("abc").Key) +} + +func TestCommitCheckpointRequestJSON(t *testing.T) { + request := CommitCheckpointRequest{ + ScannedInstances: 10, + ExportedInstances: 9, + Checkpoint: &ExportCheckpoint{LastInstanceKey: "cursor"}, + Failures: []ExportFailure{ + {InstanceID: "i1", Reason: "boom", AttemptCount: 3, LastAttempt: time.Unix(0, 0).UTC()}, + }, + } + encoded, err := json.Marshal(request) + require.NoError(t, err) + var decoded CommitCheckpointRequest + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, request.ScannedInstances, decoded.ScannedInstances) + assert.Equal(t, request.ExportedInstances, decoded.ExportedInstances) + require.NotNil(t, decoded.Checkpoint) + assert.Equal(t, "cursor", decoded.Checkpoint.LastInstanceKey) + require.Len(t, decoded.Failures, 1) + assert.Equal(t, "i1", decoded.Failures[0].InstanceID) + assert.Equal(t, 3, decoded.Failures[0].AttemptCount) + + // A failed batch commits a nil checkpoint so the cursor stays put. + failed, err := json.Marshal(CommitCheckpointRequest{Failures: request.Failures}) + require.NoError(t, err) + assert.NotContains(t, string(failed), `"Checkpoint"`) +} diff --git a/exporthistory/options.go b/exporthistory/options.go new file mode 100644 index 00000000..00884ec2 --- /dev/null +++ b/exporthistory/options.go @@ -0,0 +1,262 @@ +package exporthistory + +import ( + "encoding/json" + "errors" + "fmt" + "slices" + "strings" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" +) + +// JobCreationOptions configures a new export job. +// +// Zero values select defaults: an empty JobID generates one, a nil Format uses +// [DefaultExportFormat], an empty RuntimeStatus exports every terminal status, +// and a zero MaxInstancesPerBatch uses [DefaultMaxInstancesPerBatch]. A zero +// CompletedTimeTo means the window has no upper bound, which is required for +// [ExportModeContinuous] and rejected for [ExportModeBatch]. +// +// Destination may be left nil when the client is configured with a default +// container; the client fills it in before the job reaches the entity. +type JobCreationOptions struct { + JobID string + Mode ExportMode + CompletedTimeFrom time.Time + CompletedTimeTo time.Time + Destination *ExportDestination + Format *ExportFormat + RuntimeStatus []api.OrchestrationStatus + MaxInstancesPerBatch int +} + +// jobCreationJSON is the .NET-compatible wire shape of JobCreationOptions. +type jobCreationJSON struct { + JobID string `json:"JobId"` + Mode ExportMode `json:"Mode"` + CompletedTimeFrom time.Time `json:"CompletedTimeFrom"` + CompletedTimeTo *time.Time `json:"CompletedTimeTo"` + Destination *ExportDestination `json:"Destination,omitempty"` + Format *ExportFormat `json:"Format,omitempty"` + RuntimeStatus []api.OrchestrationStatus `json:"RuntimeStatus,omitempty"` + MaxInstancesPerBatch int `json:"MaxInstancesPerBatch"` +} + +func (o JobCreationOptions) MarshalJSON() ([]byte, error) { + return json.Marshal(jobCreationJSON{ + JobID: o.JobID, + Mode: o.Mode, + CompletedTimeFrom: o.CompletedTimeFrom, + CompletedTimeTo: optionalTime(o.CompletedTimeTo), + Destination: o.Destination, + Format: o.Format, + RuntimeStatus: o.RuntimeStatus, + MaxInstancesPerBatch: o.MaxInstancesPerBatch, + }) +} + +func (o *JobCreationOptions) UnmarshalJSON(data []byte) error { + var decoded jobCreationJSON + if err := json.Unmarshal(data, &decoded); err != nil { + return err + } + *o = JobCreationOptions{ + JobID: decoded.JobID, + Mode: decoded.Mode, + CompletedTimeFrom: decoded.CompletedTimeFrom, + Destination: decoded.Destination, + Format: decoded.Format, + RuntimeStatus: decoded.RuntimeStatus, + MaxInstancesPerBatch: decoded.MaxInstancesPerBatch, + } + if decoded.CompletedTimeTo != nil { + o.CompletedTimeTo = *decoded.CompletedTimeTo + } + return nil +} + +// isZero reports whether the options carry no caller-supplied value, which is +// how an entity operation invoked without an input is detected. +func (o JobCreationOptions) isZero() bool { + return o.JobID == "" && + o.Mode == 0 && + o.CompletedTimeFrom.IsZero() && + o.CompletedTimeTo.IsZero() && + o.Destination == nil && + o.Format == nil && + len(o.RuntimeStatus) == 0 && + o.MaxInstancesPerBatch == 0 +} + +// Normalize validates the options against the wall clock and returns a copy +// with every default applied. It generates a job ID when one is not supplied, +// so calling it twice produces two different jobs. +func (o JobCreationOptions) Normalize() (JobCreationOptions, error) { + return o.normalize(time.Now().UTC(), 0) +} + +// Validate reports whether the options are usable, applying the same rules as +// [JobCreationOptions.Normalize] without mutating anything. +func (o JobCreationOptions) Validate() error { + _, err := o.normalize(time.Now().UTC(), 0) + return err +} + +// MaxCreationClockSkew is how far a batch job's CompletedTimeTo may sit ahead of +// the worker's clock and still be accepted by the ExportJob entity. +// +// A client validates the upper bound strictly against its own clock, but the +// entity runs on a worker whose clock is independent. Without this tolerance a +// window the client accepted would be rejected by a worker running slightly +// behind, so the entity allows a bounded skew while clients stay strict. +const MaxCreationClockSkew = 5 * time.Minute + +// normalize applies validation and defaults against an explicit clock. A +// positive futureSkew relaxes only the "upper bound is not in the future" rule, +// which is how entity-side validation absorbs client/worker clock skew without +// shifting the window a continuous job starts from. +func (o JobCreationOptions) normalize(now time.Time, futureSkew time.Duration) (JobCreationOptions, error) { + normalized := o + normalized.RuntimeStatus = slices.Clone(o.RuntimeStatus) + + if strings.TrimSpace(normalized.JobID) == "" { + normalized.JobID = newCompactUUID() + } + if err := validateJobID(normalized.JobID); err != nil { + return JobCreationOptions{}, err + } + // invalidf reports a validation failure against the job ID resolved above, + // which every rule below shares. + invalidf := func(format string, args ...any) (JobCreationOptions, error) { + return JobCreationOptions{}, &ValidationError{ + JobID: normalized.JobID, + Message: fmt.Sprintf(format, args...), + } + } + + switch normalized.Mode { + case ExportModeBatch: + switch { + case normalized.CompletedTimeFrom.IsZero(): + return invalidf("CompletedTimeFrom is required for Batch export mode") + case normalized.CompletedTimeTo.IsZero(): + return invalidf("CompletedTimeTo is required for Batch export mode") + case !normalized.CompletedTimeTo.After(normalized.CompletedTimeFrom): + return invalidf( + "CompletedTimeTo (%s) must be greater than CompletedTimeFrom (%s) for Batch export mode", + formatInstant(normalized.CompletedTimeTo), formatInstant(normalized.CompletedTimeFrom)) + case normalized.CompletedTimeTo.After(now.Add(futureSkew)): + return invalidf( + "CompletedTimeTo (%s) cannot be in the future; it must be less than or equal to the current time (%s)", + formatInstant(normalized.CompletedTimeTo), formatInstant(now)) + } + case ExportModeContinuous: + if !normalized.CompletedTimeTo.IsZero() { + return invalidf("CompletedTimeTo is not allowed for Continuous export mode") + } + if normalized.CompletedTimeFrom.IsZero() { + normalized.CompletedTimeFrom = now + } + default: + return invalidf("invalid export mode %d", int(normalized.Mode)) + } + + switch { + case normalized.MaxInstancesPerBatch == 0: + normalized.MaxInstancesPerBatch = DefaultMaxInstancesPerBatch + case normalized.MaxInstancesPerBatch < 1 || normalized.MaxInstancesPerBatch > MaxInstancesPerBatchLimit: + return invalidf("MaxInstancesPerBatch must be between 1 and %d, but was %d", + MaxInstancesPerBatchLimit, normalized.MaxInstancesPerBatch) + } + + if len(normalized.RuntimeStatus) == 0 { + normalized.RuntimeStatus = TerminalStatuses() + } else if err := validateTerminalStatuses(normalized.RuntimeStatus); err != nil { + return JobCreationOptions{}, withJobID(err, normalized.JobID) + } + + if normalized.Format == nil { + format := DefaultExportFormat() + normalized.Format = &format + } else { + format := *normalized.Format + if !format.Kind.IsValid() { + return invalidf("invalid export format kind %d", int(format.Kind)) + } + if format.SchemaVersion == "" { + format.SchemaVersion = DefaultSchemaVersion + } + normalized.Format = &format + } + + if normalized.Destination != nil { + destination := *normalized.Destination + if err := destination.Validate(); err != nil { + return JobCreationOptions{}, withJobID(err, normalized.JobID) + } + normalized.Destination = &destination + } + + return normalized, nil +} + +// configuration converts normalized options into the durable job configuration. +// The destination must already be resolved. +func (o JobCreationOptions) configuration() (*ExportJobConfiguration, error) { + if o.Destination == nil { + return nil, &ValidationError{JobID: o.JobID, Message: "export destination is required"} + } + if err := o.Destination.Validate(); err != nil { + return nil, withJobID(err, o.JobID) + } + format := DefaultExportFormat() + if o.Format != nil { + format = *o.Format + } + return &ExportJobConfiguration{ + Mode: o.Mode, + Filter: ExportFilter{ + CompletedTimeFrom: o.CompletedTimeFrom, + CompletedTimeTo: o.CompletedTimeTo, + RuntimeStatus: slices.Clone(o.RuntimeStatus), + }, + Destination: *o.Destination, + Format: format, + MaxParallelExports: DefaultMaxParallelExports, + MaxInstancesPerBatch: o.MaxInstancesPerBatch, + }, nil +} + +func newCompactUUID() string { + return strings.ReplaceAll(uuid.NewString(), "-", "") +} + +// validateJobID rejects IDs that would produce an ambiguous entity instance ID +// or an unusable orchestration instance ID. +func validateJobID(jobID string) error { + switch { + case strings.TrimSpace(jobID) == "": + return &ValidationError{Message: "export job ID is required"} + case strings.Contains(jobID, "@"): + return &ValidationError{JobID: jobID, Message: "export job ID must not contain '@'"} + case strings.ContainsAny(jobID, "\x00\r\n\t"): + return &ValidationError{JobID: jobID, Message: "export job ID must not contain control characters"} + default: + return nil + } +} + +func withJobID(err error, jobID string) error { + var validation *ValidationError + if errors.As(err, &validation) && validation.JobID == "" { + return &ValidationError{JobID: jobID, Message: validation.Message} + } + return err +} + +func formatInstant(value time.Time) string { + return value.UTC().Format(time.RFC3339Nano) +} diff --git a/exporthistory/options_test.go b/exporthistory/options_test.go new file mode 100644 index 00000000..7e4ce87c --- /dev/null +++ b/exporthistory/options_test.go @@ -0,0 +1,459 @@ +package exporthistory + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestJobCreationOptionsNormalize ports the upstream +// ExportJobCreationOptionsTests matrix to the Go normalization entry point. +func TestJobCreationOptionsNormalize(t *testing.T) { + from := time.Now().UTC().Add(-24 * time.Hour) + to := time.Now().UTC() + + t.Run("batch mode with valid parameters", func(t *testing.T) { + options, err := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Destination: &ExportDestination{Container: "test-container"}, + }.Normalize() + require.NoError(t, err) + assert.Equal(t, ExportModeBatch, options.Mode) + assert.Equal(t, from, options.CompletedTimeFrom) + assert.Equal(t, to, options.CompletedTimeTo) + assert.Equal(t, "test-container", options.Destination.Container) + assert.NotEmpty(t, options.JobID) + assert.Equal(t, DefaultExportFormat(), *options.Format) + assert.Equal(t, TerminalStatuses(), options.RuntimeStatus) + assert.Equal(t, DefaultMaxInstancesPerBatch, options.MaxInstancesPerBatch) + }) + + t.Run("custom job ID is preserved", func(t *testing.T) { + options, err := JobCreationOptions{ + JobID: "custom-job-id", + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + }.Normalize() + require.NoError(t, err) + assert.Equal(t, "custom-job-id", options.JobID) + }) + + t.Run("empty job ID generates a stable GUID", func(t *testing.T) { + options, err := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + }.Normalize() + require.NoError(t, err) + require.Len(t, options.JobID, 32) + assert.NotContains(t, options.JobID, "-") + + other, err := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + }.Normalize() + require.NoError(t, err) + assert.NotEqual(t, options.JobID, other.JobID) + }) + + t.Run("whitespace job ID generates a GUID", func(t *testing.T) { + options, err := JobCreationOptions{ + JobID: " ", + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + }.Normalize() + require.NoError(t, err) + assert.Len(t, options.JobID, 32) + }) + + invalid := []struct { + name string + options JobCreationOptions + message string + }{ + { + name: "batch without CompletedTimeFrom", + options: JobCreationOptions{Mode: ExportModeBatch, CompletedTimeTo: to}, + message: "CompletedTimeFrom is required for Batch export mode", + }, + { + name: "batch without CompletedTimeTo", + options: JobCreationOptions{Mode: ExportModeBatch, CompletedTimeFrom: from}, + message: "CompletedTimeTo is required for Batch export mode", + }, + { + name: "batch with CompletedTimeTo before CompletedTimeFrom", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: to, + CompletedTimeTo: from, + }, + message: "must be greater than CompletedTimeFrom", + }, + { + name: "batch with CompletedTimeTo equal to CompletedTimeFrom", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: from, + }, + message: "must be greater than CompletedTimeFrom", + }, + { + name: "batch with CompletedTimeTo in the future", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: time.Now().UTC().Add(24 * time.Hour), + }, + message: "cannot be in the future", + }, + { + name: "continuous with CompletedTimeTo", + options: JobCreationOptions{Mode: ExportModeContinuous, CompletedTimeTo: to}, + message: "CompletedTimeTo is not allowed for Continuous export mode", + }, + { + name: "unknown mode", + options: JobCreationOptions{Mode: ExportMode(999), CompletedTimeFrom: from, CompletedTimeTo: to}, + message: "invalid export mode 999", + }, + { + name: "zero mode", + options: JobCreationOptions{CompletedTimeFrom: from, CompletedTimeTo: to}, + message: "invalid export mode 0", + }, + { + name: "job ID containing an entity separator", + options: JobCreationOptions{ + JobID: "job@id", + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + }, + message: "must not contain '@'", + }, + { + name: "invalid destination container", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Destination: &ExportDestination{Container: "Invalid_Container"}, + }, + message: "is not a valid Azure Blob container name", + }, + { + name: "destination prefix escaping the container", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Destination: &ExportDestination{Container: "container", Prefix: "a/../b"}, + }, + message: "relative path segments", + }, + { + name: "invalid format kind", + options: JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Format: &ExportFormat{Kind: ExportFormatKind(7)}, + }, + message: "invalid export format kind 7", + }, + } + for _, test := range invalid { + t.Run(test.name, func(t *testing.T) { + _, err := test.options.Normalize() + require.Error(t, err) + require.ErrorIs(t, err, ErrValidation) + var validation *ValidationError + require.ErrorAs(t, err, &validation) + assert.Contains(t, err.Error(), test.message) + }) + } + + t.Run("continuous mode defaults CompletedTimeFrom to now", func(t *testing.T) { + before := time.Now().UTC() + options, err := JobCreationOptions{Mode: ExportModeContinuous}.Normalize() + require.NoError(t, err) + after := time.Now().UTC() + assert.False(t, options.CompletedTimeFrom.Before(before)) + assert.False(t, options.CompletedTimeFrom.After(after)) + assert.True(t, options.CompletedTimeTo.IsZero()) + }) + + t.Run("continuous mode preserves an explicit CompletedTimeFrom", func(t *testing.T) { + options, err := JobCreationOptions{Mode: ExportModeContinuous, CompletedTimeFrom: from}.Normalize() + require.NoError(t, err) + assert.Equal(t, from, options.CompletedTimeFrom) + }) +} + +// TestJobCreationOptionsFutureWindowSkew pins the two-sided rule for a batch +// window's upper bound: clients validate strictly against their own clock, while +// the entity absorbs a bounded skew so a worker whose clock trails the client's +// does not reject a job the client accepted. +func TestJobCreationOptionsFutureWindowSkew(t *testing.T) { + now := time.Date(2024, time.April, 1, 12, 0, 0, 0, time.UTC) + base := JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeBatch, + CompletedTimeFrom: now.Add(-time.Hour), + } + + t.Run("strict validation rejects any future upper bound", func(t *testing.T) { + options := base + options.CompletedTimeTo = now.Add(time.Second) + _, err := options.normalize(now, 0) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "cannot be in the future") + }) + + t.Run("the entity tolerates an upper bound within the documented skew", func(t *testing.T) { + for _, ahead := range []time.Duration{time.Second, MaxCreationClockSkew / 2, MaxCreationClockSkew} { + options := base + options.CompletedTimeTo = now.Add(ahead) + normalized, err := options.normalize(now, MaxCreationClockSkew) + require.NoError(t, err, "%s ahead", ahead) + assert.Equal(t, options.CompletedTimeTo, normalized.CompletedTimeTo) + } + }) + + t.Run("beyond the skew it is still rejected", func(t *testing.T) { + options := base + options.CompletedTimeTo = now.Add(MaxCreationClockSkew + time.Second) + _, err := options.normalize(now, MaxCreationClockSkew) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "cannot be in the future") + }) + + t.Run("the skew never shifts the window a continuous job starts from", func(t *testing.T) { + normalized, err := JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeContinuous, + }.normalize(now, MaxCreationClockSkew) + require.NoError(t, err) + assert.Equal(t, now, normalized.CompletedTimeFrom) + }) +} + +func TestJobCreationOptionsMaxInstancesPerBatch(t *testing.T) { + base := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC(), + } + for _, value := range []int{-1, 1001, 2000} { + options := base + options.MaxInstancesPerBatch = value + _, err := options.Normalize() + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "MaxInstancesPerBatch must be between 1 and 1000") + } + for _, value := range []int{1, 100, 500, 1000} { + options := base + options.MaxInstancesPerBatch = value + normalized, err := options.Normalize() + require.NoError(t, err) + assert.Equal(t, value, normalized.MaxInstancesPerBatch) + } + options := base + normalized, err := options.Normalize() + require.NoError(t, err) + assert.Equal(t, DefaultMaxInstancesPerBatch, normalized.MaxInstancesPerBatch) +} + +func TestJobCreationOptionsRuntimeStatusFilters(t *testing.T) { + base := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC(), + } + + t.Run("rejects non-terminal statuses", func(t *testing.T) { + for _, status := range []api.OrchestrationStatus{ + api.RUNTIME_STATUS_RUNNING, + api.RUNTIME_STATUS_PENDING, + api.RUNTIME_STATUS_SUSPENDED, + api.RUNTIME_STATUS_CONTINUED_AS_NEW, + api.RUNTIME_STATUS_CANCELED, + } { + options := base + options.RuntimeStatus = []api.OrchestrationStatus{status} + _, err := options.Normalize() + require.ErrorIs(t, err, ErrValidation, "status %v must be rejected", status) + assert.Contains(t, err.Error(), "terminal orchestration statuses only") + } + }) + + t.Run("accepts terminal statuses", func(t *testing.T) { + options := base + options.RuntimeStatus = TerminalStatuses() + normalized, err := options.Normalize() + require.NoError(t, err) + assert.Equal(t, TerminalStatuses(), normalized.RuntimeStatus) + }) + + t.Run("empty and nil default to every terminal status", func(t *testing.T) { + for _, statuses := range [][]api.OrchestrationStatus{nil, {}} { + options := base + options.RuntimeStatus = statuses + normalized, err := options.Normalize() + require.NoError(t, err) + require.Len(t, normalized.RuntimeStatus, 3) + assert.Equal(t, TerminalStatuses(), normalized.RuntimeStatus) + } + }) + + t.Run("normalization does not alias the caller slice", func(t *testing.T) { + statuses := []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED} + options := base + options.RuntimeStatus = statuses + normalized, err := options.Normalize() + require.NoError(t, err) + normalized.RuntimeStatus[0] = api.RUNTIME_STATUS_FAILED + assert.Equal(t, api.RUNTIME_STATUS_COMPLETED, statuses[0]) + }) +} + +func TestJobCreationOptionsCustomFormat(t *testing.T) { + options, err := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC(), + Format: &ExportFormat{Kind: ExportFormatJSON, SchemaVersion: "2.0"}, + }.Normalize() + require.NoError(t, err) + assert.Equal(t, ExportFormat{Kind: ExportFormatJSON, SchemaVersion: "2.0"}, *options.Format) + + blank, err := JobCreationOptions{ + Mode: ExportModeBatch, + CompletedTimeFrom: time.Now().UTC().Add(-time.Hour), + CompletedTimeTo: time.Now().UTC(), + Format: &ExportFormat{Kind: ExportFormatJSON}, + }.Normalize() + require.NoError(t, err) + assert.Equal(t, DefaultSchemaVersion, blank.Format.SchemaVersion) +} + +func TestJobCreationOptionsValidate(t *testing.T) { + require.NoError(t, JobCreationOptions{Mode: ExportModeContinuous}.Validate()) + require.ErrorIs(t, JobCreationOptions{}.Validate(), ErrValidation) +} + +// TestJobCreationOptionsJSONRoundTrip pins the .NET-compatible wire shape used +// as the Create entity operation input. +func TestJobCreationOptionsJSONRoundTrip(t *testing.T) { + from := time.Date(2024, time.March, 1, 10, 0, 0, 0, time.UTC) + to := time.Date(2024, time.March, 2, 10, 0, 0, 0, time.UTC) + options := JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Destination: &ExportDestination{Container: "container", Prefix: "batch-job-1/"}, + Format: &ExportFormat{Kind: ExportFormatJSONL, SchemaVersion: "1.0"}, + RuntimeStatus: TerminalStatuses(), + MaxInstancesPerBatch: 25, + } + encoded, err := json.Marshal(options) + require.NoError(t, err) + + var raw map[string]any + require.NoError(t, json.Unmarshal(encoded, &raw)) + assert.Equal(t, "job-1", raw["JobId"]) + assert.Equal(t, float64(1), raw["Mode"]) + assert.Equal(t, float64(25), raw["MaxInstancesPerBatch"]) + assert.Contains(t, string(encoded), `"Kind":"Jsonl"`) + + var decoded JobCreationOptions + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.Equal(t, options.JobID, decoded.JobID) + assert.Equal(t, options.Mode, decoded.Mode) + assert.True(t, options.CompletedTimeFrom.Equal(decoded.CompletedTimeFrom)) + assert.True(t, options.CompletedTimeTo.Equal(decoded.CompletedTimeTo)) + assert.Equal(t, options.Destination, decoded.Destination) + assert.Equal(t, options.Format, decoded.Format) + assert.Equal(t, options.MaxInstancesPerBatch, decoded.MaxInstancesPerBatch) +} + +// TestJobCreationOptionsUnsetUpperBoundSerializesNull keeps a continuous job's +// open-ended window distinguishable from the zero instant on the wire. +func TestJobCreationOptionsUnsetUpperBoundSerializesNull(t *testing.T) { + options, err := JobCreationOptions{JobID: "job", Mode: ExportModeContinuous}.Normalize() + require.NoError(t, err) + encoded, err := json.Marshal(options) + require.NoError(t, err) + assert.Contains(t, string(encoded), `"CompletedTimeTo":null`) + + var decoded JobCreationOptions + require.NoError(t, json.Unmarshal(encoded, &decoded)) + assert.True(t, decoded.CompletedTimeTo.IsZero()) +} + +func TestJobCreationOptionsIsZero(t *testing.T) { + assert.True(t, JobCreationOptions{}.isZero()) + assert.False(t, JobCreationOptions{Mode: ExportModeBatch}.isZero()) + assert.False(t, JobCreationOptions{JobID: "x"}.isZero()) + assert.False(t, JobCreationOptions{MaxInstancesPerBatch: 1}.isZero()) + assert.False(t, JobCreationOptions{RuntimeStatus: TerminalStatuses()}.isZero()) + assert.False(t, JobCreationOptions{Format: &ExportFormat{}}.isZero()) + assert.False(t, JobCreationOptions{Destination: &ExportDestination{}}.isZero()) + assert.False(t, JobCreationOptions{CompletedTimeFrom: time.Unix(1, 0)}.isZero()) + assert.False(t, JobCreationOptions{CompletedTimeTo: time.Unix(1, 0)}.isZero()) +} + +func TestJobCreationOptionsConfiguration(t *testing.T) { + t.Run("requires a destination", func(t *testing.T) { + _, err := JobCreationOptions{JobID: "job", Mode: ExportModeBatch}.configuration() + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "export destination is required") + }) + + t.Run("carries filter, format, and limits", func(t *testing.T) { + from := time.Now().UTC().Add(-time.Hour) + to := time.Now().UTC() + options, err := JobCreationOptions{ + JobID: "job", + Mode: ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: to, + Destination: &ExportDestination{Container: "container"}, + MaxInstancesPerBatch: 42, + }.Normalize() + require.NoError(t, err) + config, err := options.configuration() + require.NoError(t, err) + assert.Equal(t, ExportModeBatch, config.Mode) + assert.Equal(t, from, config.Filter.CompletedTimeFrom) + assert.Equal(t, to, config.Filter.CompletedTimeTo) + assert.Equal(t, TerminalStatuses(), config.Filter.RuntimeStatus) + assert.Equal(t, "container", config.Destination.Container) + assert.Equal(t, DefaultExportFormat(), config.Format) + assert.Equal(t, DefaultMaxParallelExports, config.MaxParallelExports) + assert.Equal(t, 42, config.MaxInstancesPerBatch) + }) +} + +// TestValidateJobIDRejectsControlCharacters keeps a job ID from producing an +// unusable entity or orchestration instance ID. +func TestValidateJobIDRejectsControlCharacters(t *testing.T) { + for _, jobID := range []string{"", " ", "a\x00b", "a\nb", "a\tb", "a\rb", "a@b"} { + require.ErrorIs(t, validateJobID(jobID), ErrValidation, "job ID %q must be rejected", jobID) + } + require.NoError(t, validateJobID("job-1")) + require.NoError(t, validateJobID(strings.Repeat("a", 128))) +} diff --git a/exporthistory/orchestrations.go b/exporthistory/orchestrations.go new file mode 100644 index 00000000..6ce637a7 --- /dev/null +++ b/exporthistory/orchestrations.go @@ -0,0 +1,387 @@ +package exporthistory + +import ( + "fmt" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/task" +) + +// Export orchestration tuning. These defaults mirror the .NET preview implementation. +const ( + // maxBatchRetryAttempts bounds how many times a page whose exports failed is + // retried before the job fails. + maxBatchRetryAttempts = 3 + // minBatchRetryBackoff is the delay before the first whole-page retry. Each + // further retry doubles it, so the reachable schedule is 1 then 2 minutes: + // the final attempt fails the page instead of waiting again. + minBatchRetryBackoff = time.Minute + // continueAsNewFrequency bounds how many pages one orchestration execution + // processes before continuing as new, keeping history small. + continueAsNewFrequency = 5 + // continuousIdleDelay is how long a continuous job waits after draining its + // backlog before listing again. + continuousIdleDelay = time.Minute +) + +// exportActivityRetryPolicy retries a single instance export before the page is +// declared failed. +func exportActivityRetryPolicy() *task.RetryPolicy { + return &task.RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 15 * time.Second, + BackoffCoefficient: 2, + MaxRetryInterval: time.Minute, + } +} + +// ExecuteExportJobOperationOrchestrator invokes one export job entity operation. +// It must remain unversioned because export clients explicitly target the +// unversioned system orchestrator. +func ExecuteExportJobOperationOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var request ExportJobOperationRequest + if err := ctx.GetInput(&request); err != nil { + return nil, err + } + if request.OperationName == "" { + return nil, &ValidationError{ + JobID: request.EntityID.Key, + Message: "operation name is required", + } + } + operation := ctx.CallEntity( + request.EntityID, + request.OperationName, + task.WithEntityInput(request.Input), + ) + // Non-JSON converters need the concrete result type, not *any. + if strings.EqualFold(request.OperationName, deleteOperation) { + var instanceID string + if err := operation.Await(&instanceID); err != nil { + return nil, err + } + return instanceID, nil + } + var result any + if err := operation.Await(&result); err != nil { + return nil, err + } + return result, nil +} + +// ExportJobOrchestrator performs the export work for one job: it lists terminal +// instances matching the job's filter, exports each instance's history, and +// commits a checkpoint after every page. +// +// It must remain unversioned because the entity starts it explicitly +// unversioned so it stays reachable when application default versioning is on. +func ExportJobOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input ExportJobRunRequest + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + jobID := input.JobEntityID.Key + logger := ctx.Logger() + + result, err := runExportJobOrchestration(ctx, input) + if err == nil { + return result, nil + } + logger.Error("export orchestrator failed", "jobId", jobID, "error", err) + // Best-effort: a job already moved out of Active by a failed checkpoint + // rejects MarkAsFailed, and that rejection must not mask the original error. + // The run token keeps the failure from landing on a newer generation. + markRequest := MarkAsFailedRequest{RunToken: input.RunToken, Error: err.Error()} + if markErr := callEntityVoid(ctx, input.JobEntityID, markAsFailedOperation, markRequest); markErr != nil { + logger.Warn("failed to mark export job as failed", "jobId", jobID, "error", markErr) + } + return nil, err +} + +func runExportJobOrchestration(ctx *task.OrchestrationContext, input ExportJobRunRequest) (any, error) { + jobID := input.JobEntityID.Key + logger := ctx.Logger() + + // A missing job is an error only on the very first execution, where it means + // the job was never readable. Once the run has continued as new the job may + // legitimately have been deleted underneath it, so it stops quietly. + firstExecution := !input.ContinuedExecution + runToken := input.RunToken + processedCycles := input.ProcessedCycles + for { + processedCycles++ + if processedCycles > continueAsNewFrequency { + ctx.ContinueAsNew( + ExportJobRunRequest{ + JobEntityID: input.JobEntityID, + RunToken: runToken, + ContinuedExecution: true, + }, + task.WithContinueAsNewVersion(task.UnversionedTaskVersion), + ) + return nil, nil + } + + // Re-read the job each cycle: it may have been deleted, failed by a + // checkpoint commit, or recreated with a different configuration. + current, err := fetchExportJobState(ctx, input.JobEntityID) + if err != nil { + return nil, err + } + if current == nil || current.Config == nil { + if firstExecution && processedCycles == 1 { + return nil, fmt.Errorf("export job %q not found or has no configuration", jobID) + } + logger.Warn("export job is no longer available", "jobId", jobID) + return nil, nil + } + // The job may have been deleted and recreated while this run was in + // flight. The new generation owns the job, so this run stops without + // touching it rather than checkpointing or completing it. + if !runTokenMatches(current, runToken) { + logger.Warn("export orchestrator stopping because its generation token does not match", + "jobId", jobID, "runToken", runToken, "currentToken", current.RunToken) + return nil, nil + } + if current.Status != ExportJobStatusActive { + logger.Warn("export orchestrator stopping because the job is not active", + "jobId", jobID, "status", current.Status.String()) + return nil, nil + } + config := *current.Config + + listRequest := ListTerminalInstancesRequest{ + CompletedTimeFrom: config.Filter.CompletedTimeFrom, + CompletedTimeTo: optionalTime(config.Filter.CompletedTimeTo), + RuntimeStatus: config.Filter.RuntimeStatus, + MaxInstancesPerBatch: config.MaxInstancesPerBatch, + } + if current.Checkpoint != nil { + listRequest.LastInstanceKey = current.Checkpoint.LastInstanceKey + } + + var page InstancePage + if err := ctx.CallActivity( + ListTerminalInstancesActivityName, + task.WithActivityInput(listRequest), + ).Await(&page); err != nil { + return nil, err + } + + if len(page.InstanceIDs) == 0 { + // A page can legitimately be empty while the task hub still has + // more pages, for example when a filtered scan exhausts a page + // without a match. Advancing the cursor keeps a batch job from + // completing while instances remain, and keeps a continuous job + // from re-reading the same empty page forever. + if page.NextCheckpoint != nil { + if err := commitBatchCheckpoint(ctx, input.JobEntityID, CommitCheckpointRequest{ + Checkpoint: page.NextCheckpoint, + RunToken: runToken, + }); err != nil { + return nil, err + } + continue + } + } else { + batch, err := processBatchWithRetry(ctx, jobID, page.InstanceIDs, config, maxBatchRetryAttempts) + if err != nil { + return nil, err + } + if len(batch.failures) > 0 { + // A failed page commits without a checkpoint so the cursor stays + // put and the entity implicitly fails the job. + if err := commitBatchCheckpoint(ctx, input.JobEntityID, CommitCheckpointRequest{ + Failures: batch.failures, + RunToken: runToken, + }); err != nil { + return nil, err + } + return nil, fmt.Errorf( + "export job %q batch export failed after %d retry attempts. Failure details: %s", + jobID, maxBatchRetryAttempts, summarizeFailures(batch.failures)) + } + + if err := commitBatchCheckpoint(ctx, input.JobEntityID, CommitCheckpointRequest{ + ScannedInstances: int64(len(page.InstanceIDs)), + ExportedInstances: int64(batch.exportedCount), + Checkpoint: page.NextCheckpoint, + RunToken: runToken, + }); err != nil { + return nil, err + } + } + + if page.NextCheckpoint != nil { + continue + } + // A page without a next checkpoint is the last one the task hub has. + // A continuous job re-lists from the last committed cursor after idling, + // which re-scans at most one page and overwrites deterministic blob names. + if config.Mode == ExportModeContinuous { + logger.Info("export job drained its backlog; waiting for new instances", "jobId", jobID) + if err := ctx.CreateTimer(continuousIdleDelay).Await(nil); err != nil { + return nil, err + } + continue + } + logger.Info("export job exported every matching instance", "jobId", jobID) + break + } + + completion := MarkAsCompletedRequest{RunToken: runToken} + if err := callEntityVoid(ctx, input.JobEntityID, markAsCompletedOperation, completion); err != nil { + return nil, err + } + logger.Info("export orchestrator completed", "jobId", jobID) + return nil, nil +} + +type batchExportResult struct { + exportedCount int + failures []ExportFailure +} + +func processBatchWithRetry( + ctx *task.OrchestrationContext, + jobID string, + instanceIDs []string, + config ExportJobConfiguration, + maxAttempts int, +) (batchExportResult, error) { + logger := ctx.Logger() + for attempt := 1; attempt <= maxAttempts; attempt++ { + results, err := exportBatch(ctx, instanceIDs, config) + if err != nil { + return batchExportResult{}, err + } + failed := make([]ExportResult, 0, len(results)) + succeeded := 0 + for _, result := range results { + if result.Success { + succeeded++ + continue + } + failed = append(failed, result) + } + if len(failed) == 0 { + logger.Info("export batch succeeded", + "jobId", jobID, "attempt", attempt, "exported", succeeded) + return batchExportResult{exportedCount: succeeded}, nil + } + logger.Warn("export batch failed", + "jobId", jobID, "attempt", attempt, "failures", len(failed), "instances", len(instanceIDs)) + if attempt == maxAttempts { + failures := make([]ExportFailure, 0, len(failed)) + for _, result := range failed { + reason := result.Error + if reason == "" { + reason = "unknown error" + } + failures = append(failures, ExportFailure{ + InstanceID: result.InstanceID, + Reason: reason, + AttemptCount: attempt, + LastAttempt: ctx.CurrentTimeUtc, + }) + } + return batchExportResult{exportedCount: succeeded, failures: failures}, nil + } + if err := ctx.CreateTimer(batchRetryBackoff(attempt)).Await(nil); err != nil { + return batchExportResult{}, err + } + } + // Reached only when maxAttempts is not positive, which means the page was + // never attempted. Reporting it as failed keeps the cursor on the page + // instead of committing a checkpoint that would silently skip it. + return batchExportResult{ + failures: []ExportFailure{{ + InstanceID: strings.Join(instanceIDs, ","), + Reason: fmt.Sprintf("the export batch was never attempted: %d retry attempts configured", maxAttempts), + LastAttempt: ctx.CurrentTimeUtc, + }}, + }, nil +} + +// batchRetryBackoff is the delay before retry number attempt+1, doubling from +// [minBatchRetryBackoff]. Only attempts 1 and 2 schedule a retry, because the +// final attempt fails the page, so the reachable schedule is 1 then 2 minutes. +func batchRetryBackoff(attempt int) time.Duration { + backoff := minBatchRetryBackoff + for i := 1; i < attempt; i++ { + backoff *= 2 + } + return backoff +} + +// exportBatch fans out per-instance exports in deterministic windows bounded by +// MaxParallelExports and preserves the input order of the results. +func exportBatch( + ctx *task.OrchestrationContext, + instanceIDs []string, + config ExportJobConfiguration, +) ([]ExportResult, error) { + parallelism := config.MaxParallelExports + if parallelism <= 0 { + parallelism = DefaultMaxParallelExports + } + results := make([]ExportResult, 0, len(instanceIDs)) + retryPolicy := exportActivityRetryPolicy() + for start := 0; start < len(instanceIDs); start += parallelism { + end := min(start+parallelism, len(instanceIDs)) + window := instanceIDs[start:end] + pending := make([]task.Task, 0, len(window)) + for _, instanceID := range window { + pending = append(pending, ctx.CallActivity( + ExportInstanceHistoryActivityName, + task.WithActivityInput(ExportRequest{ + InstanceID: instanceID, + Destination: config.Destination, + Format: config.Format, + }), + task.WithActivityRetryPolicy(retryPolicy), + )) + } + for i, pendingTask := range pending { + var result ExportResult + if err := pendingTask.Await(&result); err != nil { + // Every attempt of the activity failed, so record the instance as + // failed instead of aborting the page: other instances in the same + // window may still have succeeded and their failures matter too. + result = ExportResult{InstanceID: window[i], Success: false, Error: err.Error()} + } + if result.InstanceID == "" { + result.InstanceID = window[i] + } + results = append(results, result) + } + } + return results, nil +} + +func commitBatchCheckpoint( + ctx *task.OrchestrationContext, + entityID api.EntityID, + request CommitCheckpointRequest, +) error { + return callEntityVoid(ctx, entityID, commitCheckpointOperation, request) +} + +func fetchExportJobState(ctx *task.OrchestrationContext, entityID api.EntityID) (*ExportJobState, error) { + var state *ExportJobState + if err := ctx.CallEntity(entityID, getOperation, task.WithEntityInput(nil)).Await(&state); err != nil { + return nil, err + } + return state, nil +} + +func callEntityVoid( + ctx *task.OrchestrationContext, + entityID api.EntityID, + operation string, + input any, +) error { + return ctx.CallEntity(entityID, operation, task.WithEntityInput(input)).Await(nil) +} diff --git a/exporthistory/orchestrations_test.go b/exporthistory/orchestrations_test.go new file mode 100644 index 00000000..32b015be --- /dev/null +++ b/exporthistory/orchestrations_test.go @@ -0,0 +1,1131 @@ +package exporthistory + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/contextprop" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// entityMessage is an entity-directed message the scheduler would deliver to the +// target entity, paired with that target's instance ID. +type entityMessage struct { + historyEvent *protos.HistoryEvent + targetInstanceID string +} + +// orchestrationDriver replays an orchestration turn by turn against synthesized +// history, so timer-heavy paths such as batch retry backoff are covered without +// waiting on the wall clock. Each turn's actions are folded back into history the +// way the scheduler does, so replay stays faithful without a live service. +type orchestrationDriver struct { + t *testing.T + executor task.Executor + converter api.DataConverter + instanceID api.InstanceID + oldEvents []*protos.HistoryEvent + newEvents []*protos.HistoryEvent + + pendingTasks []*protos.HistoryEvent + pendingTimers []*protos.HistoryEvent + // createdTimers holds the TimerCreated history entries for the timers this + // turn scheduled, which carry the requested delay; pendingTimers holds the + // matching TimerFired messages the service would deliver back. + createdTimers []*protos.HistoryEvent + pendingEntities []entityMessage + completion *protos.CompleteOrchestrationAction + continuedAsNew bool +} + +func newOrchestrationDriver( + t *testing.T, + registry *task.TaskRegistry, + name string, + instanceID api.InstanceID, + input any, + converters ...api.DataConverter, +) *orchestrationDriver { + t.Helper() + converter := api.DefaultDataConverter() + if len(converters) > 0 { + converter = api.NormalizeDataConverter(converters[0]) + } + payload, err := api.SerializeData(converter, input) + require.NoError(t, err) + return &orchestrationDriver{ + t: t, + executor: task.NewTaskExecutor(registry, task.WithDataConverter(converter)), + converter: converter, + instanceID: instanceID, + newEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(name, string(instanceID), wrapperspb.String(payload), nil, nil, nil), + }, + } +} + +// turn executes one orchestration episode and folds the resulting actions into +// history. +func (d *orchestrationDriver) turn() { + d.t.Helper() + results, err := d.executor.ExecuteOrchestrator( + context.Background(), + d.instanceID, + d.oldEvents, + d.newEvents, + &protos.OrchestratorEntityParameters{EntityMessageReorderWindow: durationpb.New(0)}, + ) + require.NoError(d.t, err) + + history := append(append([]*protos.HistoryEvent{}, d.oldEvents...), d.newEvents...) + d.oldEvents = history + d.foldActions(results.Response.GetActions(), history) +} + +// foldActions appends the history each action produces and records the messages +// the scheduler would deliver back. Export orchestrations only schedule +// activities, call entities, create timers, and complete, so any other action is +// a gap in this harness rather than something to silently drop. +func (d *orchestrationDriver) foldActions( + actions []*protos.OrchestratorAction, + history []*protos.HistoryEvent, +) { + d.t.Helper() + d.newEvents = nil + d.pendingTasks = nil + d.pendingTimers = nil + d.pendingEntities = nil + d.createdTimers = nil + d.completion = nil + d.continuedAsNew = false + + started := latestExecutionStarted(d.t, history) + for _, action := range actions { + switch { + case action.GetScheduleTask() != nil: + scheduled := action.GetScheduleTask() + event := helpers.NewTaskScheduledEvent( + action.GetId(), scheduled.GetName(), scheduled.GetVersion(), scheduled.GetInput(), nil) + event.GetTaskScheduled().Tags = contextprop.Clone(scheduled.GetTags()) + d.newEvents = append(d.newEvents, event) + d.pendingTasks = append(d.pendingTasks, event) + + case action.GetCreateTimer() != nil: + fireAt := action.GetCreateTimer().GetFireAt() + created := helpers.NewTimerCreatedEvent(action.GetId(), fireAt) + d.newEvents = append(d.newEvents, created) + d.createdTimers = append(d.createdTimers, created) + d.pendingTimers = append(d.pendingTimers, helpers.NewTimerFiredEvent(action.GetId(), fireAt, nil)) + + case action.GetSendEntityMessage().GetEntityOperationCalled() != nil: + d.foldEntityCall(action, started) + + case action.GetCompleteOrchestration() != nil: + complete := action.GetCompleteOrchestration() + d.completion = complete + if complete.GetOrchestrationStatus() == protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW { + d.continuedAsNew = true + d.restart(started, complete) + return + } + d.newEvents = append(d.newEvents, &protos.HistoryEvent{ + EventId: action.GetId(), + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{ + OrchestrationStatus: complete.GetOrchestrationStatus(), + Result: complete.GetResult(), + FailureDetails: complete.GetFailureDetails(), + }, + }, + }) + + default: + require.FailNowf(d.t, "unsupported orchestrator action", "action %v", action) + } + } +} + +// latestExecutionStarted returns the start event of the execution the next turn +// belongs to, which is the last one in history after any ContinueAsNew. +func latestExecutionStarted(t *testing.T, history []*protos.HistoryEvent) *protos.ExecutionStartedEvent { + t.Helper() + var started *protos.ExecutionStartedEvent + for _, event := range history { + if candidate := event.GetExecutionStarted(); candidate != nil { + started = candidate + } + } + require.NotNil(t, started, "history has no ExecutionStarted event") + return started +} + +// foldEntityCall mirrors how the scheduler splits an entity call: the caller's +// history keeps the target instance ID, while the message delivered to the entity +// drops it and carries the calling execution's identity instead. +func (d *orchestrationDriver) foldEntityCall( + action *protos.OrchestratorAction, + started *protos.ExecutionStartedEvent, +) { + d.t.Helper() + historyValue := proto.Clone( + action.GetSendEntityMessage().GetEntityOperationCalled()).(*protos.EntityOperationCalledEvent) + target := historyValue.GetTargetInstanceId().GetValue() + require.NotEmpty(d.t, target, "entity call has no target instance ID") + historyValue.ParentInstanceId = nil + historyValue.ParentExecutionId = nil + + messageValue := proto.Clone(historyValue).(*protos.EntityOperationCalledEvent) + messageValue.TargetInstanceId = nil + messageValue.ParentInstanceId = wrapperspb.String(string(d.instanceID)) + messageValue.ParentExecutionId = started.GetOrchestrationInstance().GetExecutionId() + + timestamp := timestamppb.Now() + d.newEvents = append(d.newEvents, &protos.HistoryEvent{ + EventId: action.GetId(), + Timestamp: timestamp, + EventType: &protos.HistoryEvent_EntityOperationCalled{EntityOperationCalled: historyValue}, + }) + d.pendingEntities = append(d.pendingEntities, entityMessage{ + historyEvent: &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamp, + EventType: &protos.HistoryEvent_EntityOperationCalled{EntityOperationCalled: messageValue}, + }, + targetInstanceID: target, + }) +} + +// restart truncates history the way the scheduler does for ContinueAsNew: the new +// execution starts from a fresh start event carrying the previous execution's +// identity, the new input, and any carryover events. Work scheduled earlier in +// the same action batch does not survive the execution boundary. +func (d *orchestrationDriver) restart( + started *protos.ExecutionStartedEvent, + complete *protos.CompleteOrchestrationAction, +) { + d.t.Helper() + d.pendingTasks = nil + d.pendingTimers = nil + d.createdTimers = nil + d.pendingEntities = nil + + version := started.GetVersion() + if complete.GetNewVersion() != nil { + version = complete.GetNewVersion() + } + startEvent := helpers.NewExecutionStartedEvent( + started.GetName(), + string(d.instanceID), + complete.GetResult(), + started.GetParentInstance(), + started.GetParentTraceContext(), + nil, + version) + startEvent.GetExecutionStarted().Tags = contextprop.Clone(started.GetTags()) + + d.oldEvents = nil + d.newEvents = append([]*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + startEvent, + }, complete.GetCarryoverEvents()...) +} + +// nextTurn starts a new episode by appending the orchestrator-started marker +// that sets the deterministic current time. +func (d *orchestrationDriver) nextTurn() { + d.newEvents = append(d.newEvents, helpers.NewOrchestratorStartedEvent()) +} + +// completeEntityCall answers the pending entity call whose operation matches, +// with the supplied result. +func (d *orchestrationDriver) completeEntityCall(operation string, result any) { + d.t.Helper() + output := (*wrapperspb.StringValue)(nil) + if result != nil { + payload, err := api.SerializeData(d.converter, result) + require.NoError(d.t, err) + output = wrapperspb.String(payload) + } + d.completeEntityCallOutput(operation, output) +} + +func (d *orchestrationDriver) completeEntityCallOutput(operation string, output *wrapperspb.StringValue) { + d.t.Helper() + called := d.pendingEntityCall(operation) + d.newEvents = append(d.newEvents, &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{ + RequestId: called.RequestId, + Output: output, + }, + }, + }) +} + +// failEntityCall answers the pending entity call whose operation matches, with a +// failure carrying errorType. +func (d *orchestrationDriver) failEntityCall(operation, errorType, message string) { + d.t.Helper() + d.failEntityCallWithDetails(operation, &protos.TaskFailureDetails{ErrorType: errorType, ErrorMessage: message}) +} + +func (d *orchestrationDriver) failEntityCallWithDetails(operation string, details *protos.TaskFailureDetails) { + d.t.Helper() + called := d.pendingEntityCall(operation) + d.newEvents = append(d.newEvents, &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationFailed{ + EntityOperationFailed: &protos.EntityOperationFailedEvent{ + RequestId: called.RequestId, + FailureDetails: details, + }, + }, + }) +} + +func (d *orchestrationDriver) pendingEntityCall(operation string) *protos.EntityOperationCalledEvent { + d.t.Helper() + called, _ := d.pendingEntityMessage(operation) + return called +} + +func (d *orchestrationDriver) pendingEntityMessage( + operation string, +) (*protos.EntityOperationCalledEvent, string) { + d.t.Helper() + for _, message := range d.pendingEntities { + called := message.historyEvent.GetEntityOperationCalled() + if called != nil && called.GetOperation() == operation { + return called, message.targetInstanceID + } + } + require.FailNowf(d.t, "no pending entity call", "operation %q", operation) + return nil, "" +} + +// entityCallInput returns the serialized input of the pending call for operation. +func (d *orchestrationDriver) entityCallInput(operation string, target any) { + d.t.Helper() + called := d.pendingEntityCall(operation) + require.NoError(d.t, d.converter.Deserialize(called.GetInput().GetValue(), target)) +} + +// completeActivities answers every pending activity call using results keyed by +// the activity name, in scheduling order. +func (d *orchestrationDriver) completeActivities(results func(name string, index int) any) { + d.t.Helper() + counts := map[string]int{} + for _, pending := range d.pendingTasks { + scheduled := pending.GetTaskScheduled() + require.NotNil(d.t, scheduled) + name := scheduled.GetName() + index := counts[name] + counts[name]++ + payload, err := api.SerializeData(d.converter, results(name, index)) + require.NoError(d.t, err) + d.newEvents = append(d.newEvents, + helpers.NewTaskCompletedEvent(pending.GetEventId(), wrapperspb.String(payload))) + } +} + +// fireTimers delivers every pending durable timer and returns the delay each one +// was created with. +func (d *orchestrationDriver) fireTimers() []time.Duration { + d.t.Helper() + delays := make([]time.Duration, 0, len(d.pendingTimers)) + for _, created := range d.createdTimers { + delays = append(delays, + created.GetTimerCreated().GetFireAt().AsTime().Sub(created.GetTimestamp().AsTime())) + } + d.newEvents = append(d.newEvents, d.pendingTimers...) + return delays +} + +func (d *orchestrationDriver) activityInput(name string, target any) { + d.t.Helper() + for _, pending := range d.pendingTasks { + scheduled := pending.GetTaskScheduled() + if scheduled != nil && scheduled.GetName() == name { + require.NoError(d.t, d.converter.Deserialize(scheduled.GetInput().GetValue(), target)) + return + } + } + require.FailNowf(d.t, "no pending activity", "name %q", name) +} + +// assertDelay compares a durable timer delay allowing for the small skew between +// the orchestrator's deterministic clock and the timestamp the driver stamps on +// the emitted TimerCreated event. +func assertDelay(t *testing.T, expected, actual time.Duration) { + t.Helper() + assert.InDelta(t, expected.Seconds(), actual.Seconds(), 1, "expected a %s timer", expected) +} + +func newExportRegistry(t *testing.T) *task.TaskRegistry { + t.Helper() + registry := task.NewTaskRegistry() + require.NoError(t, Register(registry, WorkerOptions{Source: newFakeSource(), Store: newMemoryStore()})) + return registry +} + +func activeJobState(t *testing.T, mode ExportMode, checkpoint string, runToken ...string) ExportJobState { + t.Helper() + options := batchOptions("job-1") + if mode == ExportModeContinuous { + var err error + options, err = JobCreationOptions{ + JobID: "job-1", + Mode: ExportModeContinuous, + Destination: &ExportDestination{Container: "test-container"}, + }.Normalize() + require.NoError(t, err) + } + config, err := options.configuration() + require.NoError(t, err) + state := ExportJobState{Status: ExportJobStatusActive, Config: config, RunToken: "run-a"} + if checkpoint != "" { + state.Checkpoint = &ExportCheckpoint{LastInstanceKey: checkpoint} + } + if len(runToken) > 0 { + state.RunToken = runToken[0] + } + return state +} + +func startExportOrchestrationDriver(t *testing.T, input ...ExportJobRunRequest) *orchestrationDriver { + t.Helper() + request := ExportJobRunRequest{JobEntityID: EntityID("job-1"), RunToken: "run-a"} + if len(input) > 0 { + request = input[0] + } + return newOrchestrationDriver( + t, + newExportRegistry(t), + ExportJobOrchestratorName, + api.InstanceID(OrchestratorInstanceIDPrefix+"job-1-"+request.RunToken), + request, + ) +} + +// TestExportJobOrchestratorBatchCompletes covers the happy path: one page of +// instances is exported, a checkpoint is committed, and the empty follow-up page +// completes the job. +func TestExportJobOrchestratorBatchCompletes(t *testing.T) { + driver := startExportOrchestrationDriver(t) + + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "")) + + driver.nextTurn() + driver.turn() + var listRequest ListTerminalInstancesRequest + driver.activityInput(ListTerminalInstancesActivityName, &listRequest) + assert.Equal(t, DefaultMaxInstancesPerBatch, listRequest.MaxInstancesPerBatch) + assert.Equal(t, TerminalStatuses(), listRequest.RuntimeStatus) + assert.Empty(t, listRequest.LastInstanceKey) + driver.completeActivities(func(string, int) any { + return InstancePage{ + InstanceIDs: []string{"i1", "i2"}, + NextCheckpoint: &ExportCheckpoint{LastInstanceKey: "cursor-1"}, + } + }) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 2) + var exportRequest ExportRequest + driver.activityInput(ExportInstanceHistoryActivityName, &exportRequest) + assert.Equal(t, "test-container", exportRequest.Destination.Container) + assert.Equal(t, DefaultExportFormat(), exportRequest.Format) + driver.completeActivities(func(_ string, index int) any { + return ExportResult{InstanceID: []string{"i1", "i2"}[index], Success: true} + }) + + driver.nextTurn() + driver.turn() + var commit CommitCheckpointRequest + driver.entityCallInput(commitCheckpointOperation, &commit) + assert.Equal(t, int64(2), commit.ScannedInstances) + assert.Equal(t, int64(2), commit.ExportedInstances) + require.NotNil(t, commit.Checkpoint) + assert.Equal(t, "cursor-1", commit.Checkpoint.LastInstanceKey) + assert.Empty(t, commit.Failures) + driver.completeEntityCall(commitCheckpointOperation, nil) + + // Second cycle: the job re-reads its state and the advanced cursor is used. + driver.nextTurn() + driver.turn() + state := activeJobState(t, ExportModeBatch, "cursor-1") + state.ScannedInstances, state.ExportedInstances = 2, 2 + driver.completeEntityCall(getOperation, state) + + driver.nextTurn() + driver.turn() + driver.activityInput(ListTerminalInstancesActivityName, &listRequest) + assert.Equal(t, "cursor-1", listRequest.LastInstanceKey) + driver.completeActivities(func(string, int) any { return InstancePage{} }) + + driver.nextTurn() + driver.turn() + require.NotEmpty(t, driver.pendingEntities) + driver.completeEntityCall(markAsCompletedOperation, nil) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, driver.completion.GetOrchestrationStatus()) +} + +// TestExportJobOrchestratorCompletesOnTheFinalPage covers a non-empty last page, +// which a task hub signals by omitting the continuation token. The job must +// commit its progress without resetting the cursor and then complete. +func TestExportJobOrchestratorCompletesOnTheFinalPage(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "cursor-1")) + + driver.nextTurn() + driver.turn() + // The final page carries instances but no next checkpoint. + driver.completeActivities(func(string, int) any { + return InstancePage{InstanceIDs: []string{"i1"}} + }) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return ExportResult{InstanceID: "i1", Success: true} + }) + + driver.nextTurn() + driver.turn() + var commit CommitCheckpointRequest + driver.entityCallInput(commitCheckpointOperation, &commit) + assert.Equal(t, int64(1), commit.ScannedInstances) + assert.Equal(t, int64(1), commit.ExportedInstances) + assert.Nil(t, commit.Checkpoint, "the backend cursor must remain opaque") + assert.Empty(t, commit.Failures) + driver.completeEntityCall(commitCheckpointOperation, nil) + + // The job completes without listing again. + driver.nextTurn() + driver.turn() + assert.Empty(t, driver.pendingTasks) + driver.completeEntityCall(markAsCompletedOperation, nil) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, driver.completion.GetOrchestrationStatus()) +} + +// TestExportJobOrchestratorAdvancesPastAnEmptyPage covers a task hub that +// returns an empty page while more pages remain, which a filtered scan can +// produce. The cursor must advance so a batch job does not complete early and a +// continuous job does not re-read the same page forever. +func TestExportJobOrchestratorAdvancesPastAnEmptyPage(t *testing.T) { + for _, mode := range []ExportMode{ExportModeBatch, ExportModeContinuous} { + t.Run(mode.String(), func(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, mode, "cursor-1")) + + driver.nextTurn() + driver.turn() + var listRequest ListTerminalInstancesRequest + driver.activityInput(ListTerminalInstancesActivityName, &listRequest) + assert.Equal(t, "cursor-1", listRequest.LastInstanceKey) + driver.completeActivities(func(string, int) any { + return InstancePage{NextCheckpoint: &ExportCheckpoint{LastInstanceKey: "cursor-2"}} + }) + + driver.nextTurn() + driver.turn() + // The empty page still moves the durable cursor forward. + var commit CommitCheckpointRequest + driver.entityCallInput(commitCheckpointOperation, &commit) + require.NotNil(t, commit.Checkpoint) + assert.Equal(t, "cursor-2", commit.Checkpoint.LastInstanceKey) + assert.Zero(t, commit.ScannedInstances) + assert.Zero(t, commit.ExportedInstances) + assert.Empty(t, commit.Failures) + assert.Empty(t, driver.pendingTimers, "an empty page with more pages must not idle") + assert.Nil(t, driver.completion, "an empty page with more pages must not complete the job") + driver.completeEntityCall(commitCheckpointOperation, nil) + + // The next cycle resumes from the advanced cursor. + driver.nextTurn() + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, mode, "cursor-2")) + driver.nextTurn() + driver.turn() + driver.activityInput(ListTerminalInstancesActivityName, &listRequest) + assert.Equal(t, "cursor-2", listRequest.LastInstanceKey) + }) + } +} + +// TestExportJobOrchestratorContinuousIdlesOnTheFinalPage keeps a continuous job +// running after it drains a non-empty last page. +func TestExportJobOrchestratorContinuousIdlesOnTheFinalPage(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeContinuous, "cursor-1")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return InstancePage{InstanceIDs: []string{"i1"}} + }) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return ExportResult{InstanceID: "i1", Success: true} + }) + + driver.nextTurn() + driver.turn() + driver.completeEntityCall(commitCheckpointOperation, nil) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTimers, 1) + assertDelay(t, continuousIdleDelay, driver.fireTimers()[0]) + assert.Nil(t, driver.completion) +} + +// TestExportJobOrchestratorContinuousIdles covers a continuous job waiting on a +// durable timer instead of completing when a page comes back empty. +func TestExportJobOrchestratorContinuousIdles(t *testing.T) { + driver := startExportOrchestrationDriver(t) + + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeContinuous, "")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { return InstancePage{} }) + + driver.nextTurn() + driver.turn() + // The job idles instead of completing or marking itself completed. + assert.Nil(t, driver.completion) + assert.Empty(t, driver.pendingEntities) + delays := driver.fireTimers() + require.Len(t, delays, 1) + assertDelay(t, continuousIdleDelay, delays[0]) + + // After the idle delay it re-reads the job and lists again. + driver.nextTurn() + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeContinuous, "")) + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 1) + assert.Equal(t, ListTerminalInstancesActivityName, driver.pendingTasks[0].GetTaskScheduled().GetName()) +} + +// TestExportJobOrchestratorRetriesAndFailsBatch covers whole-page retry with +// exponential backoff, the checkpoint-free commit that implicitly fails the job, +// and the terminal error's failure detail. +func TestExportJobOrchestratorRetriesAndFailsBatch(t *testing.T) { + driver := startExportOrchestrationDriver(t) + + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return InstancePage{ + InstanceIDs: []string{"i1", "i2"}, + NextCheckpoint: &ExportCheckpoint{LastInstanceKey: "cursor-1"}, + } + }) + + backoffs := []time.Duration{} + for attempt := 1; attempt <= maxBatchRetryAttempts; attempt++ { + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 2, "attempt %d", attempt) + driver.completeActivities(func(_ string, index int) any { + if index == 0 { + return ExportResult{InstanceID: "i1", Success: true} + } + return ExportResult{InstanceID: "i2", Success: false, Error: "upload rejected"} + }) + if attempt == maxBatchRetryAttempts { + break + } + driver.nextTurn() + driver.turn() + backoffs = append(backoffs, driver.fireTimers()...) + } + require.Len(t, backoffs, 2) + assertDelay(t, minBatchRetryBackoff, backoffs[0]) + assertDelay(t, 2*minBatchRetryBackoff, backoffs[1]) + + driver.nextTurn() + driver.turn() + var commit CommitCheckpointRequest + driver.entityCallInput(commitCheckpointOperation, &commit) + // The cursor must not move so the same page can be retried after a fix. + assert.Nil(t, commit.Checkpoint) + assert.Zero(t, commit.ScannedInstances) + assert.Zero(t, commit.ExportedInstances) + require.Len(t, commit.Failures, 1) + assert.Equal(t, "i2", commit.Failures[0].InstanceID) + assert.Equal(t, "upload rejected", commit.Failures[0].Reason) + assert.Equal(t, maxBatchRetryAttempts, commit.Failures[0].AttemptCount) + assert.False(t, commit.Failures[0].LastAttempt.IsZero()) + driver.completeEntityCall(commitCheckpointOperation, nil) + + // The orchestration then tries to mark the job failed and fails itself. + driver.nextTurn() + driver.turn() + var failure MarkAsFailedRequest + driver.entityCallInput(markAsFailedOperation, &failure) + assert.Contains(t, failure.Error, "batch export failed after 3 retry attempts") + assert.Contains(t, failure.Error, "InstanceId: i2, Reason: upload rejected") + driver.completeEntityCall(markAsFailedOperation, nil) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, driver.completion.GetOrchestrationStatus()) + assert.Contains(t, driver.completion.GetFailureDetails().GetErrorMessage(), "batch export failed") +} + +// TestExportJobOrchestratorTreatsActivityFailureAsInstanceFailure keeps one +// instance whose export activity exhausted its own retries from aborting the +// page, so every failing instance in the page is reported. +func TestExportJobOrchestratorTreatsActivityFailureAsInstanceFailure(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return InstancePage{ + InstanceIDs: []string{"i1", "i2"}, + NextCheckpoint: &ExportCheckpoint{LastInstanceKey: "cursor-1"}, + } + }) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 2) + // The first instance succeeds; the second activity fails outright. + success, err := json.Marshal(ExportResult{InstanceID: "i1", Success: true}) + require.NoError(t, err) + driver.newEvents = append(driver.newEvents, + helpers.NewTaskCompletedEvent(driver.pendingTasks[0].GetEventId(), wrapperspb.String(string(success))), + helpers.NewTaskFailedEvent(driver.pendingTasks[1].GetEventId(), &protos.TaskFailureDetails{ + ErrorType: "Contoso.Boom", + ErrorMessage: "activity exhausted its retries", + IsNonRetriable: true, + })) + + driver.nextTurn() + driver.turn() + // The page is retried rather than abandoned, which proves the failure was + // collected instead of propagated. + require.Len(t, driver.pendingTimers, 1) +} + +// TestExportJobOrchestratorStopsWhenTheJobIsNoLongerActive covers a job deleted +// or failed underneath a running orchestration. +func TestExportJobOrchestratorStopsWhenTheJobIsNoLongerActive(t *testing.T) { + tests := []struct { + name string + state any + }{ + {"failed by a checkpoint commit", ExportJobState{Status: ExportJobStatusFailed, Config: &ExportJobConfiguration{}, RunToken: "run-a"}}, + {"completed", ExportJobState{Status: ExportJobStatusCompleted, Config: &ExportJobConfiguration{}, RunToken: "run-a"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, test.state) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + // A stopped orchestration must not touch the job again. + assert.Empty(t, driver.pendingEntities) + assert.Empty(t, driver.pendingTasks) + }) + } +} + +// TestExportJobOrchestratorFailsWhenTheJobIsMissing covers a first cycle that +// cannot find the job at all. +func TestExportJobOrchestratorFailsWhenTheJobIsMissing(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, nil) + + driver.nextTurn() + driver.turn() + var failure MarkAsFailedRequest + driver.entityCallInput(markAsFailedOperation, &failure) + assert.Contains(t, failure.Error, "not found or has no configuration") + driver.completeEntityCall(markAsFailedOperation, nil) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, driver.completion.GetOrchestrationStatus()) +} + +// TestExportJobOrchestratorMarkAsFailedRejectionDoesNotMaskTheCause covers the +// interplay between a checkpoint-driven implicit failure and the orchestration's +// own best-effort MarkAsFailed. +func TestExportJobOrchestratorMarkAsFailedRejectionDoesNotMaskTheCause(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + driver.completeEntityCall(getOperation, nil) + + driver.nextTurn() + driver.turn() + driver.failEntityCall(markAsFailedOperation, string(invalidTransitionErrorType), "job is already failed") + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, driver.completion.GetOrchestrationStatus()) + assert.Contains(t, driver.completion.GetFailureDetails().GetErrorMessage(), "not found or has no configuration") + assert.NotContains(t, driver.completion.GetFailureDetails().GetErrorMessage(), "already failed") +} + +// TestExportJobOrchestratorContinuesAsNew keeps orchestration history bounded +// for long-running continuous jobs. +func TestExportJobOrchestratorContinuesAsNew(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + }) + for cycle := 1; cycle <= continueAsNewFrequency; cycle++ { + if cycle > 1 { + driver.nextTurn() + } + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeContinuous, "", "run-a")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { return InstancePage{} }) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTimers, 1, "cycle %d", cycle) + driver.fireTimers() + } + + driver.nextTurn() + driver.turn() + assert.True(t, driver.continuedAsNew) + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + driver.completion.GetOrchestrationStatus()) + + var request ExportJobRunRequest + require.NoError(t, json.Unmarshal([]byte(driver.completion.GetResult().GetValue()), &request)) + assert.Equal(t, EntityID("job-1"), request.JobEntityID) + // The cycle counter resets so the next execution gets a full budget. + assert.Zero(t, request.ProcessedCycles) + // The run generation carries forward, so the continued execution keeps + // fencing its mutations to the job it was started for. + assert.Equal(t, "run-a", request.RunToken) + // The continued execution is no longer the first one, so a job that + // disappears underneath it is not an error. + assert.True(t, request.ContinuedExecution) +} + +// TestExportJobOrchestratorContinuedExecutionToleratesAMissingJob covers the +// explicit first-execution flag: after a ContinueAsNew the cycle counter resets, +// so a job deleted underneath the run must still stop quietly rather than fail. +func TestExportJobOrchestratorContinuedExecutionToleratesAMissingJob(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + ContinuedExecution: true, + }) + driver.turn() + // The very first cycle of this execution finds nothing, which for a + // continued execution means the job was deleted while it ran. + driver.completeEntityCall(getOperation, nil) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + // A deleted job must not be marked failed, which would resurrect it. + assert.Empty(t, driver.pendingEntities) +} + +// TestExportJobOrchestratorStopsWhenTheJobWasRecreated covers run fencing from +// the orchestration side: a run left over from a deleted-and-recreated job sees +// a different generation token and stops without touching the new job. +func TestExportJobOrchestratorStopsWhenTheJobWasRecreated(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + }) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "", "run-b")) + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + // The stale run must neither list, checkpoint, complete, nor fail the new + // generation of the job. + assert.Empty(t, driver.pendingEntities) + assert.Empty(t, driver.pendingTasks) +} + +// TestExportJobOrchestratorFencesItsMutations proves every entity mutation the +// run performs carries the generation token it was started for. +func TestExportJobOrchestratorFencesItsMutations(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + }) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "", "run-a")) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return InstancePage{InstanceIDs: []string{"i1"}} + }) + + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return ExportResult{InstanceID: "i1", Success: true} + }) + + driver.nextTurn() + driver.turn() + var commit CommitCheckpointRequest + driver.entityCallInput(commitCheckpointOperation, &commit) + assert.Equal(t, "run-a", commit.RunToken) + driver.completeEntityCall(commitCheckpointOperation, nil) + + driver.nextTurn() + driver.turn() + var completion MarkAsCompletedRequest + driver.entityCallInput(markAsCompletedOperation, &completion) + assert.Equal(t, "run-a", completion.RunToken) +} + +func TestExportJobOrchestratorRequiresMatchingNonemptyTokens(t *testing.T) { + for _, tokens := range []struct { + name string + request string + state string + }{ + {"missing request token", "", "run-a"}, + {"missing state token", "run-a", ""}, + {"both tokens missing", "", ""}, + {"different tokens", "run-a", "run-b"}, + } { + t.Run(tokens.name, func(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: tokens.request, + }) + driver.turn() + driver.completeEntityCall(getOperation, activeJobState(t, ExportModeBatch, "", tokens.state)) + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + assert.Empty(t, driver.pendingEntities) + assert.Empty(t, driver.pendingTasks) + assert.Empty(t, driver.pendingTimers) + }) + } +} + +// TestExportJobOrchestratorFailureCarriesTheRunToken keeps a failing run from +// failing a newer generation of the same job. +func TestExportJobOrchestratorFailureCarriesTheRunToken(t *testing.T) { + driver := startExportOrchestrationDriver(t, ExportJobRunRequest{ + JobEntityID: EntityID("job-1"), + RunToken: "run-a", + }) + driver.turn() + driver.completeEntityCall(getOperation, nil) + + driver.nextTurn() + driver.turn() + var failure MarkAsFailedRequest + driver.entityCallInput(markAsFailedOperation, &failure) + assert.Equal(t, "run-a", failure.RunToken) + assert.Contains(t, failure.Error, "not found or has no configuration") +} + +// TestExportJobOrchestratorFansOutWithinMaxParallelExports keeps a large page +// from scheduling unbounded concurrent activities. +func TestExportJobOrchestratorFansOutWithinMaxParallelExports(t *testing.T) { + driver := startExportOrchestrationDriver(t) + driver.turn() + state := activeJobState(t, ExportModeBatch, "") + state.Config.MaxParallelExports = 3 + driver.completeEntityCall(getOperation, state) + + instances := []string{"i1", "i2", "i3", "i4", "i5"} + driver.nextTurn() + driver.turn() + driver.completeActivities(func(string, int) any { + return InstancePage{InstanceIDs: instances, NextCheckpoint: &ExportCheckpoint{}} + }) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 3) + driver.completeActivities(func(_ string, index int) any { + return ExportResult{InstanceID: instances[index], Success: true} + }) + + driver.nextTurn() + driver.turn() + require.Len(t, driver.pendingTasks, 2) +} + +// TestExecuteExportJobOperationOrchestrator ports the upstream +// ExecuteExportJobOperationOrchestratorTests matrix. +func TestExecuteExportJobOperationOrchestrator(t *testing.T) { + newDriver := func(t *testing.T, request ExportJobOperationRequest) *orchestrationDriver { + return newOrchestrationDriver( + t, + newExportRegistry(t), + ExecuteExportJobOperationOrchestratorName, + "operation-instance", + request, + ) + } + + t.Run("forwards the operation and input to the entity", func(t *testing.T) { + options := batchOptions("job-1") + driver := newDriver(t, ExportJobOperationRequest{ + EntityID: EntityID("job-1"), + OperationName: createOperation, + Input: options, + }) + driver.turn() + _, target := driver.pendingEntityMessage(createOperation) + assert.Equal(t, EntityID("job-1").String(), target) + var forwarded JobCreationOptions + driver.entityCallInput(createOperation, &forwarded) + assert.Equal(t, options.JobID, forwarded.JobID) + assert.Equal(t, options.Mode, forwarded.Mode) + + driver.completeEntityCall(createOperation, ExportJobState{Status: ExportJobStatusActive}) + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + assert.Contains(t, driver.completion.GetResult().GetValue(), `"Status":1`) + }) + + t.Run("forwards an operation without input", func(t *testing.T) { + driver := newDriver(t, ExportJobOperationRequest{ + EntityID: EntityID("job-1"), + OperationName: getOperation, + }) + driver.turn() + called := driver.pendingEntityCall(getOperation) + assert.Empty(t, called.GetInput().GetValue()) + + driver.completeEntityCall(getOperation, nil) + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + driver.completion.GetOrchestrationStatus()) + }) + + t.Run("preserves typed delete results for case-insensitive operation names", func(t *testing.T) { + converter := lifecycleGobConverter{} + for _, operation := range []string{deleteOperation, "dElEtE"} { + for _, instanceID := range []string{"ExportJob-job-1-run-a", ""} { + driver := newOrchestrationDriver(t, newExportRegistry(t), + ExecuteExportJobOperationOrchestratorName, "operation-instance", + ExportJobOperationRequest{EntityID: EntityID("job-1"), OperationName: operation}, + converter) + driver.turn() + driver.completeEntityCall(operation, instanceID) + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, driver.completion.GetOrchestrationStatus()) + var result string + require.NoError(t, converter.Deserialize(driver.completion.GetResult().GetValue(), &result)) + assert.Equal(t, instanceID, result) + } + } + }) + + t.Run("propagates entity failures", func(t *testing.T) { + driver := newDriver(t, ExportJobOperationRequest{ + EntityID: EntityID("job-1"), + OperationName: createOperation, + Input: batchOptions("job-1"), + }) + driver.turn() + driver.failEntityCall(createOperation, string(invalidTransitionErrorType), "cannot recreate an active job") + + driver.nextTurn() + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + driver.completion.GetOrchestrationStatus()) + assert.Contains(t, driver.completion.GetFailureDetails().GetErrorMessage(), "cannot recreate an active job") + }) + + t.Run("requires an operation name", func(t *testing.T) { + driver := newDriver(t, ExportJobOperationRequest{EntityID: EntityID("job-1")}) + driver.turn() + require.NotNil(t, driver.completion) + assert.Equal(t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + driver.completion.GetOrchestrationStatus()) + assert.Equal(t, string(validationErrorType), driver.completion.GetFailureDetails().GetErrorType()) + }) +} diff --git a/exporthistory/registration.go b/exporthistory/registration.go new file mode 100644 index 00000000..0954f36b --- /dev/null +++ b/exporthistory/registration.go @@ -0,0 +1,93 @@ +package exporthistory + +import ( + "fmt" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/task" +) + +// WorkerOptions configures the export history system tasks a worker runs. +type WorkerOptions struct { + // Source supplies the management reads the export activities perform. It is + // required and is normally the same task hub client the worker connects with. + Source HistorySource + // Store persists exported objects. It is required. + Store Store + // HistoryQuery bounds each instance-history read. Zero values select the + // api package defaults. + HistoryQuery api.HistoryQuery +} + +// Register adds the export history system entity, orchestrators, and activities +// to registry. +// +// Every system task is registered unversioned so it stays reachable when an +// application enables default versioning; pair it with [WithExportHistory] so +// the worker's derived work-item filters keep accepting them under strict +// version matching. +func Register(registry *task.TaskRegistry, options WorkerOptions) error { + if registry == nil { + return &ValidationError{Message: "task registry is required"} + } + if options.Source == nil { + return &ValidationError{Message: "export history worker requires a history source"} + } + if options.Store == nil { + return &ValidationError{Message: "export history worker requires a store"} + } + historyQuery, err := api.NormalizeHistoryQuery(options.HistoryQuery) + if err != nil { + return fmt.Errorf("invalid export history query: %w", err) + } + runtime := &exportRuntime{ + source: options.Source, + store: options.Store, + historyPage: historyQuery, + } + + if err := registry.AddEntityN(ExportJobEntityName, exportJobEntity); err != nil { + return err + } + if err := registry.AddOrchestratorN( + ExecuteExportJobOperationOrchestratorName, + ExecuteExportJobOperationOrchestrator, + ); err != nil { + return err + } + if err := registry.AddOrchestratorN(ExportJobOrchestratorName, ExportJobOrchestrator); err != nil { + return err + } + if err := registry.AddActivityN( + ListTerminalInstancesActivityName, + runtime.listTerminalInstancesActivity, + ); err != nil { + return err + } + return registry.AddActivityN( + ExportInstanceHistoryActivityName, + runtime.exportInstanceHistoryActivity, + ) +} + +// WithExportHistory marks the export history system orchestrators and +// activities as unversioned so they remain routable when the worker runs with +// strict version matching, and so registry-derived work-item filters advertise +// them with the version they were registered under. +// +// The export orchestrations are started unversioned, and an activity inherits +// its caller's version, so both kinds must be allow-listed. Use it together +// with [Register]; on its own it changes no behavior. +func WithExportHistory() durabletaskclient.TaskHubGrpcWorkerOption { + return durabletaskclient.CombineTaskHubGrpcWorkerOptions( + durabletaskclient.WithUnversionedOrchestratorNames( + ExecuteExportJobOperationOrchestratorName, + ExportJobOrchestratorName, + ), + durabletaskclient.WithUnversionedActivityNames( + ListTerminalInstancesActivityName, + ExportInstanceHistoryActivityName, + ), + ) +} diff --git a/exporthistory/storage.go b/exporthistory/storage.go new file mode 100644 index 00000000..a8741ade --- /dev/null +++ b/exporthistory/storage.go @@ -0,0 +1,447 @@ +package exporthistory + +import ( + "bytes" + "compress/gzip" + "context" + "fmt" + "net" + "net/url" + "strings" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" +) + +// ExportObject is one serialized orchestration history ready to be persisted. +type ExportObject struct { + // Container is the destination container name. + Container string + // Name is the full object path inside the container, prefix included. + Name string + // Content is the object body, already compressed when the format requires it. + Content []byte + // ContentType is the MIME type of Content. A compressed format reports the + // compressed type, such as application/gzip, rather than declaring the + // compression as a separate content coding, so a reader always receives the + // exact bytes the object name describes. + ContentType string + // Metadata is attached to the stored object. + Metadata map[string]string +} + +// Store persists exported history objects. It is deliberately narrower than +// [github.com/microsoft/durabletask-go/payload.AzureBlobStore], whose +// large-payload contract assigns random object names and owns a single +// container, because export jobs choose deterministic names in a caller-chosen +// container. +// +// Implementations must be safe for concurrent use and should treat a repeated +// write of the same object name as an overwrite, since an activity retry can +// re-export an instance whose object already exists. +type Store interface { + Write(ctx context.Context, object ExportObject) error +} + +// AzureBlobHistoryStoreOptions configures [AzureBlobHistoryStore]. Exactly one authentication +// mode must be supplied: ConnectionString, or AccountURL with Credential. +type AzureBlobHistoryStoreOptions struct { + ConnectionString string + AccountURL string + Credential azcore.TokenCredential + // ContainerName is the default destination container. It is required so a + // misconfigured worker fails at construction instead of at export time. + ContainerName string + // AllowInsecureHTTP permits plaintext endpoints such as Azurite. + AllowInsecureHTTP bool + // AllowedContainers optionally restricts the containers an export job may + // write to. When empty only ContainerName is allowed. + AllowedContainers []string + // AllowAnyContainer disables container allow-listing entirely. + AllowAnyContainer bool +} + +// AzureBlobHistoryStore writes exported history objects to Azure Blob Storage. +type AzureBlobHistoryStore struct { + client *azblob.Client + defaultContainer string + allowedContainers map[string]struct{} + allowAnyContainer bool + + containersMu sync.Mutex + // containers records the container initializations this process already + // started, so a job exporting thousands of instances issues one create call + // per container. An entry is removed before its initialization is reported + // as failed, so a waiter never mistakes a failure for a success. + containers map[string]*containerInit + + // Narrow hooks stand in for the two *azblob.Client calls whose ordering and + // retry behavior matter. Both are nil in production. + createContainerHook func(ctx context.Context, container string) error + uploadBlobHook func(ctx context.Context, container, name string, body []byte, options *azblob.UploadBufferOptions) error + waitHook func(ctx context.Context, d time.Duration) error +} + +// containerInit is one in-flight or completed container initialization. err is +// written before done is closed, so a waiter that observes the close also +// observes the result. +type containerInit struct { + done chan struct{} + err error +} + +var _ Store = (*AzureBlobHistoryStore)(nil) + +// Azure keeps a deleted container's name reserved until the delete finishes, +// rejecting recreation with ContainerBeingDeleted for up to about half a +// minute. Creation therefore retries with a capped exponential backoff. +const ( + containerBeingDeletedAttempts = 9 + containerBeingDeletedInitialBackoff = 250 * time.Millisecond + containerBeingDeletedMaxBackoff = 8 * time.Second +) + +// NewAzureBlobHistoryStore constructs a production Azure Blob Storage export store. +func NewAzureBlobHistoryStore(options AzureBlobHistoryStoreOptions) (*AzureBlobHistoryStore, error) { + if strings.TrimSpace(options.ContainerName) == "" { + return nil, &ValidationError{Message: "azure blob container name is required"} + } + if !validBlobContainerName(options.ContainerName) { + return nil, &ValidationError{ + Message: fmt.Sprintf("azure blob container name %q is not valid", options.ContainerName), + } + } + if (options.ConnectionString == "") == (options.AccountURL == "") { + return nil, &ValidationError{Message: "exactly one of connection string or account URL is required"} + } + if options.ConnectionString == "" && options.Credential == nil { + return nil, &ValidationError{Message: "azure blob account URL requires a token credential"} + } + if options.ConnectionString != "" && options.Credential != nil { + return nil, &ValidationError{Message: "connection string and token credential cannot be combined"} + } + + clientOptions := &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{Retry: policy.RetryOptions{ + MaxRetries: 8, + RetryDelay: 250 * time.Millisecond, + MaxRetryDelay: 10 * time.Second, + TryTimeout: 2 * time.Minute, + }}} + var ( + client *azblob.Client + err error + ) + if options.ConnectionString != "" { + client, err = azblob.NewClientFromConnectionString(options.ConnectionString, clientOptions) + } else { + accountURL, parseErr := url.Parse(options.AccountURL) + if parseErr != nil || !isSafeBlobURL(accountURL, options.AllowInsecureHTTP) { + return nil, &ValidationError{Message: "invalid Azure Blob account URL"} + } + client, err = azblob.NewClient(options.AccountURL, options.Credential, clientOptions) + } + if err != nil { + return nil, fmt.Errorf("create Azure Blob client: %w", err) + } + serviceURL, err := url.Parse(client.URL()) + if err != nil { + return nil, &ValidationError{Message: "Azure Blob service URL is not allowed"} + } + publicURL := *serviceURL + publicURL.RawQuery, publicURL.Fragment, publicURL.User = "", "", nil + publicURL.ForceQuery = false + if !isSafeBlobURL(&publicURL, options.AllowInsecureHTTP) { + return nil, &ValidationError{Message: "Azure Blob service URL is not allowed"} + } + + allowed := make(map[string]struct{}, len(options.AllowedContainers)+1) + allowed[options.ContainerName] = struct{}{} + for _, container := range options.AllowedContainers { + container = strings.TrimSpace(container) + if !validBlobContainerName(container) { + return nil, &ValidationError{ + Message: fmt.Sprintf("allowed azure blob container name %q is not valid", container), + } + } + allowed[container] = struct{}{} + } + + return &AzureBlobHistoryStore{ + client: client, + defaultContainer: options.ContainerName, + allowedContainers: allowed, + allowAnyContainer: options.AllowAnyContainer, + containers: make(map[string]*containerInit), + }, nil +} + +// DefaultContainer returns the container configured for this store. +func (s *AzureBlobHistoryStore) DefaultContainer() string { return s.defaultContainer } + +// Write uploads object, creating its container on first use. +func (s *AzureBlobHistoryStore) Write(ctx context.Context, object ExportObject) error { + if err := s.validateObject(object); err != nil { + return err + } + init, err := s.ensureContainer(ctx, object.Container) + if err != nil { + return err + } + metadata := make(map[string]*string, len(object.Metadata)) + for key, value := range object.Metadata { + metadata[key] = to.Ptr(value) + } + headers := &blob.HTTPHeaders{} + if object.ContentType != "" { + headers.BlobContentType = to.Ptr(object.ContentType) + } + options := &azblob.UploadBufferOptions{HTTPHeaders: headers, Metadata: metadata} + if err := s.uploadBlob(ctx, object.Container, object.Name, object.Content, options); err != nil { + // A container deleted between initialization and upload must not wedge + // the store: forget the exact initialization this write relied on so the + // next write recreates the container, without discarding a newer one. + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + s.forgetContainer(object.Container, init) + } + return fmt.Errorf("upload export blob %q: %w", object.Name, err) + } + return nil +} + +func (s *AzureBlobHistoryStore) validateObject(object ExportObject) error { + container := object.Container + if container == "" { + return &ValidationError{Message: "export object container is required"} + } + if !validBlobContainerName(container) { + return &ValidationError{Message: fmt.Sprintf("export object container %q is not valid", container)} + } + if !s.allowAnyContainer { + if _, ok := s.allowedContainers[container]; !ok { + return &ValidationError{ + Message: fmt.Sprintf("export destination container %q is not allowed by this worker", container), + } + } + } + if err := validateBlobName(object.Name); err != nil { + return err + } + return nil +} + +// ensureContainer creates container once per process and returns the +// initialization the caller joined, so a later failure can invalidate exactly +// that initialization. +func (s *AzureBlobHistoryStore) ensureContainer(ctx context.Context, container string) (*containerInit, error) { + s.containersMu.Lock() + if existing, ok := s.containers[container]; ok { + s.containersMu.Unlock() + select { + case <-existing.done: + return existing, existing.err + case <-ctx.Done(): + return nil, ctx.Err() + } + } + init := &containerInit{done: make(chan struct{})} + s.containers[container] = init + s.containersMu.Unlock() + + init.err = s.createContainerWithRetry(ctx, container) + if init.err != nil { + // Retire the entry before publishing the result so a waiter that wakes + // on the close cannot observe this failed initialization as cached. + s.forgetContainer(container, init) + } + close(init.done) + return init, init.err +} + +// forgetContainer drops a cached initialization, but only when it is still the +// current one, so it never discards a newer initialization started after this +// one was retired. +func (s *AzureBlobHistoryStore) forgetContainer(container string, init *containerInit) { + s.containersMu.Lock() + defer s.containersMu.Unlock() + if current, ok := s.containers[container]; ok && current == init { + delete(s.containers, container) + } +} + +func (s *AzureBlobHistoryStore) createContainerWithRetry(ctx context.Context, container string) error { + backoff := containerBeingDeletedInitialBackoff + for attempt := 0; ; attempt++ { + err := s.createContainer(ctx, container) + switch { + case err == nil, + bloberror.HasCode(err, bloberror.ContainerAlreadyExists): + return nil + case bloberror.HasCode(err, bloberror.ContainerBeingDeleted) && attempt < containerBeingDeletedAttempts-1: + if waitErr := s.waitForRetry(ctx, backoff); waitErr != nil { + return waitErr + } + backoff = min(backoff*2, containerBeingDeletedMaxBackoff) + default: + return fmt.Errorf("create export container %q: %w", container, err) + } + } +} + +func (s *AzureBlobHistoryStore) createContainer(ctx context.Context, container string) error { + if s.createContainerHook != nil { + return s.createContainerHook(ctx, container) + } + _, err := s.client.CreateContainer(ctx, container, nil) + return err +} + +func (s *AzureBlobHistoryStore) uploadBlob( + ctx context.Context, + container string, + name string, + body []byte, + options *azblob.UploadBufferOptions, +) error { + if s.uploadBlobHook != nil { + return s.uploadBlobHook(ctx, container, name, body, options) + } + _, err := s.client.UploadBuffer(ctx, container, name, body, options) + return err +} + +func (s *AzureBlobHistoryStore) waitForRetry(ctx context.Context, d time.Duration) error { + if s.waitHook != nil { + return s.waitHook(ctx, d) + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +// gzipContent compresses content with the deterministic settings the JSONL +// export format expects. +func gzipContent(content []byte) ([]byte, error) { + var buffer bytes.Buffer + writer, err := gzip.NewWriterLevel(&buffer, gzip.BestCompression) + if err != nil { + return nil, fmt.Errorf("create gzip writer: %w", err) + } + if _, err := writer.Write(content); err != nil { + return nil, fmt.Errorf("compress export content: %w", err) + } + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("finish export compression: %w", err) + } + return buffer.Bytes(), nil +} + +// validBlobContainerName mirrors the Azure Blob container naming rules. +func validBlobContainerName(name string) bool { + if len(name) < 3 || len(name) > 63 { + return false + } + if strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") { + return false + } + previousDash := false + for _, r := range name { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + previousDash = false + case r == '-': + if previousDash { + return false + } + previousDash = true + default: + return false + } + } + return true +} + +// validateBlobPrefix rejects prefixes that would escape the destination or +// produce an unusable blob path. +func validateBlobPrefix(prefix string) error { + if prefix == "" { + return nil + } + if len(prefix) > 900 { + return &ValidationError{Message: "export destination prefix is too long"} + } + if strings.HasPrefix(prefix, "/") { + return &ValidationError{Message: "export destination prefix must not start with '/'"} + } + if strings.Contains(prefix, "//") { + return &ValidationError{Message: "export destination prefix must not contain '//'"} + } + for _, segment := range strings.Split(prefix, "/") { + if segment == "." || segment == ".." { + return &ValidationError{Message: "export destination prefix must not contain relative path segments"} + } + } + if strings.ContainsAny(prefix, "\x00\r\n\t\\") { + return &ValidationError{Message: "export destination prefix contains an unsupported character"} + } + return nil +} + +func validateBlobName(name string) error { + switch { + case name == "": + return &ValidationError{Message: "export object name is required"} + case len(name) > 1024: + return &ValidationError{Message: "export object name is too long"} + case strings.HasPrefix(name, "/"), strings.HasSuffix(name, "/"): + return &ValidationError{Message: "export object name must not start or end with '/'"} + case strings.ContainsAny(name, "\x00\r\n\t\\"): + return &ValidationError{Message: "export object name contains an unsupported character"} + default: + return nil + } +} + +// isSafeBlobURL rejects endpoints that carry credentials or extra URL parts, and +// endpoints that are neither HTTPS nor an explicitly permitted loopback HTTP +// endpoint. +// +// It is at least as strict as the equivalent check in +// [github.com/microsoft/durabletask-go/payload]: userinfo, a query string, and a +// fragment are all rejected outright so a configured account URL cannot smuggle +// a SAS token or credentials, and plaintext HTTP is additionally confined to +// loopback. +func isSafeBlobURL(u *url.URL, allowHTTP bool) bool { + if u == nil || !u.IsAbs() || u.Host == "" || u.Opaque != "" { + return false + } + if u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" { + return false + } + switch strings.ToLower(u.Scheme) { + case "https": + return true + case "http": + if !allowHTTP { + return false + } + host := u.Hostname() + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() + default: + return false + } +} diff --git a/exporthistory/storage_azurite_test.go b/exporthistory/storage_azurite_test.go new file mode 100644 index 00000000..1dfb1e06 --- /dev/null +++ b/exporthistory/storage_azurite_test.go @@ -0,0 +1,179 @@ +package exporthistory + +import ( + "context" + "crypto/rand" + "encoding/hex" + "io" + "os" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newAzuriteExportStore builds a store against a throwaway container so Azurite +// runs exercise the real container-initialization and upload paths. +func newAzuriteExportStore(t *testing.T) (*AzureBlobHistoryStore, string) { + t.Helper() + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run against Azurite") + } + container := randomContainerName(t) + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: connectionString, + ContainerName: container, + AllowInsecureHTTP: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = store.client.DeleteContainer(context.Background(), container, nil) + }) + return store, container +} + +func randomContainerName(t *testing.T) string { + t.Helper() + var suffix [8]byte + _, err := rand.Read(suffix[:]) + require.NoError(t, err) + return "dtgoexport" + hex.EncodeToString(suffix[:]) +} + +// TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects covers the production write +// path against real storage, including container creation, headers, metadata, +// and overwrite-on-reexport. +func TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects(t *testing.T) { + store, container := newAzuriteExportStore(t) + ctx := context.Background() + + events := []*api.HistoryEvent{ + {Type: api.HistoryEventExecutionStarted}, + {Type: api.HistoryEventExecutionCompleted}, + } + content, contentType, err := serializeHistory(events, DefaultExportFormat()) + require.NoError(t, err) + object := ExportObject{ + Container: container, + Name: "batch-job/" + strings.Repeat("a", 64) + ".jsonl.gz", + Content: content, + ContentType: contentType, + Metadata: map[string]string{"instanceId": "instance-1", "schemaVersion": DefaultSchemaVersion}, + } + require.NoError(t, store.Write(ctx, object)) + + properties := azuriteBlobProperties(t, store, container, object.Name) + // The object is stored as an opaque gzip file, so no content coding is set + // and no client transparently decompresses it. + assert.Equal(t, "application/gzip", derefString(properties.ContentType)) + assert.Empty(t, derefString(properties.ContentEncoding)) + assert.Equal(t, "instance-1", metadataValue(properties.Metadata, "instanceId")) + assert.Equal(t, DefaultSchemaVersion, metadataValue(properties.Metadata, "schemaVersion")) + + // Downloading always yields exactly the gzip bytes that were uploaded. + downloaded := downloadAzuriteBlob(t, store, container, object.Name) + assert.Equal(t, content, downloaded) + + decompressed := decompressGzip(t, downloaded) + lines := strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") + require.Len(t, lines, 2) + assert.Contains(t, lines[0], string(api.HistoryEventExecutionStarted)) + assert.Contains(t, lines[1], string(api.HistoryEventExecutionCompleted)) + + // Re-exporting the same instance overwrites its object rather than failing. + object.Content = []byte("second write") + object.ContentType = "text/plain" + require.NoError(t, store.Write(ctx, object)) + assert.Equal(t, []byte("second write"), downloadAzuriteBlob(t, store, container, object.Name)) +} + +// TestAzureBlobHistoryStoreAzuriteRejectsDisallowedContainer keeps a job from writing +// outside the containers the worker was configured for, against real storage. +func TestAzureBlobHistoryStoreAzuriteRejectsDisallowedContainer(t *testing.T) { + store, _ := newAzuriteExportStore(t) + err := store.Write(context.Background(), ExportObject{ + Container: randomContainerName(t), + Name: "object.jsonl.gz", + Content: []byte("payload"), + }) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "is not allowed by this worker") +} + +// TestAzureBlobHistoryStoreAzuriteWritesAcrossAllowedContainers covers the allow-list +// and per-container initialization against real storage. +func TestAzureBlobHistoryStoreAzuriteWritesAcrossAllowedContainers(t *testing.T) { + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run against Azurite") + } + primary := randomContainerName(t) + secondary := randomContainerName(t) + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: connectionString, + ContainerName: primary, + AllowedContainers: []string{secondary}, + AllowInsecureHTTP: true, + }) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = store.client.DeleteContainer(context.Background(), primary, nil) + _, _ = store.client.DeleteContainer(context.Background(), secondary, nil) + }) + + ctx := context.Background() + for _, container := range []string{primary, secondary} { + require.NoError(t, store.Write(ctx, ExportObject{ + Container: container, + Name: "prefix/object.json", + Content: []byte(`[]`), + Metadata: map[string]string{"instanceId": "instance-1"}, + }), container) + assert.Equal(t, []byte(`[]`), downloadAzuriteBlob(t, store, container, "prefix/object.json")) + } +} + +func downloadAzuriteBlob(t *testing.T, store *AzureBlobHistoryStore, container, name string) []byte { + t.Helper() + response, err := store.client.DownloadStream(context.Background(), container, name, nil) + require.NoError(t, err) + defer func() { require.NoError(t, response.Body.Close()) }() + payload, err := io.ReadAll(response.Body) + require.NoError(t, err) + return payload +} + +func azuriteBlobProperties( + t *testing.T, + store *AzureBlobHistoryStore, + container string, + name string, +) blob.GetPropertiesResponse { + t.Helper() + blobClient := store.client.ServiceClient().NewContainerClient(container).NewBlobClient(name) + properties, err := blobClient.GetProperties(context.Background(), nil) + require.NoError(t, err) + return properties +} + +// metadataValue reads a blob metadata entry case-insensitively, since Azure +// normalizes metadata key casing. +func metadataValue(metadata map[string]*string, key string) string { + for name, value := range metadata { + if strings.EqualFold(name, key) { + return derefString(value) + } + } + return "" +} + +func derefString(value *string) string { + if value == nil { + return "" + } + return *value +} diff --git a/exporthistory/storage_test.go b/exporthistory/storage_test.go new file mode 100644 index 00000000..153a22be --- /dev/null +++ b/exporthistory/storage_test.go @@ -0,0 +1,626 @@ +package exporthistory + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "maps" + "net/url" + "strings" + "sync" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// placeholderAccountKey is a syntactically valid base64 value, not a credential. +// These tests only construct clients to exercise URL and transport policy; they +// never authenticate against a storage service. Its decoded text says as much. +const placeholderAccountKey = "ZHVyYWJsZXRhc2stZ28gZXhwb3J0aGlzdG9yeSB1bml0IHRlc3RzIHVzZSB0aGlzIHBsYWNlaG9sZGVyIGFjY291bnQga2V5OyBpdCBpcyBub3QgYSByZWFsIGNyZWRlbnRpYWwgYW5kIG5ldmVyIGF1dGhlbnRpY2F0ZXMu" + +// testConnectionString points at an HTTPS endpoint that is never contacted: +// every test that uses it installs client hooks. +const testConnectionString = "DefaultEndpointsProtocol=https;AccountName=devstoreaccount1;" + + "AccountKey=" + placeholderAccountKey + ";EndpointSuffix=core.windows.net" + +// azuriteConnectionString mirrors the shape of the Azurite development endpoint, +// which is plaintext and therefore requires AllowInsecureHTTP. Live Azurite tests +// read a real connection string from AZURITE_CONNECTION_STRING instead. +const azuriteConnectionString = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + + "AccountKey=" + placeholderAccountKey + ";BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;" + +// memoryStore is an in-memory [Store] used by tests that exercise the export +// pipeline without Azure Storage. +type memoryStore struct { + mu sync.Mutex + objects map[string]ExportObject + writes int + failure error + // failFor fails the write for a specific instance ID, so a test can drive a + // partially failing batch. + failFor map[string]error +} + +func newMemoryStore() *memoryStore { + return &memoryStore{objects: make(map[string]ExportObject), failFor: make(map[string]error)} +} + +func (s *memoryStore) Write(_ context.Context, object ExportObject) error { + s.mu.Lock() + defer s.mu.Unlock() + s.writes++ + if s.failure != nil { + return s.failure + } + if err, ok := s.failFor[object.Metadata["instanceId"]]; ok { + return err + } + s.objects[object.Container+"/"+object.Name] = object + return nil +} + +func (s *memoryStore) count() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.objects) +} + +func (s *memoryStore) writeCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return s.writes +} + +func (s *memoryStore) snapshot() map[string]ExportObject { + s.mu.Lock() + defer s.mu.Unlock() + return maps.Clone(s.objects) +} + +func (s *memoryStore) failInstance(instanceID string, err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.failFor[instanceID] = err +} + +var _ Store = (*memoryStore)(nil) + +func TestValidBlobContainerName(t *testing.T) { + valid := []string{"abc", "a-b-c", "container1", strings.Repeat("a", 63)} + for _, name := range valid { + assert.True(t, validBlobContainerName(name), name) + } + invalid := []string{ + "", "ab", strings.Repeat("a", 64), "-abc", "abc-", "a--b", + "ABC", "a_b", "a.b", "a b", "contaìner", + } + for _, name := range invalid { + assert.False(t, validBlobContainerName(name), name) + } +} + +func TestValidateBlobName(t *testing.T) { + require.NoError(t, validateBlobName("a/b/c.jsonl.gz")) + for _, name := range []string{ + "", "/leading", "trailing/", strings.Repeat("a", 1025), "a\x00b", "a\nb", "a\\b", + } { + require.ErrorIs(t, validateBlobName(name), ErrValidation, name) + } +} + +func TestValidateBlobPrefix(t *testing.T) { + require.NoError(t, validateBlobPrefix("")) + require.NoError(t, validateBlobPrefix("a/b/")) + for _, prefix := range []string{ + "/a", "a//b", "../a", "a/../b", "a/./b", strings.Repeat("a", 901), "a\tb", "a\\b", + } { + require.ErrorIs(t, validateBlobPrefix(prefix), ErrValidation, prefix) + } +} + +func TestIsSafeBlobURL(t *testing.T) { + parse := func(raw string) *url.URL { + parsed, err := url.Parse(raw) + require.NoError(t, err) + return parsed + } + assert.True(t, isSafeBlobURL(parse("https://account.blob.core.windows.net"), false)) + assert.False(t, isSafeBlobURL(parse("http://account.blob.core.windows.net"), false)) + assert.False(t, isSafeBlobURL(parse("http://account.blob.core.windows.net"), true)) + assert.True(t, isSafeBlobURL(parse("http://127.0.0.1:10000"), true)) + assert.True(t, isSafeBlobURL(parse("http://localhost:10000"), true)) + assert.True(t, isSafeBlobURL(parse("http://[::1]:10000"), true)) + assert.False(t, isSafeBlobURL(parse("http://127.0.0.1:10000"), false)) + assert.False(t, isSafeBlobURL(parse("ftp://127.0.0.1"), true)) + assert.False(t, isSafeBlobURL(nil, true)) +} + +// TestIsSafeBlobURLRejectsCredentialCarryingURLs keeps the export store's +// endpoint check at least as strict as the large-payload store's: userinfo, a +// query string, and a fragment are all rejected, so a configured endpoint cannot +// smuggle a SAS token or credentials past the scheme and host checks. +func TestIsSafeBlobURLRejectsCredentialCarryingURLs(t *testing.T) { + parse := func(raw string) *url.URL { + parsed, err := url.Parse(raw) + require.NoError(t, err) + return parsed + } + rejected := []string{ + "https://user@account.blob.core.windows.net", + "https://user:password@account.blob.core.windows.net", + "https://account.blob.core.windows.net/?sig=redacted&se=2030-01-01", + "https://account.blob.core.windows.net/?", + "https://account.blob.core.windows.net/#fragment", + "https://account.blob.core.windows.net/container?sv=2021#frag", + // A relative or opaque reference has no host to validate at all. + "account.blob.core.windows.net", + "//account.blob.core.windows.net", + "mailto:someone@example.com", + } + for _, raw := range rejected { + assert.False(t, isSafeBlobURL(parse(raw), true), raw) + assert.False(t, isSafeBlobURL(parse(raw), false), raw) + } + // A loopback endpoint stays usable, but only without credentials in the URL. + assert.True(t, isSafeBlobURL(parse("http://127.0.0.1:10000/devstoreaccount1"), true)) + assert.False(t, isSafeBlobURL(parse("http://user:pass@127.0.0.1:10000"), true)) +} + +// TestNewAzureBlobHistoryStoreRejectsCredentialCarryingAccountURL covers the same rule +// at the constructor boundary, which is where an operator-supplied endpoint +// actually enters the process. +func TestNewAzureBlobHistoryStoreRejectsCredentialCarryingAccountURL(t *testing.T) { + for _, accountURL := range []string{ + "https://user:password@account.blob.core.windows.net", + "https://account.blob.core.windows.net/?sig=redacted", + "https://account.blob.core.windows.net/#fragment", + } { + _, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ContainerName: "container", + AccountURL: accountURL, + Credential: fakeCredential{}, + }) + require.ErrorIs(t, err, ErrValidation, accountURL) + assert.Contains(t, err.Error(), "invalid Azure Blob account URL", accountURL) + } +} + +func TestNewAzureBlobHistoryStoreValidation(t *testing.T) { + tests := []struct { + name string + options AzureBlobHistoryStoreOptions + message string + }{ + {"missing container", AzureBlobHistoryStoreOptions{ConnectionString: testConnectionString}, "container name is required"}, + { + name: "invalid container", + options: AzureBlobHistoryStoreOptions{ConnectionString: testConnectionString, ContainerName: "BAD"}, + message: "is not valid", + }, + { + name: "no credentials", + options: AzureBlobHistoryStoreOptions{ContainerName: "container"}, + message: "exactly one of connection string or account URL is required", + }, + { + name: "both credential modes", + options: AzureBlobHistoryStoreOptions{ + ContainerName: "container", + ConnectionString: testConnectionString, + AccountURL: "https://account.blob.core.windows.net", + }, + message: "exactly one of connection string or account URL is required", + }, + { + name: "account URL without a credential", + options: AzureBlobHistoryStoreOptions{ContainerName: "container", AccountURL: "https://account.blob.core.windows.net"}, + message: "requires a token credential", + }, + { + name: "connection string with a credential", + options: AzureBlobHistoryStoreOptions{ + ContainerName: "container", + ConnectionString: testConnectionString, + Credential: fakeCredential{}, + }, + message: "cannot be combined", + }, + { + name: "insecure account URL", + options: AzureBlobHistoryStoreOptions{ + ContainerName: "container", + AccountURL: "http://account.blob.core.windows.net", + Credential: fakeCredential{}, + }, + message: "invalid Azure Blob account URL", + }, + { + name: "invalid allow-listed container", + options: AzureBlobHistoryStoreOptions{ + ContainerName: "container", + ConnectionString: testConnectionString, + AllowedContainers: []string{"BAD"}, + }, + message: "allowed azure blob container name", + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := NewAzureBlobHistoryStore(test.options) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), test.message) + }) + } + + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "container", + }) + require.NoError(t, err) + assert.Equal(t, "container", store.DefaultContainer()) + + // The well-known Azurite endpoint is plaintext, so it needs an explicit opt-in. + _, err = NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: azuriteConnectionString, + ContainerName: "container", + }) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "service URL is not allowed") + + _, err = NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: azuriteConnectionString, + ContainerName: "container", + AllowInsecureHTTP: true, + }) + require.NoError(t, err) +} + +func TestAzureBlobHistoryStoreRejectsDisallowedContainers(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + AllowedContainers: []string{"secondary"}, + }) + require.NoError(t, err) + store.createContainerHook = func(context.Context, string) error { return nil } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + + ctx := context.Background() + require.NoError(t, store.Write(ctx, ExportObject{Container: "primary", Name: "a"})) + require.NoError(t, store.Write(ctx, ExportObject{Container: "secondary", Name: "a"})) + + err = store.Write(ctx, ExportObject{Container: "tertiary", Name: "a"}) + require.ErrorIs(t, err, ErrValidation) + assert.Contains(t, err.Error(), "is not allowed by this worker") + + err = store.Write(ctx, ExportObject{Container: "BAD", Name: "a"}) + require.ErrorIs(t, err, ErrValidation) + err = store.Write(ctx, ExportObject{Name: "a"}) + require.ErrorIs(t, err, ErrValidation) + err = store.Write(ctx, ExportObject{Container: "primary", Name: "/bad"}) + require.ErrorIs(t, err, ErrValidation) +} + +func TestAzureBlobHistoryStoreAllowAnyContainer(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + AllowAnyContainer: true, + }) + require.NoError(t, err) + store.createContainerHook = func(context.Context, string) error { return nil } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + require.NoError(t, store.Write(context.Background(), ExportObject{Container: "anything", Name: "a"})) +} + +// TestAzureBlobHistoryStoreCreatesEachContainerOnce keeps a large export from issuing +// one container create per object. +func TestAzureBlobHistoryStoreCreatesEachContainerOnce(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + AllowAnyContainer: true, + }) + require.NoError(t, err) + + var mu sync.Mutex + creates := map[string]int{} + store.createContainerHook = func(_ context.Context, container string) error { + mu.Lock() + defer mu.Unlock() + creates[container]++ + return nil + } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + + var group sync.WaitGroup + for i := 0; i < 16; i++ { + group.Add(1) + go func(i int) { + defer group.Done() + container := "primary" + if i%2 == 0 { + container = "secondary" + } + assert.NoError(t, store.Write(context.Background(), ExportObject{Container: container, Name: "object"})) + }(i) + } + group.Wait() + + mu.Lock() + defer mu.Unlock() + assert.Equal(t, 1, creates["primary"]) + assert.Equal(t, 1, creates["secondary"]) +} + +// TestAzureBlobHistoryStoreRetriesContainerBeingDeleted covers Azure keeping a deleted +// container's name reserved for a short window. +func TestAzureBlobHistoryStoreRetriesContainerBeingDeleted(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + + attempts := 0 + store.createContainerHook = func(context.Context, string) error { + attempts++ + if attempts < 3 { + return &azcore.ResponseError{ErrorCode: string(bloberror.ContainerBeingDeleted)} + } + return nil + } + waits := []time.Duration{} + store.waitHook = func(_ context.Context, d time.Duration) error { + waits = append(waits, d) + return nil + } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + + require.NoError(t, store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"})) + assert.Equal(t, 3, attempts) + assert.Equal(t, []time.Duration{ + containerBeingDeletedInitialBackoff, + 2 * containerBeingDeletedInitialBackoff, + }, waits) +} + +func TestAzureBlobHistoryStoreTreatsExistingContainerAsSuccess(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + store.createContainerHook = func(context.Context, string) error { + return &azcore.ResponseError{ErrorCode: string(bloberror.ContainerAlreadyExists)} + } + uploaded := 0 + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + uploaded++ + return nil + } + require.NoError(t, store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"})) + assert.Equal(t, 1, uploaded) +} + +// TestAzureBlobHistoryStoreRetriesAfterContainerCreationFailure keeps a transient +// creation failure from permanently wedging the store. +func TestAzureBlobHistoryStoreRetriesAfterContainerCreationFailure(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + + attempts := 0 + store.createContainerHook = func(context.Context, string) error { + attempts++ + if attempts == 1 { + return errors.New("transient") + } + return nil + } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + + require.Error(t, store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"})) + require.NoError(t, store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"})) + assert.Equal(t, 2, attempts) +} + +// TestAzureBlobHistoryStoreForgetsDeletedContainer covers a container removed between +// initialization and a later upload. +func TestAzureBlobHistoryStoreForgetsDeletedContainer(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + + creates := 0 + store.createContainerHook = func(context.Context, string) error { + creates++ + return nil + } + uploads := 0 + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + uploads++ + if uploads == 2 { + return &azcore.ResponseError{ErrorCode: string(bloberror.ContainerNotFound)} + } + return nil + } + + ctx := context.Background() + require.NoError(t, store.Write(ctx, ExportObject{Container: "primary", Name: "object"})) + require.Error(t, store.Write(ctx, ExportObject{Container: "primary", Name: "object"})) + require.NoError(t, store.Write(ctx, ExportObject{Container: "primary", Name: "object"})) + assert.Equal(t, 2, creates) +} + +// TestAzureBlobHistoryStoreReportsCreationFailureToWaiters keeps a caller that joined a +// failing container initialization from proceeding to upload as if the container +// existed. +func TestAzureBlobHistoryStoreReportsCreationFailureToWaiters(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + + release := make(chan struct{}) + creationFailure := errors.New("creation denied") + store.createContainerHook = func(context.Context, string) error { + <-release + return creationFailure + } + uploads := 0 + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + uploads++ + return nil + } + + const writers = 8 + errs := make(chan error, writers) + for i := 0; i < writers; i++ { + go func() { + errs <- store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"}) + }() + } + // Give every writer a chance to join the single in-flight initialization. + time.Sleep(50 * time.Millisecond) + close(release) + + for i := 0; i < writers; i++ { + require.ErrorIs(t, <-errs, creationFailure, "writer %d", i) + } + assert.Zero(t, uploads, "no upload may run against a container that was never created") + + // The failed initialization is not cached, so a later write retries it. + store.createContainerHook = func(context.Context, string) error { return nil } + require.NoError(t, store.Write(context.Background(), ExportObject{Container: "primary", Name: "object"})) + assert.Equal(t, 1, uploads) +} + +// TestAzureBlobHistoryStoreInvalidationKeepsNewerInitialization keeps a stale +// ContainerNotFound from discarding a container initialization that started +// after the failing write began. +func TestAzureBlobHistoryStoreInvalidationKeepsNewerInitialization(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + creates := 0 + store.createContainerHook = func(context.Context, string) error { + creates++ + return nil + } + store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + return nil + } + + ctx := context.Background() + stale, err := store.ensureContainer(ctx, "primary") + require.NoError(t, err) + store.forgetContainer("primary", stale) + + current, err := store.ensureContainer(ctx, "primary") + require.NoError(t, err) + require.NotSame(t, stale, current) + + // Retiring the already-retired initialization must not evict the current one. + store.forgetContainer("primary", stale) + again, err := store.ensureContainer(ctx, "primary") + require.NoError(t, err) + assert.Same(t, current, again) + assert.Equal(t, 2, creates) +} + +func TestAzureBlobHistoryStoreSendsHeadersAndMetadata(t *testing.T) { + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: testConnectionString, + ContainerName: "primary", + }) + require.NoError(t, err) + store.createContainerHook = func(context.Context, string) error { return nil } + + var captured *azblob.UploadBufferOptions + var capturedBody []byte + var capturedName string + store.uploadBlobHook = func( + _ context.Context, + _ string, + name string, + body []byte, + options *azblob.UploadBufferOptions, + ) error { + capturedName, capturedBody, captured = name, body, options + return nil + } + + require.NoError(t, store.Write(context.Background(), ExportObject{ + Container: "primary", + Name: "prefix/object.jsonl.gz", + Content: []byte("body"), + ContentType: "application/gzip", + Metadata: map[string]string{"instanceId": "abc"}, + })) + assert.Equal(t, "prefix/object.jsonl.gz", capturedName) + assert.Equal(t, []byte("body"), capturedBody) + require.NotNil(t, captured) + require.NotNil(t, captured.HTTPHeaders.BlobContentType) + assert.Equal(t, "application/gzip", *captured.HTTPHeaders.BlobContentType) + // The store never declares a content coding, so no reader transparently + // decompresses an object whose name promises gzip bytes. + assert.Nil(t, captured.HTTPHeaders.BlobContentEncoding) + require.NotNil(t, captured.Metadata["instanceId"]) + assert.Equal(t, "abc", *captured.Metadata["instanceId"]) +} + +func TestGzipContentRoundTrip(t *testing.T) { + payload := []byte(strings.Repeat("history event\n", 512)) + compressed, err := gzipContent(payload) + require.NoError(t, err) + assert.Less(t, len(compressed), len(payload)) + assert.Equal(t, payload, decompressGzip(t, compressed)) + + empty, err := gzipContent(nil) + require.NoError(t, err) + assert.Empty(t, decompressGzip(t, empty)) +} + +func decompressGzip(t *testing.T, content []byte) []byte { + t.Helper() + reader, err := gzip.NewReader(bytes.NewReader(content)) + require.NoError(t, err) + decompressed, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + return decompressed +} + +// fakeCredential satisfies azcore.TokenCredential for construction-time tests +// that never issue a request. +type fakeCredential struct{} + +func (fakeCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{}, errors.New("not implemented") +} diff --git a/exporthistory/versioning_test.go b/exporthistory/versioning_test.go new file mode 100644 index 00000000..8a4c56c5 --- /dev/null +++ b/exporthistory/versioning_test.go @@ -0,0 +1,181 @@ +package exporthistory + +import ( + "context" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/emptypb" +) + +// filterCapturingServer answers just enough of the DTS gRPC protocol for a +// worker to complete its handshake, and records the work-item filters the +// worker advertises. +type filterCapturingServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + + mu sync.Mutex + filters *protos.WorkItemFilters + requested chan struct{} + once sync.Once +} + +func newFilterCapturingServer() *filterCapturingServer { + return &filterCapturingServer{requested: make(chan struct{})} +} + +func (s *filterCapturingServer) Hello(context.Context, *emptypb.Empty) (*emptypb.Empty, error) { + return &emptypb.Empty{}, nil +} + +func (s *filterCapturingServer) GetWorkItems( + request *protos.GetWorkItemsRequest, + stream protos.TaskHubSidecarService_GetWorkItemsServer, +) error { + s.mu.Lock() + s.filters = request.GetWorkItemFilters() + s.mu.Unlock() + s.once.Do(func() { close(s.requested) }) + <-stream.Context().Done() + return stream.Context().Err() +} + +// advertised blocks until the worker has sent its first GetWorkItems request +// and returns the filters it carried. +func (s *filterCapturingServer) advertised(t *testing.T) *protos.WorkItemFilters { + t.Helper() + select { + case <-s.requested: + case <-time.After(10 * time.Second): + t.Fatal("the worker never requested work items") + } + s.mu.Lock() + defer s.mu.Unlock() + return s.filters +} + +// startFilterCapturingWorker registers the export history system tasks on a +// worker configured with workerOptions and returns the filters it advertises. +func startFilterCapturingWorker( + t *testing.T, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) *protos.WorkItemFilters { + t.Helper() + + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorNVersion( + "Application", + "2.0", + func(*task.OrchestrationContext) (any, error) { return nil, nil }, + )) + require.NoError(t, Register(registry, WorkerOptions{Source: newFakeSource(), Store: newMemoryStore()})) + + serverImpl := newFilterCapturingServer() + server := grpc.NewServer() + protos.RegisterTaskHubSidecarServiceServer(server, serverImpl) + listener := bufconn.Listen(1024 * 1024) + go func() { _ = server.Serve(listener) }() + + connection, err := grpc.NewClient( + "passthrough:///export-history-filters", + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithContextDialer(func(context.Context, string) (net.Conn, error) { + return listener.Dial() + }), + ) + require.NoError(t, err) + + worker, err := durabletaskclient.NewTaskHubGrpcWorker( + connection, + registry, + api.DefaultLogger(), + workerOptions..., + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, connection.Close()) + server.Stop() + require.NoError(t, listener.Close()) + }) + + return serverImpl.advertised(t) +} + +func orchestrationFilterVersions(filters *protos.WorkItemFilters) map[string][]string { + versions := map[string][]string{} + for _, filter := range filters.GetOrchestrations() { + versions[filter.GetName()] = filter.GetVersions() + } + return versions +} + +func activityFilterVersions(filters *protos.WorkItemFilters) map[string][]string { + versions := map[string][]string{} + for _, filter := range filters.GetActivities() { + versions[filter.GetName()] = filter.GetVersions() + } + return versions +} + +// TestWithExportHistoryKeepsSystemTasksRoutableUnderStrictVersioning pins the +// interaction between application default versioning and the unversioned export +// history system tasks. With [WithExportHistory] the worker's derived work-item +// filters keep advertising them unversioned, so the service dispatches export +// work; without it the strict worker version is advertised for them instead and +// the service never dispatches an unversioned export work item. +func TestWithExportHistoryKeepsSystemTasksRoutableUnderStrictVersioning(t *testing.T) { + strictVersioning := durabletaskclient.WithTaskVersioning(task.VersioningOptions{ + Version: "2.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureReject, + }) + + t.Run("routable with the export history worker option", func(t *testing.T) { + filters := startFilterCapturingWorker( + t, strictVersioning, durabletaskclient.WithAutoWorkItemFilters(), WithExportHistory()) + + orchestrations := orchestrationFilterVersions(filters) + require.Equal(t, []string{""}, orchestrations[ExportJobOrchestratorName]) + require.Equal(t, []string{""}, orchestrations[ExecuteExportJobOperationOrchestratorName]) + // The application's own orchestrator still demands the worker version. + require.Equal(t, []string{"2.0"}, orchestrations["Application"]) + + activities := activityFilterVersions(filters) + require.Equal(t, []string{""}, activities[ListTerminalInstancesActivityName]) + require.Equal(t, []string{""}, activities[ExportInstanceHistoryActivityName]) + + entities := make([]string, 0, len(filters.GetEntities())) + for _, filter := range filters.GetEntities() { + entities = append(entities, filter.GetName()) + } + require.Contains(t, entities, strings.ToLower(ExportJobEntityName)) + }) + + t.Run("unroutable without it", func(t *testing.T) { + filters := startFilterCapturingWorker( + t, strictVersioning, durabletaskclient.WithAutoWorkItemFilters()) + + orchestrations := orchestrationFilterVersions(filters) + require.Equal(t, []string{"2.0"}, orchestrations[ExportJobOrchestratorName]) + require.Equal(t, []string{"2.0"}, orchestrations[ExecuteExportJobOperationOrchestratorName]) + + activities := activityFilterVersions(filters) + require.Equal(t, []string{"2.0"}, activities[ListTerminalInstancesActivityName]) + require.Equal(t, []string{"2.0"}, activities[ExportInstanceHistoryActivityName]) + }) +} diff --git a/go.mod b/go.mod index 1d8b7a30..db602f64 100644 --- a/go.mod +++ b/go.mod @@ -1,56 +1,37 @@ module github.com/microsoft/durabletask-go -go 1.23.0 +go 1.25.0 require ( - github.com/cenkalti/backoff/v4 v4.1.3 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 + github.com/cenkalti/backoff/v4 v4.3.0 github.com/google/uuid v1.6.0 - github.com/jackc/pgx/v5 v5.7.1 - github.com/marusama/semaphore/v2 v2.5.0 - github.com/stretchr/testify v1.8.4 - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 - go.opentelemetry.io/otel v1.18.0 - go.opentelemetry.io/otel/exporters/zipkin v1.11.1 - go.opentelemetry.io/otel/sdk v1.11.1 - go.opentelemetry.io/otel/trace v1.18.0 - google.golang.org/grpc v1.56.3 - google.golang.org/protobuf v1.33.0 - modernc.org/sqlite v1.22.1 + github.com/stretchr/testify v1.12.1 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 + google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a + google.golang.org/grpc v1.83.2 + google.golang.org/protobuf v1.36.12 ) require ( - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect - github.com/felixge/httpsnoop v1.0.3 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/golang/protobuf v1.5.3 // indirect - github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect - github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect - github.com/mattn/go-isatty v0.0.16 // indirect - github.com/openzipkin/zipkin-go v0.4.1 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect - github.com/stretchr/objx v0.5.0 // indirect - go.opentelemetry.io/otel/metric v1.18.0 // indirect - golang.org/x/crypto v0.36.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.38.0 // indirect - golang.org/x/sync v0.12.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/text v0.23.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/uint128 v1.2.0 // indirect - modernc.org/cc/v3 v3.40.0 // indirect - modernc.org/ccgo/v3 v3.16.13 // indirect - modernc.org/libc v1.22.5 // indirect - modernc.org/mathutil v1.5.0 // indirect - modernc.org/memory v1.5.0 // indirect - modernc.org/opt v0.1.3 // indirect - modernc.org/strutil v1.1.3 // indirect - modernc.org/token v1.0.1 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index aae8ee04..4ebda6d9 100644 --- a/go.sum +++ b/go.sum @@ -1,135 +1,76 @@ -github.com/cenkalti/backoff/v4 v4.1.3 h1:cFAlzYUlVYDysBEH2T5hyJZMh3+5+WCBvSnK6Q8UtC4= -github.com/cenkalti/backoff/v4 v4.1.3/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= -github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= -github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= -github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= -github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= -github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= -github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.7.1 h1:x7SYsPBYDkHDksogeSmZZ5xzThcTgRz++I5E+ePFUcs= -github.com/jackc/pgx/v5 v5.7.1/go.mod h1:e7O26IywZZ+naJtWWos6i6fvWK+29etgITqrqHLfoZA= -github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= -github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= -github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/marusama/semaphore/v2 v2.5.0 h1:o/1QJD9DBYOWRnDhPwDVAXQn6mQYD0gZaS1Tpx6DJGM= -github.com/marusama/semaphore/v2 v2.5.0/go.mod h1:z9nMiNUekt/LTpTUQdpp+4sJeYqUGpwMHfW0Z8V8fnQ= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= -github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= -github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= -github.com/openzipkin/zipkin-go v0.4.1 h1:kNd/ST2yLLWhaWrkgchya40TJabe8Hioj9udfPcEO5A= -github.com/openzipkin/zipkin-go v0.4.1/go.mod h1:qY0VqDSN1pOBN94dBc6w2GJlWLiovAyg7Qt6/I9HecM= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= -github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 h1:KfYpVmrjI7JuToy5k8XV3nkapjWx48k4E4JOtVstzQI= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0/go.mod h1:SeQhzAEccGVZVEy7aH87Nh0km+utSpo1pTv6eMMop48= -go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= -go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= -go.opentelemetry.io/otel/exporters/zipkin v1.11.1 h1:JlJ3/oQoyqlrPDCfsSVFcHgGeHvZq+hr1VPWtiYCXTo= -go.opentelemetry.io/otel/exporters/zipkin v1.11.1/go.mod h1:T4S6aVwIS1+MHA+dJHCcPROtZe6ORwnv5vMKPRapsFw= -go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= -go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= -go.opentelemetry.io/otel/sdk v1.11.1 h1:F7KmQgoHljhUuJyA+9BiU+EkJfyX5nVVF4wyzWZpKxs= -go.opentelemetry.io/otel/sdk v1.11.1/go.mod h1:/l3FE4SupHJ12TduVjUkZtlfFqDCQJlOlithYrdktys= -go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= -go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= -golang.org/x/crypto v0.36.0 h1:AnAEvhDddvBdpY+uR+MyHmuZzzNqXSe/GvuDeob5L34= -golang.org/x/crypto v0.36.0/go.mod h1:Y4J0ReaxCR1IMaabaSMugxJES1EpwhBHhv2bDHklZvc= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= -golang.org/x/sync v0.12.0 h1:MHc5BpPuC30uJk597Ri8TV3CNZcTLu6B6z4lJy+g6Jw= -golang.org/x/sync v0.12.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= -golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -lukechampine.com/uint128 v1.2.0 h1:mBi/5l91vocEN8otkC5bDLhi2KdCticRiwbdB0O+rjI= -lukechampine.com/uint128 v1.2.0/go.mod h1:c4eWIwlEGaxC/+H1VguhU4PHXNWDCDMUlWdIWl2j1gk= -modernc.org/cc/v3 v3.40.0 h1:P3g79IUS/93SYhtoeaHW+kRCIrYaxJ27MFPv+7kaTOw= -modernc.org/cc/v3 v3.40.0/go.mod h1:/bTg4dnWkSXowUO6ssQKnOV0yMVxDYNIsIrzqTFDGH0= -modernc.org/ccgo/v3 v3.16.13 h1:Mkgdzl46i5F/CNR/Kj80Ri59hC8TKAhZrYSaqvkwzUw= -modernc.org/ccgo/v3 v3.16.13/go.mod h1:2Quk+5YgpImhPjv2Qsob1DnZ/4som1lJTodubIcoUkY= -modernc.org/ccorpus v1.11.6 h1:J16RXiiqiCgua6+ZvQot4yUuUy8zxgqbqEEUuGPlISk= -modernc.org/ccorpus v1.11.6/go.mod h1:2gEUTrWqdpH2pXsmTM1ZkjeSrUWDpjMu2T6m29L/ErQ= -modernc.org/httpfs v1.0.6 h1:AAgIpFZRXuYnkjftxTAZwMIiwEqAfk8aVB2/oA6nAeM= -modernc.org/httpfs v1.0.6/go.mod h1:7dosgurJGp0sPaRanU53W4xZYKh14wfzX420oZADeHM= -modernc.org/libc v1.22.5 h1:91BNch/e5B0uPbJFgqbxXuOnxBQjlS//icfQEGmvyjE= -modernc.org/libc v1.22.5/go.mod h1:jj+Z7dTNX8fBScMVNRAYZ/jF91K8fdT2hYMThc3YjBY= -modernc.org/mathutil v1.5.0 h1:rV0Ko/6SfM+8G+yKiyI830l3Wuz1zRutdslNoQ0kfiQ= -modernc.org/mathutil v1.5.0/go.mod h1:mZW8CKdRPY1v87qxC/wUdX5O1qDzXMP5TH3wjfpga6E= -modernc.org/memory v1.5.0 h1:N+/8c5rE6EqugZwHii4IFsaJ7MUhoWX07J5tC/iI5Ds= -modernc.org/memory v1.5.0/go.mod h1:PkUhL0Mugw21sHPeskwZW4D6VscE/GQJOnIpCnW6pSU= -modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4= -modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0= -modernc.org/sqlite v1.22.1 h1:P2+Dhp5FR1RlVRkQ3dDfCiv3Ok8XPxqpe70IjYVA9oE= -modernc.org/sqlite v1.22.1/go.mod h1:OrDj17Mggn6MhE+iPbBNf7RGKODDE9NFT0f3EwDzJqk= -modernc.org/strutil v1.1.3 h1:fNMm+oJklMGYfU9Ylcywl0CO5O6nTfaowNsh2wpPjzY= -modernc.org/strutil v1.1.3/go.mod h1:MEHNA7PdEnEwLvspRMtWTNnp2nnyvMfkimT1NKNAGbw= -modernc.org/tcl v1.15.2 h1:C4ybAYCGJw968e+Me18oW55kD/FexcHbqH2xak1ROSY= -modernc.org/tcl v1.15.2/go.mod h1:3+k/ZaEbKrC8ePv8zJWPtBSW0V7Gg9g8rkmhI1Kfs3c= -modernc.org/token v1.0.1 h1:A3qvTqOwexpfZZeyI0FeGPDlSWX5pjZu9hF4lU+EKWg= -modernc.org/token v1.0.1/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -modernc.org/z v1.7.3 h1:zDJf6iHjrnB+WRD88stbXokugjyc0/pB91ri1gO6LZY= -modernc.org/z v1.7.3/go.mod h1:Ipv4tsdxZRbQyLq9Q1M6gdbkxYzdlrciF2Hi/lS7nWE= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a h1:3Dnd1cDaZlB68lziofO+bJXpjOy8UfRv8Unt+yH8tQ4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/internal/contextprop/tags.go b/internal/contextprop/tags.go new file mode 100644 index 00000000..d8ebb595 --- /dev/null +++ b/internal/contextprop/tags.go @@ -0,0 +1,57 @@ +package contextprop + +import ( + "maps" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/tagcodec" +) + +const ( + instanceIDTag = api.ReservedContextFieldPrefix + "instance_id" + nameTag = api.ReservedContextFieldPrefix + "orchestration_name" + versionTag = api.ReservedContextFieldPrefix + "orchestration_version" + parentInstanceIDTag = api.ReservedContextFieldPrefix + "parent_instance_id" +) + +// Encode returns a new tag map containing immutable fields and orchestration identity. +func Encode( + info api.OrchestrationContextInfo, + fields api.ContextFields, + userTags ...map[string]string, +) map[string]string { + tags := tagcodec.EncodeContextFields(fields) + if len(userTags) > 0 { + tags = tagcodec.Merge(tags, tagcodec.EncodeUserTags(userTags[0])) + } + if tags == nil { + tags = make(map[string]string, 5) + } + tags[tagcodec.ContextEncodingTag] = "1" + tags[instanceIDTag] = string(info.InstanceID) + tags[nameTag] = info.Name + tags[versionTag] = info.Version + tags[parentInstanceIDTag] = string(info.ParentInstanceID) + return tags +} + +// Decode separates orchestration identity from caller-supplied immutable fields. +func Decode(tags map[string]string) (api.OrchestrationContextInfo, api.ContextFields) { + info := api.OrchestrationContextInfo{ + InstanceID: api.InstanceID(tags[instanceIDTag]), + Name: tags[nameTag], + Version: tags[versionTag], + ParentInstanceID: api.InstanceID(tags[parentInstanceIDTag]), + } + return info, api.ContextFields(tagcodec.DecodeContextFields(tags)) +} + +// Clone returns a defensive copy of tags, or nil when there is nothing to copy. +func Clone[T ~map[string]string](tags T) map[string]string { + if len(tags) == 0 { + return nil + } + copyOfTags := make(map[string]string, len(tags)) + maps.Copy(copyOfTags, tags) + return copyOfTags +} diff --git a/internal/contextprop/tags_test.go b/internal/contextprop/tags_test.go new file mode 100644 index 00000000..6ad00751 --- /dev/null +++ b/internal/contextprop/tags_test.go @@ -0,0 +1,61 @@ +package contextprop + +import ( + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/tagcodec" +) + +func TestEncodeDecode(t *testing.T) { + tags := Encode(api.OrchestrationContextInfo{ + InstanceID: "instance", + Name: "orchestration", + Version: "v1", + ParentInstanceID: "parent", + }, api.ContextFields{"tenant": "alpha"}) + + info, fields := Decode(tags) + if info.InstanceID != "instance" || + info.Name != "orchestration" || + info.Version != "v1" || + info.ParentInstanceID != "parent" { + t.Fatalf("unexpected info: %+v", info) + } + + if fields["tenant"] != "alpha" { + t.Fatalf("tenant = %q, want alpha", fields["tenant"]) + } +} + +func TestEncodeSeparatesContextFieldsAndUserTags(t *testing.T) { + tags := Encode( + api.OrchestrationContextInfo{}, + api.ContextFields{"tenant": "context"}, + map[string]string{"team": "tag"}, + ) + _, fields := Decode(tags) + if fields["tenant"] != "context" { + t.Fatalf("tenant = %q, want context", fields["tenant"]) + } + userTags := tagcodec.DecodeUserTags(tags) + if userTags["team"] != "tag" { + t.Fatalf("team = %q, want tag", userTags["team"]) + } + if _, ok := fields["team"]; ok { + t.Fatalf("user tag leaked into context fields: %v", fields) + } +} + +func TestEncodeOverwritesReservedCallerFields(t *testing.T) { + tags := Encode(api.OrchestrationContextInfo{}, api.ContextFields{ + api.ReservedContextFieldPrefix + "orchestration_version": "spoofed", + }) + info, fields := Decode(tags) + if info.Version != "" { + t.Fatalf("version = %q, want empty", info.Version) + } + if fields != nil { + t.Fatalf("reserved field leaked into caller fields: %v", fields) + } +} diff --git a/internal/failure/failure.go b/internal/failure/failure.go new file mode 100644 index 00000000..e72e9828 --- /dev/null +++ b/internal/failure/failure.go @@ -0,0 +1,399 @@ +package failure + +import ( + "errors" + "fmt" + "math" + "reflect" + "sort" + "time" + "unicode/utf8" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const ( + maxErrorTypeBytes = 1024 + maxErrorMessageBytes = 16 * 1024 + maxStackTraceBytes = 16 * 1024 + maxPropertyCount = 64 + maxPropertyValueBytes = 32 * 1024 + maxPropertiesBytes = 64 * 1024 +) + +const propertiesTruncatedKey = "go.propertiesTruncated" + +type propertyBudget struct { + count int + bytes int + truncated bool +} + +// FromProto converts wire failure details into API-owned failure data. +func FromProto(details *protos.TaskFailureDetails) *api.FailureDetails { + return fromProto(details, 0, new(propertyBudget)) +} + +func fromProto(details *protos.TaskFailureDetails, depth int, budget *propertyBudget) *api.FailureDetails { + if details == nil { + return nil + } + result := &api.FailureDetails{ + ErrorType: api.ErrorType(truncateUTF8(details.GetErrorType(), maxErrorTypeBytes)), + ErrorMessage: truncateUTF8(details.GetErrorMessage(), maxErrorMessageBytes), + IsNonRetriable: details.GetIsNonRetriable(), + StackTrace: truncateUTF8(details.GetStackTrace().GetValue(), maxStackTraceBytes), + } + if len(details.GetProperties()) > 0 { + result.Properties = make(map[string]any, min(len(details.GetProperties()), maxPropertyCount)) + keys := make([]string, 0, len(details.GetProperties())) + for key := range details.GetProperties() { + keys = append(keys, key) + } + sort.Strings(keys) + truncated := false + for _, key := range keys { + size := len(key) + proto.Size(details.GetProperties()[key]) + if budget.count >= maxPropertyCount || budget.bytes+size > maxPropertiesBytes { + truncated = true + break + } + result.Properties[key] = valueFromProto(details.GetProperties()[key]) + budget.count++ + budget.bytes += size + } + if truncated && !budget.truncated { + result.Properties[propertiesTruncatedKey] = true + budget.truncated = true + } + } + if depth+1 < api.MaxFailureDetailsDepth { + result.InnerFailure = fromProto(details.GetInnerFailure(), depth+1, budget) + } + return result +} + +// ToProto converts API-owned failure data to its wire representation. +func ToProto(details *api.FailureDetails) *protos.TaskFailureDetails { + return toProto(details, 0, new(propertyBudget)) +} + +func toProto(details *api.FailureDetails, depth int, budget *propertyBudget) *protos.TaskFailureDetails { + if details == nil { + return nil + } + result := &protos.TaskFailureDetails{ + ErrorType: truncateUTF8(string(details.ErrorType), maxErrorTypeBytes), + ErrorMessage: truncateUTF8(details.ErrorMessage, maxErrorMessageBytes), + IsNonRetriable: details.IsNonRetriable, + } + if stack := truncateUTF8(details.StackTrace, maxStackTraceBytes); stack != "" { + result.StackTrace = wrapperspb.String(stack) + } + addProperties(result, details.Properties, budget) + if depth+1 < api.MaxFailureDetailsDepth { + result.InnerFailure = toProto(details.InnerFailure, depth+1, budget) + } + return result +} + +// FromError converts a Go error into wire failure details while preserving wrapped durable failures. +func FromError(err error, providers ...api.ErrorPropertiesProvider) *protos.TaskFailureDetails { + var provider api.ErrorPropertiesProvider + if len(providers) > 0 { + provider = providers[0] + } + return fromError(err, provider, 0, new(propertyBudget)) +} + +func fromError( + err error, + provider api.ErrorPropertiesProvider, + depth int, + budget *propertyBudget, +) *protos.TaskFailureDetails { + if err == nil { + return nil + } + result := &protos.TaskFailureDetails{ + ErrorType: truncateUTF8(string(errorType(err)), maxErrorTypeBytes), + ErrorMessage: truncateUTF8(err.Error(), maxErrorMessageBytes), + } + var stackProvider api.DurableTaskStackTraceProvider + if errors.As(err, &stackProvider) { + if stack := truncateUTF8(stackProvider.DurableTaskStackTrace(), maxStackTraceBytes); stack != "" { + result.StackTrace = wrapperspb.String(stack) + } + } + var marker api.NonRetriable + if errors.As(err, &marker) { + result.IsNonRetriable = marker.NonRetriable() + } + var propertiesProvider api.DurableTaskErrorPropertiesProvider + if errors.As(err, &propertiesProvider) { + addProperties(result, propertiesProvider.DurableTaskErrorProperties(), budget) + } + if provider != nil { + addProperties(result, provider.ErrorProperties(err), budget) + } + if depth+1 >= api.MaxFailureDetailsDepth { + return result + } + var detailsProvider api.DurableTaskFailureDetailsProvider + if errors.As(err, &detailsProvider) { + result.InnerFailure = toProto(detailsProvider.DurableTaskFailureDetails(), depth+1, budget) + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + joinedErrors := joined.Unwrap() + primary := joinedPrimary(joinedErrors) + if result.InnerFailure == nil && len(joinedErrors) > 0 { + result.InnerFailure = fromError(joinedErrors[primary], provider, depth+1, budget) + } + if len(joinedErrors) > 1 { + addAdditionalErrors(result, joinedErrors, primary, provider, depth+1, budget) + } + return result + } + if result.InnerFailure == nil { + result.InnerFailure = fromError(errors.Unwrap(err), provider, depth+1, budget) + } + return result +} + +func errorType(err error) api.ErrorType { + var provider api.DurableTaskErrorTypeProvider + if errors.As(err, &provider) { + if errorType := provider.DurableTaskErrorType(); errorType != "" { + return errorType + } + } + return api.ErrorType(reflect.TypeOf(err).String()) +} + +func joinedPrimary(joined []error) int { + for i, err := range joined { + var provider api.DurableTaskFailureDetailsProvider + if errors.As(err, &provider) { + return i + } + } + for i, err := range joined { + var provider api.DurableTaskErrorTypeProvider + if errors.As(err, &provider) && provider.DurableTaskErrorType() != "" { + return i + } + } + return 0 +} + +func addAdditionalErrors( + details *protos.TaskFailureDetails, + joined []error, + primary int, + provider api.ErrorPropertiesProvider, + depth int, + budget *propertyBudget, +) { + values := make([]any, 0, len(joined)-1) + for i, err := range joined { + if i == primary || err == nil { + continue + } + branchBudget := new(propertyBudget) + branch := fromError(err, provider, depth, branchBudget) + values = append(values, failureDetailsProperty(branch, 0)) + } + if len(values) == 0 { + return + } + properties := map[string]any{"go.additionalErrors": values} + addProperties(details, properties, budget) +} + +func addProperties( + details *protos.TaskFailureDetails, + properties map[string]any, + budget *propertyBudget, +) { + if len(properties) == 0 { + return + } + keys := make([]string, 0, len(properties)) + for key := range properties { + keys = append(keys, key) + } + sort.Strings(keys) + truncated := false + for _, key := range keys { + value := valueToProto(properties[key], 0) + size := len(key) + proto.Size(value) + if size > maxPropertyValueBytes { + value = structpb.NewStringValue("") + size = len(key) + proto.Size(value) + truncated = true + } + oldValue, exists := details.GetProperties()[key] + oldSize := 0 + if exists { + oldSize = len(key) + proto.Size(oldValue) + } + countDelta := 1 + if exists { + countDelta = 0 + } + if budget.count+countDelta > maxPropertyCount || + budget.bytes-oldSize+size > maxPropertiesBytes { + truncated = true + break + } + if details.Properties == nil { + details.Properties = make(map[string]*structpb.Value) + } + details.Properties[key] = value + budget.count += countDelta + budget.bytes = budget.bytes - oldSize + size + } + + if truncated && !budget.truncated { + if details.Properties == nil { + details.Properties = make(map[string]*structpb.Value) + } + details.Properties[propertiesTruncatedKey] = structpb.NewBoolValue(true) + budget.truncated = true + } +} + +func failureDetailsProperty(details *protos.TaskFailureDetails, depth int) map[string]any { + if details == nil || depth >= api.MaxFailureDetailsDepth { + return nil + } + result := map[string]any{ + "type": details.GetErrorType(), + "message": details.GetErrorMessage(), + } + if details.GetStackTrace().GetValue() != "" { + result["stackTrace"] = details.GetStackTrace().GetValue() + } + if details.GetIsNonRetriable() { + result["isNonRetriable"] = true + } + if len(details.GetProperties()) > 0 { + properties := make(map[string]any, len(details.GetProperties())) + for key, value := range details.GetProperties() { + properties[key] = valueFromProto(value) + } + result["properties"] = properties + } + if inner := failureDetailsProperty(details.GetInnerFailure(), depth+1); inner != nil { + result["innerFailure"] = inner + } + return result +} + +func valueToProto(value any, depth int) *structpb.Value { + if depth >= 16 { + return structpb.NewStringValue("") + } + reflected := reflect.ValueOf(value) + for reflected.IsValid() && (reflected.Kind() == reflect.Interface || reflected.Kind() == reflect.Pointer) { + if reflected.IsNil() { + return structpb.NewNullValue() + } + reflected = reflected.Elem() + } + switch typed := value.(type) { + case nil: + return structpb.NewNullValue() + case bool: + return structpb.NewBoolValue(typed) + case string: + return structpb.NewStringValue(typed) + case time.Time: + return structpb.NewStringValue("dt:" + typed.Format(time.RFC3339Nano)) + case fmt.Stringer: + return structpb.NewStringValue(typed.String()) + } + + if !reflected.IsValid() { + return structpb.NewNullValue() + } + switch reflected.Kind() { + case reflect.Bool: + return structpb.NewBoolValue(reflected.Bool()) + case reflect.String: + return structpb.NewStringValue(reflected.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return structpb.NewNumberValue(float64(reflected.Int())) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return structpb.NewNumberValue(float64(reflected.Uint())) + case reflect.Float32, reflect.Float64: + number := reflected.Float() + if math.IsNaN(number) || math.IsInf(number, 0) { + return structpb.NewStringValue(fmt.Sprint(number)) + } + return structpb.NewNumberValue(number) + case reflect.Map: + if reflected.Type().Key().Kind() != reflect.String { + return structpb.NewStringValue(fmt.Sprint(value)) + } + keys := make([]string, 0, reflected.Len()) + for _, key := range reflected.MapKeys() { + keys = append(keys, key.String()) + } + sort.Strings(keys) + fields := make(map[string]*structpb.Value, len(keys)) + for _, key := range keys { + mapKey := reflect.ValueOf(key).Convert(reflected.Type().Key()) + fields[key] = valueToProto(reflected.MapIndex(mapKey).Interface(), depth+1) + } + return structpb.NewStructValue(&structpb.Struct{Fields: fields}) + case reflect.Array, reflect.Slice: + values := make([]*structpb.Value, reflected.Len()) + for i := 0; i < reflected.Len(); i++ { + values[i] = valueToProto(reflected.Index(i).Interface(), depth+1) + } + return structpb.NewListValue(&structpb.ListValue{Values: values}) + default: + return structpb.NewStringValue(fmt.Sprint(value)) + } +} + +func valueFromProto(value *structpb.Value) any { + if value == nil { + return nil + } + switch typed := value.Kind.(type) { + case *structpb.Value_StringValue: + return typed.StringValue + case *structpb.Value_StructValue: + result := make(map[string]any, len(typed.StructValue.Fields)) + for key, field := range typed.StructValue.Fields { + result[key] = valueFromProto(field) + } + return result + case *structpb.Value_ListValue: + result := make([]any, len(typed.ListValue.Values)) + for i, item := range typed.ListValue.Values { + result[i] = valueFromProto(item) + } + return result + default: + return value.AsInterface() + } +} + +func truncateUTF8(value string, limit int) string { + if len(value) <= limit { + return value + } + const marker = "\n... truncated" + limit -= len(marker) + for limit > 0 && !utf8.RuneStart(value[limit]) { + limit-- + } + return value[:limit] + marker +} diff --git a/internal/failure/failure_test.go b/internal/failure/failure_test.go new file mode 100644 index 00000000..3357d0aa --- /dev/null +++ b/internal/failure/failure_test.go @@ -0,0 +1,157 @@ +package failure_test + +import ( + "bytes" + "errors" + "fmt" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/task" +) + +func TestTaskFailedErrorConversionPreservesInnerFailure(t *testing.T) { + inner := &api.FailureDetails{ + ErrorType: "ActivityTaskNotFound", + ErrorMessage: "No activity task named 'missing' was found.", + IsNonRetriable: true, + Properties: map[string]any{"attempt": float64(2)}, + } + + err := &task.TaskFailedError{ + TaskName: "missing", + TaskID: 7, + FailureDetails: inner, + } + + wire := failure.FromError(err) + if wire.GetErrorType() != "TaskFailedException" { + t.Fatalf("ErrorType = %q", wire.GetErrorType()) + } + if wire.GetInnerFailure().GetErrorType() != string(inner.ErrorType) { + t.Fatalf("inner ErrorType = %q", wire.GetInnerFailure().GetErrorType()) + } + if !wire.GetIsNonRetriable() { + t.Fatal("outer failure did not preserve non-retriable state") + } + + roundTrip := failure.FromProto(wire) + if roundTrip.InnerFailure == nil || roundTrip.InnerFailure.ErrorMessage != inner.ErrorMessage { + t.Fatalf("round-trip failure = %#v", roundTrip) + } +} + +func TestWrappedDurableMarkersRemainVisible(t *testing.T) { + inner := &task.VersionMismatchError{ + TaskVersion: "2.0", + WorkerVersion: "1.0", + Strategy: task.VersionMatchStrict, + } + + wire := failure.FromError(fmt.Errorf("dispatch failed: %w", inner)) + if wire.GetErrorType() != string(api.ErrorTypeVersionMismatch) { + t.Fatalf("ErrorType = %q", wire.GetErrorType()) + } + if !wire.GetIsNonRetriable() { + t.Fatal("wrapped non-retriable marker was lost") + } +} + +func TestFailurePropertiesProviderRoundTripsStructuredValues(t *testing.T) { + when := time.Date(2026, time.August, 27, 12, 30, 0, 0, time.FixedZone("offset", -7*60*60)) + provider := api.ErrorPropertiesProviderFunc(func(error) map[string]any { + return map[string]any{ + "attempt": 3, + "when": when, + "nested": map[string]any{"values": []any{"a", true, 4}}, + } + }) + + wire := failure.FromError(errors.New("boom"), provider) + roundTrip := failure.FromProto(wire) + if roundTrip.Properties["attempt"] != float64(3) { + t.Fatalf("attempt = %#v", roundTrip.Properties["attempt"]) + } + if got, ok := roundTrip.Properties["when"].(string); !ok || got != "dt:"+when.Format(time.RFC3339Nano) { + t.Fatalf("when = %#v", roundTrip.Properties["when"]) + } + nested, ok := roundTrip.Properties["nested"].(map[string]any) + if !ok { + t.Fatalf("nested = %#v", roundTrip.Properties["nested"]) + } + if _, ok := nested["values"].([]any); !ok { + t.Fatalf("nested values = %#v", nested["values"]) + } +} + +func TestJoinedErrorsUseOneCauseAndAdditionalProperties(t *testing.T) { + wire := failure.FromError(errors.Join(errors.New("primary"), errors.New("secondary"))) + + if wire.GetInnerFailure().GetErrorMessage() != "primary" { + t.Fatalf("inner failure = %#v", wire.GetInnerFailure()) + } + additional := wire.GetProperties()["go.additionalErrors"].GetListValue() + if additional == nil || len(additional.Values) != 1 { + t.Fatalf("additional errors = %#v", additional) + } +} + +func TestJoinedErrorsPromoteDurableFailure(t *testing.T) { + taskErr := &task.TaskFailedError{ + TaskName: "activity", + TaskID: 1, + FailureDetails: &api.FailureDetails{ + ErrorType: "ActivityTaskNotFound", + ErrorMessage: "missing", + IsNonRetriable: true, + }, + } + + wire := failure.FromError(errors.Join(errors.New("cleanup"), taskErr)) + if wire.GetErrorType() != "TaskFailedException" { + t.Fatalf("failure = %#v", wire) + } + if wire.GetInnerFailure().GetErrorType() != "ActivityTaskNotFound" { + t.Fatalf("durable chain = %#v", wire) + } +} + +func TestPropertiesHandleTypedNilStringer(t *testing.T) { + var buffer *bytes.Buffer + provider := api.ErrorPropertiesProviderFunc(func(error) map[string]any { + return map[string]any{"buffer": buffer} + }) + + wire := failure.FromError(errors.New("boom"), provider) + if wire.GetProperties()["buffer"].GetNullValue().String() != "NULL_VALUE" { + t.Fatalf("buffer = %#v", wire.GetProperties()["buffer"]) + } +} + +func TestPropertyBudgetAppliesAcrossFailureChain(t *testing.T) { + properties := make(map[string]any, 100) + for i := 0; i < 100; i++ { + properties[fmt.Sprintf("property-%03d", i)] = "value" + } + provider := api.ErrorPropertiesProviderFunc(func(error) map[string]any { + return properties + }) + + wire := failure.FromError(fmt.Errorf("outer: %w", errors.New("inner")), provider) + propertyCount := 0 + truncationMarkers := 0 + for current := wire; current != nil; current = current.GetInnerFailure() { + propertyCount += len(current.GetProperties()) + if current.GetProperties()["go.propertiesTruncated"].GetBoolValue() { + truncationMarkers++ + } + } + if propertyCount > 65 { + t.Fatalf("property count = %d", propertyCount) + } + if truncationMarkers != 1 { + t.Fatalf("truncation markers = %d", truncationMarkers) + } +} diff --git a/internal/grpcerrors/reasons.go b/internal/grpcerrors/reasons.go new file mode 100644 index 00000000..337722a4 --- /dev/null +++ b/internal/grpcerrors/reasons.go @@ -0,0 +1,39 @@ +package grpcerrors + +import ( + "fmt" + + "google.golang.org/genproto/googleapis/rpc/errdetails" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +const ( + ReasonDuplicateInstance = "DUPLICATE_INSTANCE" + ReasonFeatureUnsupported = "FEATURE_UNSUPPORTED" + ReasonInstanceNotFound = "INSTANCE_NOT_FOUND" + ReasonInvalidArgument = "INVALID_ARGUMENT" + ReasonInvalidState = "INVALID_STATE" + ReasonNotCompleted = "NOT_COMPLETED" + ReasonTaskHubExists = "TASK_HUB_EXISTS" + ReasonTaskHubNotFound = "TASK_HUB_NOT_FOUND" +) + +// New returns a gRPC status error with a machine-readable durable error reason. +func New(code codes.Code, message, reason string) error { + result, err := status.New(code, message).WithDetails(&errdetails.ErrorInfo{Reason: reason}) + if err != nil { + return status.Error(codes.Internal, fmt.Sprintf("failed to attach durable error reason %q: %v", reason, err)) + } + return result.Err() +} + +// Reason returns the durable error reason attached to a gRPC status. +func Reason(value *status.Status) string { + for _, detail := range value.Details() { + if info, ok := detail.(*errdetails.ErrorInfo); ok { + return info.Reason + } + } + return "" +} diff --git a/internal/helpers/casing.go b/internal/helpers/casing.go new file mode 100644 index 00000000..637ca846 --- /dev/null +++ b/internal/helpers/casing.go @@ -0,0 +1,31 @@ +package helpers + +import ( + "strings" + "unicode" +) + +// dottedCapitalI is LATIN CAPITAL LETTER I WITH DOT ABOVE. Go's Unicode tables +// map it to a plain "i", but .NET's ToLowerInvariant leaves it unchanged. Entity +// names are lowercased on both sides of the wire, so the two SDKs would other- +// wise address different entities for the same name. +const dottedCapitalI = '\u0130' + +// ToLowerInvariant lowercases s exactly like .NET's String.ToLowerInvariant, so +// entity names and operation names resolve identically across SDKs. It agrees +// with strings.ToLower for every code point except [dottedCapitalI]. +func ToLowerInvariant(s string) string { + if !strings.ContainsRune(s, dottedCapitalI) { + return strings.ToLower(s) + } + var lowered strings.Builder + lowered.Grow(len(s)) + for _, r := range s { + if r == dottedCapitalI { + lowered.WriteRune(r) + continue + } + lowered.WriteRune(unicode.ToLower(r)) + } + return lowered.String() +} diff --git a/internal/helpers/casing_test.go b/internal/helpers/casing_test.go new file mode 100644 index 00000000..e8f491b9 --- /dev/null +++ b/internal/helpers/casing_test.go @@ -0,0 +1,55 @@ +package helpers + +import ( + "strings" + "testing" +) + +// Entity names are lowercased on both sides of the wire, so Go and .NET must +// agree for every code point or the two SDKs address different entities. The +// expectations below were produced by running String.ToLowerInvariant on .NET. +func TestToLowerInvariantMatchesDotNet(t *testing.T) { + tests := map[string]string{ + "Counter": "counter", + "COUNTER": "counter", + "counter": "counter", + "": "", + "Ünïcode": "ünïcode", + "STRASSE": "strasse", + "ΣIGMA": "σigma", + "Ⅻ": "ⅻ", + "ĸ": "ĸ", + "Dž": "dž", + "K": "k", // U+212A KELVIN SIGN + "İstanbul": "İstanbul", + "İ": "İ", // U+0130 is unchanged by ToLowerInvariant + "ı": "ı", + "AİB": "aİb", + } + for input, want := range tests { + if got := ToLowerInvariant(input); got != want { + t.Errorf("ToLowerInvariant(%q) = %q, want %q", input, got, want) + } + } +} + +// U+0130 is the only code point where Go's Unicode tables disagree with .NET. +func TestToLowerInvariantOnlyDivergesForDottedCapitalI(t *testing.T) { + for cp := rune(0); cp <= 0x10FFFF; cp++ { + if cp >= 0xD800 && cp <= 0xDFFF { + continue + } + input := string(cp) + got := ToLowerInvariant(input) + want := strings.ToLower(input) + if cp == dottedCapitalI { + if got != input { + t.Fatalf("ToLowerInvariant(U+0130) = %q, want it unchanged", got) + } + continue + } + if got != want { + t.Fatalf("ToLowerInvariant(%U) = %q, want %q", cp, got, want) + } + } +} diff --git a/internal/helpers/entity.go b/internal/helpers/entity.go new file mode 100644 index 00000000..cb10763d --- /dev/null +++ b/internal/helpers/entity.go @@ -0,0 +1,21 @@ +package helpers + +// EntityRequestEventName is the event name used for all entity operation requests. +// This matches the .NET DTFx EntityMessageEventNames.RequestMessageEventName constant. +const EntityRequestEventName = "op" + +const entitySignalPropertyPrefix = "__durabletask.entity.signal." + +func EntitySignalProperty(requestID string) string { + return entitySignalPropertyPrefix + requestID +} + +// EntityRequestMessage is the payload sent to an entity for operation requests. +// This matches the .NET DTFx RequestMessage format for wire compatibility. +type EntityRequestMessage struct { + ID string `json:"id"` + ParentInstanceID string `json:"parentInstanceId,omitempty"` + IsSignal bool `json:"isSignal"` + Operation string `json:"operation"` + Input string `json:"input,omitempty"` +} diff --git a/internal/helpers/entity_ids.go b/internal/helpers/entity_ids.go new file mode 100644 index 00000000..20e92826 --- /dev/null +++ b/internal/helpers/entity_ids.go @@ -0,0 +1,52 @@ +package helpers + +import ( + "fmt" + "strings" +) + +// ValidateEntityName applies the durable-entity naming constraints used by the wire format. +func ValidateEntityName(name string) error { + switch { + case name == "": + return fmt.Errorf("invalid entity id: entity name must not be empty") + case strings.Contains(name, "@"): + return fmt.Errorf("invalid entity id: entity name %q must not contain '@'", name) + default: + return nil + } +} + +// ParseEntityInstanceID parses an entity instance ID in the format "@@". +func ParseEntityInstanceID(instanceID string) (string, string, error) { + if !strings.HasPrefix(instanceID, "@") { + return "", "", fmt.Errorf("invalid entity instance ID format: %q", instanceID) + } + + trimmed := instanceID[1:] + name, key, ok := strings.Cut(trimmed, "@") + if !ok { + return "", "", fmt.Errorf("invalid entity instance ID format: missing second '@'") + } + if err := ValidateEntityName(name); err != nil { + return "", "", err + } + return ToLowerInvariant(name), key, nil +} + +// IsEntityInstanceID reports whether the instance ID uses the reserved entity format. +func IsEntityInstanceID(instanceID string) bool { + _, _, err := ParseEntityInstanceID(instanceID) + return err == nil +} + +// ValidateOrchestrationInstanceID rejects orchestration IDs that collide with entity instance IDs. +func ValidateOrchestrationInstanceID(instanceID string) error { + if instanceID == "" { + return nil + } + if IsEntityInstanceID(instanceID) { + return fmt.Errorf("orchestration instance ID %q uses the reserved entity format", instanceID) + } + return nil +} diff --git a/internal/helpers/history.go b/internal/helpers/history.go index 0bf7dbf5..963b2dcf 100644 --- a/internal/helpers/history.go +++ b/internal/helpers/history.go @@ -1,9 +1,6 @@ package helpers import ( - "reflect" - "strconv" - "strings" "time" "github.com/google/uuid" @@ -20,12 +17,13 @@ func NewExecutionStartedEvent( parent *protos.ParentInstanceInfo, parentTraceContext *protos.TraceContext, scheduledStartTimeStamp *timestamppb.Timestamp, + version ...*wrapperspb.StringValue, ) *protos.HistoryEvent { u, err := uuid.NewV7() if err != nil { u = uuid.New() } - return &protos.HistoryEvent{ + event := &protos.HistoryEvent{ EventId: -1, Timestamp: timestamppb.New(time.Now()), EventType: &protos.HistoryEvent_ExecutionStarted{ @@ -42,20 +40,10 @@ func NewExecutionStartedEvent( }, }, } -} - -func NewExecutionCompletedEvent(eventID int32, status protos.OrchestrationStatus, result *wrapperspb.StringValue, failureDetails *protos.TaskFailureDetails) *protos.HistoryEvent { - return &protos.HistoryEvent{ - EventId: eventID, - Timestamp: timestamppb.Now(), - EventType: &protos.HistoryEvent_ExecutionCompleted{ - ExecutionCompleted: &protos.ExecutionCompletedEvent{ - OrchestrationStatus: status, - Result: result, - FailureDetails: failureDetails, - }, - }, + if len(version) > 0 { + event.GetExecutionStarted().Version = version[0] } + return event } func NewExecutionTerminatedEvent(rawReason *wrapperspb.StringValue, recurse bool) *protos.HistoryEvent { @@ -242,13 +230,22 @@ func NewParentInfo(taskID int32, name string, iid string) *protos.ParentInstance } } -func NewScheduleTaskAction(taskID int32, name string, input *wrapperspb.StringValue) *protos.OrchestratorAction { - return &protos.OrchestratorAction{ +func NewScheduleTaskAction( + taskID int32, + name string, + input *wrapperspb.StringValue, + version ...*wrapperspb.StringValue, +) *protos.OrchestratorAction { + action := &protos.OrchestratorAction{ Id: taskID, OrchestratorActionType: &protos.OrchestratorAction_ScheduleTask{ ScheduleTask: &protos.ScheduleTaskAction{Name: name, Input: input}, }, } + if len(version) > 0 { + action.GetScheduleTask().Version = version[0] + } + return action } func NewCreateTimerAction(taskID int32, fireAt time.Time) *protos.OrchestratorAction { @@ -273,13 +270,112 @@ func NewSendEventAction(iid string, name string, data *wrapperspb.StringValue) * } } +func NewEntityOperationSignaledAction( + id int32, + requestID string, + entityID string, + operation string, + input *wrapperspb.StringValue, + scheduledTime *timestamppb.Timestamp, +) *protos.OrchestratorAction { + return &protos.OrchestratorAction{ + Id: id, + OrchestratorActionType: &protos.OrchestratorAction_SendEntityMessage{ + SendEntityMessage: &protos.SendEntityMessageAction{ + EntityMessageType: &protos.SendEntityMessageAction_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{ + RequestId: requestID, + Operation: operation, + ScheduledTime: scheduledTime, + Input: input, + TargetInstanceId: wrapperspb.String(entityID), + }, + }, + }, + }, + } +} + +func NewEntityOperationCalledAction( + id int32, + requestID string, + entityID string, + parentInstanceID string, + parentExecutionID string, + operation string, + input *wrapperspb.StringValue, +) *protos.OrchestratorAction { + return &protos.OrchestratorAction{ + Id: id, + OrchestratorActionType: &protos.OrchestratorAction_SendEntityMessage{ + SendEntityMessage: &protos.SendEntityMessageAction{ + EntityMessageType: &protos.SendEntityMessageAction_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: requestID, + Operation: operation, + Input: input, + ParentInstanceId: wrapperspb.String(parentInstanceID), + ParentExecutionId: wrapperspb.String(parentExecutionID), + TargetInstanceId: wrapperspb.String(entityID), + }, + }, + }, + }, + } +} + +func NewEntityLockRequestedAction( + id int32, + criticalSectionID string, + parentInstanceID string, + lockSet []string, +) *protos.OrchestratorAction { + return &protos.OrchestratorAction{ + Id: id, + OrchestratorActionType: &protos.OrchestratorAction_SendEntityMessage{ + SendEntityMessage: &protos.SendEntityMessageAction{ + EntityMessageType: &protos.SendEntityMessageAction_EntityLockRequested{ + EntityLockRequested: &protos.EntityLockRequestedEvent{ + CriticalSectionId: criticalSectionID, + LockSet: append([]string(nil), lockSet...), + ParentInstanceId: wrapperspb.String(parentInstanceID), + }, + }, + }, + }, + } +} + +func NewEntityUnlockSentAction( + id int32, + criticalSectionID string, + parentInstanceID string, + entityID string, +) *protos.OrchestratorAction { + return &protos.OrchestratorAction{ + Id: id, + OrchestratorActionType: &protos.OrchestratorAction_SendEntityMessage{ + SendEntityMessage: &protos.SendEntityMessageAction{ + EntityMessageType: &protos.SendEntityMessageAction_EntityUnlockSent{ + EntityUnlockSent: &protos.EntityUnlockSentEvent{ + CriticalSectionId: criticalSectionID, + ParentInstanceId: wrapperspb.String(parentInstanceID), + TargetInstanceId: wrapperspb.String(entityID), + }, + }, + }, + }, + } +} + func NewCreateSubOrchestrationAction( taskID int32, name string, iid string, input *wrapperspb.StringValue, + version ...*wrapperspb.StringValue, ) *protos.OrchestratorAction { - return &protos.OrchestratorAction{ + action := &protos.OrchestratorAction{ Id: taskID, OrchestratorActionType: &protos.OrchestratorAction_CreateSubOrchestration{ CreateSubOrchestration: &protos.CreateSubOrchestrationAction{ @@ -289,6 +385,10 @@ func NewCreateSubOrchestrationAction( }, }, } + if len(version) > 0 { + action.GetCreateSubOrchestration().Version = version[0] + } + return action } func NewCompleteOrchestrationAction( @@ -311,94 +411,6 @@ func NewCompleteOrchestrationAction( } } -func NewTerminateOrchestrationAction(taskID int32, iid string, recurse bool, rawReason *wrapperspb.StringValue) *protos.OrchestratorAction { - return &protos.OrchestratorAction{ - Id: taskID, - OrchestratorActionType: &protos.OrchestratorAction_TerminateOrchestration{ - TerminateOrchestration: &protos.TerminateOrchestrationAction{ - InstanceId: iid, - Recurse: recurse, - Reason: rawReason, - }, - }, - } -} - -func NewTaskFailureDetails(err error) *protos.TaskFailureDetails { - if err == nil { - return nil - } - return &protos.TaskFailureDetails{ - ErrorType: reflect.TypeOf(err).String(), - ErrorMessage: err.Error(), - } -} - -func HistoryListSummary(list []*protos.HistoryEvent) string { - var sb strings.Builder - sb.WriteString("[") - for i, e := range list { - if i > 0 { - sb.WriteString(", ") - } - if i >= 10 { - sb.WriteString("...") - break - } - name := getHistoryEventTypeName(e) - sb.WriteString(name) - taskID := GetTaskId(e) - if taskID > -0 { - sb.WriteRune('#') - sb.WriteString(strconv.FormatInt(int64(taskID), 10)) - } - } - sb.WriteString("]") - return sb.String() -} - -func ActionListSummary(actions []*protos.OrchestratorAction) string { - var sb strings.Builder - sb.WriteString("[") - for i, a := range actions { - if i > 0 { - sb.WriteString(", ") - } - if i >= 10 { - sb.WriteString("...") - break - } - name := getActionTypeName(a) - sb.WriteString(name) - if a.Id >= 0 { - sb.WriteRune('#') - sb.WriteString(strconv.FormatInt(int64(a.Id), 10)) - } - } - sb.WriteString("]") - return sb.String() -} - -func GetTaskId(e *protos.HistoryEvent) int32 { - if e.EventId >= 0 { - return e.EventId - } else if x := e.GetTaskCompleted(); x != nil { - return x.TaskScheduledId - } else if x := e.GetTaskFailed(); x != nil { - return x.TaskScheduledId - } else if x := e.GetSubOrchestrationInstanceCompleted(); x != nil { - return x.TaskScheduledId - } else if x := e.GetSubOrchestrationInstanceFailed(); x != nil { - return x.TaskScheduledId - } else if x := e.GetTimerFired(); x != nil { - return x.TimerId - } else if x := e.GetExecutionStarted().GetParentInstance(); x != nil { - return x.TaskScheduledId - } else { - return -1 - } -} - func ToRuntimeStatusString(status protos.OrchestrationStatus) string { name := protos.OrchestrationStatus_name[int32(status)] return name[len("ORCHESTRATION_STATUS_"):] @@ -408,13 +420,3 @@ func FromRuntimeStatusString(status string) protos.OrchestrationStatus { runtimeStatus := "ORCHESTRATION_STATUS_" + status return protos.OrchestrationStatus(protos.OrchestrationStatus_value[runtimeStatus]) } - -func getHistoryEventTypeName(e *protos.HistoryEvent) string { - // PERFORMANCE: Replace this with a switch statement or a map lookup to avoid this use of reflection - return reflect.TypeOf(e.EventType).Elem().Name()[len("HistoryEvent_"):] -} - -func getActionTypeName(a *protos.OrchestratorAction) string { - // PERFORMANCE: Replace this with a switch statement or a map lookup to avoid this use of reflection - return reflect.TypeOf(a.OrchestratorActionType).Elem().Name()[len("OrchestratorAction_"):] -} diff --git a/internal/helpers/tracing.go b/internal/helpers/tracing.go index 492241bb..41d137c3 100644 --- a/internal/helpers/tracing.go +++ b/internal/helpers/tracing.go @@ -2,230 +2,85 @@ package helpers import ( "context" - "encoding/hex" - "reflect" - "strings" - "time" - "unsafe" + "crypto/rand" + "fmt" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/trace" "google.golang.org/protobuf/types/known/wrapperspb" "github.com/microsoft/durabletask-go/internal/protos" ) -var tracer = otel.Tracer("durabletask") - -func StartNewCreateOrchestrationSpan( - ctx context.Context, name string, version string, instanceID string, -) (context.Context, trace.Span) { - attributes := []attribute.KeyValue{ - {Key: "durabletask.type", Value: attribute.StringValue("orchestration")}, - {Key: "durabletask.task.name", Value: attribute.StringValue(name)}, - {Key: "durabletask.task.instance_id", Value: attribute.StringValue(instanceID)}, - } - return startNewSpan(ctx, "create_orchestration", name, version, attributes, trace.SpanKindClient, time.Now().UTC()) -} - -func StartNewRunOrchestrationSpan( - ctx context.Context, es *protos.ExecutionStartedEvent, startedTime time.Time, -) (context.Context, trace.Span) { - name := es.Name - instanceID := es.OrchestrationInstance.InstanceId - version := es.Version.GetValue() - attributes := []attribute.KeyValue{ - {Key: "durabletask.type", Value: attribute.StringValue("orchestration")}, - {Key: "durabletask.task.name", Value: attribute.StringValue(name)}, - {Key: "durabletask.task.instance_id", Value: attribute.StringValue(instanceID)}, - } - return startNewSpan(ctx, "orchestration", name, version, attributes, trace.SpanKindServer, startedTime) -} - -func StartNewActivitySpan( - ctx context.Context, name string, version string, instanceID string, taskID int32, -) (context.Context, trace.Span) { - attributes := []attribute.KeyValue{ - {Key: "durabletask.type", Value: attribute.StringValue("activity")}, - {Key: "durabletask.task.name", Value: attribute.StringValue(name)}, - {Key: "durabletask.task.task_id", Value: attribute.Int64Value(int64(taskID))}, - {Key: "durabletask.task.instance_id", Value: attribute.StringValue(instanceID)}, +// TraceContextFromSpan converts a sampled OpenTelemetry span into the W3C trace +// context sent to Durable Task Scheduler. Unsampled or invalid spans return nil. +func TraceContextFromSpan(span trace.Span) *protos.TraceContext { + if span == nil || !span.SpanContext().IsSampled() { + return nil } - return startNewSpan(ctx, "activity", name, version, attributes, trace.SpanKindServer, time.Now().UTC()) -} - -func StartAndEndNewTimerSpan(ctx context.Context, tf *protos.TimerFiredEvent, createdTime time.Time, instanceID string) error { - attributes := []attribute.KeyValue{ - {Key: "durabletask.type", Value: attribute.StringValue("timer")}, - {Key: "durabletask.fire_at", Value: attribute.StringValue(tf.FireAt.AsTime().Format(time.RFC3339))}, // time.RFC3339 most closely maps to ISO 8601 - {Key: "durabletask.task.task_id", Value: attribute.Int64Value(int64(tf.TimerId))}, - {Key: "durabletask.task.instance_id", Value: attribute.StringValue(instanceID)}, + spanContext := span.SpanContext() + if !spanContext.IsValid() { + return nil } - - _, span := startNewSpan(ctx, "timer", "", "", attributes, trace.SpanKindInternal, createdTime) - span.End() - return nil -} - -func startNewSpan( - ctx context.Context, - taskType string, - taskName string, - taskVersion string, - attributes []attribute.KeyValue, - kind trace.SpanKind, - timestamp time.Time, -) (context.Context, trace.Span) { - var spanName string - switch { - case taskVersion != "": - spanName = taskType + "||" + taskName + "||" + taskVersion - attributes = append(attributes, attribute.KeyValue{ - Key: "durabletask.task.version", - Value: attribute.StringValue(taskVersion), - }) - case taskName != "": - spanName = taskType + "||" + taskName - default: - spanName = taskType + return &protos.TraceContext{ + TraceParent: traceParent(spanContext.TraceID(), spanContext.SpanID(), spanContext.TraceFlags()), + TraceState: traceStateValue(spanContext.TraceState()), } - - var span trace.Span - ctx, span = tracer.Start( - ctx, - spanName, - trace.WithSpanKind(kind), - trace.WithTimestamp(timestamp), - trace.WithAttributes(attributes...), - ) - return ctx, span } -func UnsafeSetSpanContext(span trace.Span, spanContext trace.SpanContext) bool { - if !span.IsRecording() { - // this logic only applies to recording spans - return false - } - spanPtr := reflect.ValueOf(span) - spanVal := reflect.Indirect(spanPtr) - spanContextField := spanVal.FieldByName("spanContext") - if !spanContextField.IsValid() || spanContextField.IsZero() { - // The spanContext field doesn't exist - return false - } - // TODO: Validate the type of the field? - spanContextPtr := unsafe.Pointer(spanContextField.UnsafeAddr()) - realPtrToSpanContext := (*trace.SpanContext)(spanContextPtr) - *realPtrToSpanContext = spanContext - return true +// traceParent formats the W3C traceparent header value for the given +// version-00 components. +func traceParent(traceID trace.TraceID, spanID trace.SpanID, flags trace.TraceFlags) string { + return "00-" + traceID.String() + "-" + spanID.String() + "-" + flags.String() } -func ContextFromTraceContext(ctx context.Context, tc *protos.TraceContext) (context.Context, error) { - if tc == nil { - return ctx, nil +// traceStateValue wraps a non-empty W3C tracestate for the wire, or returns +// nil if the tracestate is absent. +func traceStateValue(state trace.TraceState) *wrapperspb.StringValue { + if value := state.String(); value != "" { + return wrapperspb.String(value) } - - spanContext, err := SpanContextFromTraceContext(tc) - if err != nil { - return ctx, err - } - - ctx = trace.ContextWithRemoteSpanContext(ctx, spanContext) - return ctx, nil + return nil } -func SpanContextFromTraceContext(tc *protos.TraceContext) (trace.SpanContext, error) { - var decodedTraceID trace.TraceID - var err error - var traceID string - var spanID string - var traceFlags string - - parts := strings.Split(tc.TraceParent, "-") - if len(parts) == 4 { - traceID = parts[1] - spanID = parts[2] - traceFlags = parts[3] - } else { - // backwards compatibility with older versions of the protobuf - traceID = tc.GetTraceParent() - spanID = tc.GetSpanID() //nolint:staticcheck // backwards compatibility with older versions of the protobuf - traceFlags = "01" // sampled - } - - decodedTraceID, err = trace.TraceIDFromHex(traceID) - if err != nil { - return trace.SpanContext{}, err - } - - var decodedSpanID trace.SpanID - decodedSpanID, err = trace.SpanIDFromHex(spanID) - if err != nil { - return trace.SpanContext{}, err +// OrchestratorActionTraceContext creates the trace context for a service-owned +// activity or sub-orchestration scheduling span. +func OrchestratorActionTraceContext(parent *protos.TraceContext) (*protos.TraceContext, error) { + if parent == nil { + return nil, nil } - - var decodedTraceFlags []byte - decodedTraceFlags, err = hex.DecodeString(traceFlags) - if err != nil { - return trace.SpanContext{}, err + carrier := propagation.MapCarrier{"traceparent": parent.GetTraceParent()} + if traceState := parent.GetTraceState().GetValue(); traceState != "" { + carrier.Set("tracestate", traceState) } - - spanContextConfig := trace.SpanContextConfig{ - TraceID: decodedTraceID, - SpanID: decodedSpanID, - TraceFlags: trace.TraceFlags(decodedTraceFlags[0]), + parentContext := trace.SpanContextFromContext( + propagation.TraceContext{}.Extract(context.Background(), carrier), + ) + if !parentContext.IsValid() { + return nil, fmt.Errorf("invalid parent trace context") } - // Trace state is optional - if traceState := tc.TraceState.GetValue(); traceState != "" { - var ts trace.TraceState - ts, err = trace.ParseTraceState(traceState) - if err != nil { - return trace.SpanContext{}, err + var spanID trace.SpanID + for !spanID.IsValid() { + if _, err := rand.Read(spanID[:]); err != nil { + return nil, fmt.Errorf("generate action span ID: %w", err) } - spanContextConfig.TraceState = ts } - spanContext := trace.NewSpanContext(spanContextConfig) - return spanContext, nil + + return &protos.TraceContext{ + TraceParent: traceParent(parentContext.TraceID(), spanID, parentContext.TraceFlags()), + TraceState: traceStateValue(parentContext.TraceState()), + }, nil } -func TraceContextFromSpan(span trace.Span) *protos.TraceContext { - if span == nil { - return nil - } else if !span.SpanContext().IsSampled() { - // Don't apply trace context for anything that's not being sampled. Note that by doing this, - // we're ensuring a parent-based sampling strategy. More information on OTel sampling here: - // https://opentelemetry.io/docs/instrumentation/go/exporting_data/#sampling +// CloneTraceContext returns an independent copy of the active W3C trace fields. +func CloneTraceContext(value *protos.TraceContext) *protos.TraceContext { + if value == nil { return nil } - - var tc *protos.TraceContext - spanContext := span.SpanContext() - if spanContext.IsValid() { - tc = &protos.TraceContext{ - TraceParent: "00-" + spanContext.TraceID().String() + "-" + spanContext.SpanID().String() + "-" + spanContext.TraceFlags().String(), - } - if ts := spanContext.TraceState().String(); ts != "" { - tc.TraceState = wrapperspb.String(ts) - } + result := &protos.TraceContext{TraceParent: value.GetTraceParent()} + if value.TraceState != nil { + result.TraceState = wrapperspb.String(value.GetTraceState().GetValue()) } - return tc -} - -func ChangeSpanID(span trace.Span, newSpanID trace.SpanID) { - modifiedSpanContext := span.SpanContext().WithSpanID(newSpanID) - UnsafeSetSpanContext(span, modifiedSpanContext) -} - -func CancelSpan(span trace.Span) { - if span.SpanContext().IsSampled() { - // set the IsSampled flag to 0 (not sampled) - modifiedSpanContext := span.SpanContext().WithTraceFlags(trace.TraceFlags(0)) - UnsafeSetSpanContext(span, modifiedSpanContext) - } -} - -func NoopSpan() trace.Span { - return trace.SpanFromContext(context.Background()) + return result } diff --git a/internal/helpers/tracing_test.go b/internal/helpers/tracing_test.go new file mode 100644 index 00000000..808bd950 --- /dev/null +++ b/internal/helpers/tracing_test.go @@ -0,0 +1,47 @@ +package helpers + +import ( + "strings" + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestOrchestratorActionTraceContextCreatesDistinctScheduleSpans(t *testing.T) { + parent := &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + TraceState: wrapperspb.String("vendor=value"), + } + first, err := OrchestratorActionTraceContext(parent) + if err != nil { + t.Fatal(err) + } + second, err := OrchestratorActionTraceContext(parent) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(first.TraceParent, "00-0123456789abcdef0123456789abcdef-") { + t.Fatalf("first trace parent = %q", first.TraceParent) + } + if first.TraceParent == parent.TraceParent || first.TraceParent == second.TraceParent { + t.Fatalf("action trace parents must use fresh span IDs: first=%q second=%q", first.TraceParent, second.TraceParent) + } + if first.GetTraceState().GetValue() != "vendor=value" { + t.Fatalf("trace state = %q", first.GetTraceState().GetValue()) + } +} + +func TestOrchestratorActionTraceContextRejectsInvalidContext(t *testing.T) { + for _, parent := range []*protos.TraceContext{ + {TraceParent: "invalid"}, + {TraceParent: "00-00000000000000000000000000000000-0123456789abcdef-01"}, + } { + if got, err := OrchestratorActionTraceContext(parent); err == nil || got != nil { + t.Fatalf("OrchestratorActionTraceContext(%v) = %v, %v", parent, got, err) + } + } + if got, err := OrchestratorActionTraceContext(nil); err != nil || got != nil { + t.Fatalf("OrchestratorActionTraceContext(nil) = %v, %v", got, err) + } +} diff --git a/internal/helpers/worker.go b/internal/helpers/worker.go index a17005a0..0566198f 100644 --- a/internal/helpers/worker.go +++ b/internal/helpers/worker.go @@ -1,25 +1,6 @@ package helpers -import ( - "fmt" - "os" - - "github.com/google/uuid" -) - -func GetDefaultWorkerName() string { - hostname, err := os.Hostname() - if err != nil { - hostname = "unknown" - } - - pid := os.Getpid() - u, err := uuid.NewV7() - var uuidStr string - if err == nil { - uuidStr = u.String() - } else { - uuidStr = uuid.NewString() - } - return fmt.Sprintf("%v,%d,%v", hostname, pid, uuidStr) -} +// RejectAllWorkItemFilterName is the sentinel orchestration/activity name a +// worker advertises when its registry matches nothing, so the scheduler sends +// it no work instead of falling back to unfiltered delivery. +const RejectAllWorkItemFilterName = "\x00__durabletask_reject_all__" diff --git a/internal/historyconv/history.go b/internal/historyconv/history.go new file mode 100644 index 00000000..fc3bc5eb --- /dev/null +++ b/internal/historyconv/history.go @@ -0,0 +1,344 @@ +package historyconv + +import ( + "fmt" + "reflect" + "slices" + "strings" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" +) + +// Converter converts one sequential history stream into API-owned records while +// preserving the current orchestration identity for legacy entity events that +// omit it. A Converter is not safe for concurrent use or reuse across streams. +type Converter struct { + converter api.DataConverter + currentInstanceID string + currentExecutionID string +} + +func New(converter api.DataConverter) *Converter { + return &Converter{converter: api.NormalizeDataConverter(converter)} +} + +func (c *Converter) Convert(event *protos.HistoryEvent) (*api.HistoryEvent, error) { + if event == nil { + return nil, fmt.Errorf("history event must not be nil") + } + result := &api.HistoryEvent{ + Type: api.HistoryEventUnknown, + EventID: event.GetEventId(), + Converter: c.converter, + } + if event.GetTimestamp() != nil { + result.Timestamp = event.GetTimestamp().AsTime() + } + + switch value := event.GetEventType().(type) { + case *protos.HistoryEvent_ExecutionStarted: + started := value.ExecutionStarted + instance := started.GetOrchestrationInstance() + c.currentInstanceID = instance.GetInstanceId() + c.currentExecutionID = instance.GetExecutionId().GetValue() + result.Type = api.HistoryEventExecutionStarted + result.ExecutionStarted = &api.HistoryExecutionStartedEvent{ + Name: started.GetName(), + Version: started.GetVersion().GetValue(), + InstanceID: api.InstanceID(c.currentInstanceID), + ExecutionID: c.currentExecutionID, + SerializedInput: started.GetInput().GetValue(), + Parent: parentInfo(started.GetParentInstance()), + ParentTraceContext: traceContext(started.GetParentTraceContext()), + OrchestrationSpanID: started.GetOrchestrationSpanID().GetValue(), + Tags: tagcodec.DecodeUserTagsOrPlain(started.GetTags()), + ContextFields: api.ContextFields(tagcodec.DecodeContextFields(started.GetTags())), + } + if started.GetScheduledStartTimestamp() != nil { + result.ExecutionStarted.ScheduledStartAt = started.GetScheduledStartTimestamp().AsTime() + } + case *protos.HistoryEvent_ExecutionCompleted: + result.Type = api.HistoryEventExecutionCompleted + result.ExecutionCompleted = &api.HistoryExecutionCompletedEvent{ + RuntimeStatus: value.ExecutionCompleted.GetOrchestrationStatus(), + SerializedResult: value.ExecutionCompleted.GetResult().GetValue(), + FailureDetails: failure.FromProto(value.ExecutionCompleted.GetFailureDetails()), + } + case *protos.HistoryEvent_ExecutionTerminated: + result.Type = api.HistoryEventExecutionTerminated + result.ExecutionTerminated = &api.HistoryExecutionTerminatedEvent{ + SerializedInput: value.ExecutionTerminated.GetInput().GetValue(), + Recursive: value.ExecutionTerminated.GetRecurse(), + } + case *protos.HistoryEvent_TaskScheduled: + result.Type = api.HistoryEventTaskScheduled + result.TaskScheduled = &api.HistoryTaskScheduledEvent{ + Name: value.TaskScheduled.GetName(), + Version: value.TaskScheduled.GetVersion().GetValue(), + SerializedInput: value.TaskScheduled.GetInput().GetValue(), + ParentTraceContext: traceContext(value.TaskScheduled.GetParentTraceContext()), + Tags: tagcodec.DecodeUserTagsOrPlain(value.TaskScheduled.GetTags()), + ContextFields: api.ContextFields(tagcodec.DecodeContextFields(value.TaskScheduled.GetTags())), + } + case *protos.HistoryEvent_TaskCompleted: + result.Type = api.HistoryEventTaskCompleted + result.TaskCompleted = taskResult(value.TaskCompleted.GetTaskScheduledId(), value.TaskCompleted.GetResult().GetValue()) + case *protos.HistoryEvent_TaskFailed: + result.Type = api.HistoryEventTaskFailed + result.TaskFailed = taskFailure(value.TaskFailed.GetTaskScheduledId(), value.TaskFailed.GetFailureDetails()) + case *protos.HistoryEvent_SubOrchestrationInstanceCreated: + created := value.SubOrchestrationInstanceCreated + result.Type = api.HistoryEventSubOrchestrationInstanceCreated + result.SubOrchestrationInstanceCreated = &api.HistorySubOrchestrationInstanceCreatedEvent{ + InstanceID: api.InstanceID(created.GetInstanceId()), + Name: created.GetName(), + Version: created.GetVersion().GetValue(), + SerializedInput: created.GetInput().GetValue(), + ParentTraceContext: traceContext(created.GetParentTraceContext()), + Tags: tagcodec.DecodeUserTagsOrPlain(created.GetTags()), + ContextFields: api.ContextFields(tagcodec.DecodeContextFields(created.GetTags())), + } + case *protos.HistoryEvent_SubOrchestrationInstanceCompleted: + result.Type = api.HistoryEventSubOrchestrationInstanceCompleted + result.SubOrchestrationInstanceCompleted = taskResult( + value.SubOrchestrationInstanceCompleted.GetTaskScheduledId(), + value.SubOrchestrationInstanceCompleted.GetResult().GetValue(), + ) + case *protos.HistoryEvent_SubOrchestrationInstanceFailed: + result.Type = api.HistoryEventSubOrchestrationInstanceFailed + result.SubOrchestrationInstanceFailed = taskFailure( + value.SubOrchestrationInstanceFailed.GetTaskScheduledId(), + value.SubOrchestrationInstanceFailed.GetFailureDetails(), + ) + case *protos.HistoryEvent_TimerCreated: + result.Type = api.HistoryEventTimerCreated + result.TimerCreated = &api.HistoryTimerEvent{} + if value.TimerCreated.GetFireAt() != nil { + result.TimerCreated.FireAt = value.TimerCreated.GetFireAt().AsTime() + } + case *protos.HistoryEvent_TimerFired: + result.Type = api.HistoryEventTimerFired + result.TimerFired = &api.HistoryTimerFiredEvent{TimerID: value.TimerFired.GetTimerId()} + if value.TimerFired.GetFireAt() != nil { + result.TimerFired.FireAt = value.TimerFired.GetFireAt().AsTime() + } + case *protos.HistoryEvent_OrchestratorStarted: + result.Type = api.HistoryEventOrchestratorStarted + case *protos.HistoryEvent_OrchestratorCompleted: + result.Type = api.HistoryEventOrchestratorCompleted + case *protos.HistoryEvent_EventSent: + result.Type = api.HistoryEventEventSent + result.EventSent = &api.HistoryExternalEvent{ + InstanceID: api.InstanceID(value.EventSent.GetInstanceId()), + Name: value.EventSent.GetName(), + SerializedInput: value.EventSent.GetInput().GetValue(), + } + case *protos.HistoryEvent_EventRaised: + result.Type = api.HistoryEventEventRaised + result.EventRaised = &api.HistoryExternalEvent{ + Name: value.EventRaised.GetName(), + SerializedInput: value.EventRaised.GetInput().GetValue(), + } + case *protos.HistoryEvent_GenericEvent: + result.Type = api.HistoryEventGeneric + result.Generic = payloadEvent(value.GenericEvent.GetData().GetValue()) + case *protos.HistoryEvent_HistoryState: + result.Type = api.HistoryEventHistoryState + result.HistoryState = &api.HistoryStateEvent{ + State: orchestrationMetadata(value.HistoryState.GetOrchestrationState(), c.converter), + } + case *protos.HistoryEvent_ContinueAsNew: + result.Type = api.HistoryEventContinueAsNew + result.ContinueAsNew = payloadEvent(value.ContinueAsNew.GetInput().GetValue()) + case *protos.HistoryEvent_ExecutionSuspended: + result.Type = api.HistoryEventExecutionSuspended + result.ExecutionSuspended = payloadEvent(value.ExecutionSuspended.GetInput().GetValue()) + case *protos.HistoryEvent_ExecutionResumed: + result.Type = api.HistoryEventExecutionResumed + result.ExecutionResumed = payloadEvent(value.ExecutionResumed.GetInput().GetValue()) + case *protos.HistoryEvent_EntityOperationSignaled: + result.Type = api.HistoryEventEntityOperationSignaled + result.Entity = c.entityOperationSignaled(value.EntityOperationSignaled) + case *protos.HistoryEvent_EntityOperationCalled: + result.Type = api.HistoryEventEntityOperationCalled + result.Entity = c.entityOperationCalled(value.EntityOperationCalled) + case *protos.HistoryEvent_EntityOperationCompleted: + result.Type = api.HistoryEventEntityOperationCompleted + result.Entity = &api.HistoryEntityEvent{ + RequestID: value.EntityOperationCompleted.GetRequestId(), + SerializedOutput: value.EntityOperationCompleted.GetOutput().GetValue(), + } + case *protos.HistoryEvent_EntityOperationFailed: + result.Type = api.HistoryEventEntityOperationFailed + result.Entity = &api.HistoryEntityEvent{ + RequestID: value.EntityOperationFailed.GetRequestId(), + FailureDetails: failure.FromProto(value.EntityOperationFailed.GetFailureDetails()), + } + case *protos.HistoryEvent_EntityLockRequested: + result.Type = api.HistoryEventEntityLockRequested + lock := value.EntityLockRequested + result.Entity = &api.HistoryEntityEvent{ + RequestID: lock.GetCriticalSectionId(), + CriticalSectionID: lock.GetCriticalSectionId(), + ParentInstanceID: fallback(lock.GetParentInstanceId().GetValue(), c.currentInstanceID), + LockSet: slices.Clone(lock.GetLockSet()), + Position: lock.GetPosition(), + } + if position := int(lock.GetPosition()); position >= 0 && position < len(lock.GetLockSet()) { + result.Entity.TargetInstanceID = lock.GetLockSet()[position] + } + case *protos.HistoryEvent_EntityLockGranted: + result.Type = api.HistoryEventEntityLockGranted + result.Entity = &api.HistoryEntityEvent{ + RequestID: value.EntityLockGranted.GetCriticalSectionId(), + CriticalSectionID: value.EntityLockGranted.GetCriticalSectionId(), + } + case *protos.HistoryEvent_EntityUnlockSent: + result.Type = api.HistoryEventEntityUnlockSent + unlock := value.EntityUnlockSent + result.Entity = &api.HistoryEntityEvent{ + RequestID: unlock.GetCriticalSectionId(), + CriticalSectionID: unlock.GetCriticalSectionId(), + ParentInstanceID: fallback(unlock.GetParentInstanceId().GetValue(), c.currentInstanceID), + TargetInstanceID: unlock.GetTargetInstanceId().GetValue(), + } + case *protos.HistoryEvent_ExecutionRewound: + rewound := value.ExecutionRewound + result.Type = api.HistoryEventExecutionRewound + result.ExecutionRewound = &api.HistoryExecutionRewoundEvent{ + Reason: rewound.GetReason().GetValue(), + Name: rewound.GetName().GetValue(), + Version: rewound.GetVersion().GetValue(), + InstanceID: api.InstanceID(rewound.GetInstanceId().GetValue()), + ParentExecutionID: rewound.GetParentExecutionId().GetValue(), + SerializedInput: rewound.GetInput().GetValue(), + Parent: parentInfo(rewound.GetParentInstance()), + ParentTraceContext: traceContext(rewound.GetParentTraceContext()), + Tags: tagcodec.DecodeUserTagsOrPlain(rewound.GetTags()), + ContextFields: api.ContextFields(tagcodec.DecodeContextFields(rewound.GetTags())), + } + case nil: + // Preserve the envelope as an unknown event for forward compatibility. + default: + eventType := reflect.TypeOf(value) + if eventType.Kind() == reflect.Pointer { + eventType = eventType.Elem() + } + result.UnknownType = strings.TrimPrefix(eventType.Name(), "HistoryEvent_") + } + return result, nil +} + +func (c *Converter) entityOperationSignaled(value *protos.EntityOperationSignaledEvent) *api.HistoryEntityEvent { + result := &api.HistoryEntityEvent{ + RequestID: value.GetRequestId(), + Operation: value.GetOperation(), + TargetInstanceID: value.GetTargetInstanceId().GetValue(), + SerializedInput: value.GetInput().GetValue(), + } + if value.GetScheduledTime() != nil { + result.ScheduledAt = value.GetScheduledTime().AsTime() + } + return result +} + +func (c *Converter) entityOperationCalled(value *protos.EntityOperationCalledEvent) *api.HistoryEntityEvent { + result := &api.HistoryEntityEvent{ + RequestID: value.GetRequestId(), + Operation: value.GetOperation(), + TargetInstanceID: value.GetTargetInstanceId().GetValue(), + ParentInstanceID: fallback(value.GetParentInstanceId().GetValue(), c.currentInstanceID), + ParentExecutionID: fallback(value.GetParentExecutionId().GetValue(), c.currentExecutionID), + SerializedInput: value.GetInput().GetValue(), + } + if value.GetScheduledTime() != nil { + result.ScheduledAt = value.GetScheduledTime().AsTime() + } + return result +} + +func taskResult(taskScheduledID int32, result string) *api.HistoryTaskResultEvent { + return &api.HistoryTaskResultEvent{TaskScheduledID: taskScheduledID, SerializedResult: result} +} + +func taskFailure(taskScheduledID int32, details *protos.TaskFailureDetails) *api.HistoryTaskFailureEvent { + return &api.HistoryTaskFailureEvent{ + TaskScheduledID: taskScheduledID, + FailureDetails: failure.FromProto(details), + } +} + +func payloadEvent(payload string) *api.HistoryPayloadEvent { + return &api.HistoryPayloadEvent{SerializedInput: payload} +} + +func parentInfo(parent *protos.ParentInstanceInfo) *api.HistoryParentInstanceInfo { + if parent == nil { + return nil + } + instance := parent.GetOrchestrationInstance() + return &api.HistoryParentInstanceInfo{ + Name: parent.GetName().GetValue(), + Version: parent.GetVersion().GetValue(), + InstanceID: api.InstanceID(instance.GetInstanceId()), + ExecutionID: instance.GetExecutionId().GetValue(), + TaskScheduledID: parent.GetTaskScheduledId(), + } +} + +func traceContext(value *protos.TraceContext) *api.HistoryTraceContext { + if value == nil { + return nil + } + return &api.HistoryTraceContext{ + TraceParent: value.GetTraceParent(), + TraceState: value.GetTraceState().GetValue(), + SpanID: value.GetSpanID(), //nolint:staticcheck // preserve the deprecated wire field. + } +} + +func orchestrationMetadata( + state *protos.OrchestrationState, + converter api.DataConverter, +) *api.OrchestrationMetadata { + if state == nil { + return nil + } + result := &api.OrchestrationMetadata{ + InstanceID: api.InstanceID(state.GetInstanceId()), + Name: state.GetName(), + Version: state.GetVersion().GetValue(), + ExecutionID: state.GetExecutionId().GetValue(), + ParentInstanceID: api.InstanceID(state.GetParentInstanceId().GetValue()), + RuntimeStatus: state.GetOrchestrationStatus(), + SerializedInput: state.GetInput().GetValue(), + SerializedOutput: state.GetOutput().GetValue(), + SerializedCustomStatus: state.GetCustomStatus().GetValue(), + FailureDetails: failure.FromProto(state.GetFailureDetails()), + Tags: tagcodec.DecodeUserTagsOrPlain(state.GetTags()), + Converter: converter, + } + if state.GetScheduledStartTimestamp() != nil { + result.ScheduledStartAt = state.GetScheduledStartTimestamp().AsTime() + } + if state.GetCreatedTimestamp() != nil { + result.CreatedAt = state.GetCreatedTimestamp().AsTime() + } + if state.GetLastUpdatedTimestamp() != nil { + result.LastUpdatedAt = state.GetLastUpdatedTimestamp().AsTime() + } + if state.GetCompletedTimestamp() != nil { + result.CompletedAt = state.GetCompletedTimestamp().AsTime() + } + return result +} + +func fallback(value, fallbackValue string) string { + if value != "" { + return value + } + return fallbackValue +} diff --git a/internal/historyconv/history_test.go b/internal/historyconv/history_test.go new file mode 100644 index 00000000..f490e49d --- /dev/null +++ b/internal/historyconv/history_test.go @@ -0,0 +1,133 @@ +package historyconv + +import ( + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestConvertPreservesExecutionAndEntityDetails(t *testing.T) { + timestamp := time.Date(2026, 8, 27, 12, 30, 0, 0, time.UTC) + converter := New(api.DefaultDataConverter()) + started, err := converter.Convert(&protos.HistoryEvent{ + EventId: 4, + Timestamp: timestamppb.New(timestamp), + EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{ + Name: "orchestrator", + Version: wrapperspb.String("2.0"), + Input: wrapperspb.String(`{"value":1}`), + OrchestrationInstance: &protos.OrchestrationInstance{ + InstanceId: "instance", + ExecutionId: wrapperspb.String("execution"), + }, + Tags: map[string]string{ + "tag": "value", + "__durabletask.context.encoding": "1", + "__durabletask.context.field.tenant": "north", + "__durabletask.context.instance_id": "instance", + "__durabletask.context.orchestration_name": "orchestrator", + }, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, api.HistoryEventExecutionStarted, started.Type) + require.EqualValues(t, 4, started.EventID) + require.Equal(t, timestamp, started.Timestamp) + require.Equal(t, api.InstanceID("instance"), started.ExecutionStarted.InstanceID) + require.Equal(t, "execution", started.ExecutionStarted.ExecutionID) + require.Equal(t, map[string]string{"tag": "value"}, started.ExecutionStarted.Tags) + require.Equal(t, api.ContextFields{"tenant": "north"}, started.ExecutionStarted.ContextFields) + + entity, err := converter.Convert(&protos.HistoryEvent{ + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: "request", + Operation: "add", + TargetInstanceId: wrapperspb.String("@counter@one"), + Input: wrapperspb.String("1"), + }, + }, + }) + require.NoError(t, err) + require.Equal(t, api.HistoryEventEntityOperationCalled, entity.Type) + require.Equal(t, "instance", entity.Entity.ParentInstanceID) + require.Equal(t, "execution", entity.Entity.ParentExecutionID) + require.Equal(t, "@counter@one", entity.Entity.TargetInstanceID) +} + +func TestConvertEntityWithoutExecutionStarted(t *testing.T) { + event, err := New(nil).Convert(&protos.HistoryEvent{ + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{ + RequestId: "request", + ParentInstanceId: wrapperspb.String("parent"), + ParentExecutionId: wrapperspb.String("execution"), + }, + }, + }) + require.NoError(t, err) + require.Equal(t, "parent", event.Entity.ParentInstanceID) + require.Equal(t, "execution", event.Entity.ParentExecutionID) +} + +func TestConvertPreservesRewindAndHistoryState(t *testing.T) { + converter := New(nil) + rewound, err := converter.Convert(&protos.HistoryEvent{ + EventId: 9, + EventType: &protos.HistoryEvent_ExecutionRewound{ + ExecutionRewound: &protos.ExecutionRewoundEvent{ + Reason: wrapperspb.String("retry"), + Name: wrapperspb.String("orchestrator"), + Version: wrapperspb.String("3.0"), + InstanceId: wrapperspb.String("instance"), + ParentExecutionId: wrapperspb.String("parent-execution"), + Input: wrapperspb.String(`"input"`), + Tags: map[string]string{"tag": "value"}, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, api.HistoryEventExecutionRewound, rewound.Type) + require.Equal(t, "retry", rewound.ExecutionRewound.Reason) + require.Equal(t, api.InstanceID("instance"), rewound.ExecutionRewound.InstanceID) + require.Equal(t, map[string]string{"tag": "value"}, rewound.ExecutionRewound.Tags) + + completedAt := time.Date(2026, 8, 27, 13, 0, 0, 0, time.UTC) + state, err := converter.Convert(&protos.HistoryEvent{ + EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{ + OrchestrationState: &protos.OrchestrationState{ + InstanceId: "instance", + ExecutionId: wrapperspb.String("execution"), + Name: "orchestrator", + OrchestrationStatus: api.RUNTIME_STATUS_COMPLETED, + Output: wrapperspb.String(`"done"`), + CompletedTimestamp: timestamppb.New(completedAt), + }, + }, + }, + }) + require.NoError(t, err) + require.Equal(t, "execution", state.HistoryState.State.ExecutionID) + require.Equal(t, completedAt, state.HistoryState.State.CompletedAt) + require.Equal(t, `"done"`, state.HistoryState.State.SerializedOutput) +} + +func TestConvertPreservesTimerEnvelopeEventID(t *testing.T) { + event, err := New(nil).Convert(&protos.HistoryEvent{ + EventId: 17, + EventType: &protos.HistoryEvent_TimerFired{ + TimerFired: &protos.TimerFiredEvent{TimerId: 5}, + }, + }) + require.NoError(t, err) + require.EqualValues(t, 17, event.EventID) + require.EqualValues(t, 5, event.TimerFired.TimerID) +} diff --git a/internal/historyconv/query.go b/internal/historyconv/query.go new file mode 100644 index 00000000..921f2eb8 --- /dev/null +++ b/internal/historyconv/query.go @@ -0,0 +1,79 @@ +package historyconv + +import ( + "errors" + "fmt" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" +) + +// NormalizeStreamRequest validates the arguments shared by every +// StreamOrchestrationHistory implementation and applies query defaults. +func NormalizeStreamRequest( + id api.InstanceID, + query api.HistoryQuery, + handler api.HistoryEventHandler, +) (api.HistoryQuery, error) { + if id == api.EmptyInstanceID { + return api.HistoryQuery{}, api.WrapInvalidArgument(errors.New("instance ID cannot be empty")) + } + if err := helpers.ValidateOrchestrationInstanceID(string(id)); err != nil { + return api.HistoryQuery{}, api.WrapInvalidArgument(err) + } + if handler == nil { + return api.HistoryQuery{}, api.WrapInvalidArgument(errors.New("history event handler is required")) + } + return api.NormalizeHistoryQuery(query) +} + +// Collect buffers a bounded history snapshot from a streaming history read. +func Collect( + id api.InstanceID, + query api.HistoryQuery, + stream func(api.HistoryEventHandler) error, +) (*api.OrchestrationHistory, error) { + normalized, err := api.NormalizeHistoryQuery(query) + if err != nil { + return nil, err + } + result := &api.OrchestrationHistory{InstanceID: id} + totalBytes := 0 + err = stream(func(event *api.HistoryEvent) error { + if event == nil { + return errors.New("history event must not be nil") + } + if len(result.Events) >= normalized.MaxEvents { + return fmt.Errorf("%w: limit %d", api.ErrHistoryLimitExceeded, normalized.MaxEvents) + } + if event.ExecutionStarted != nil || event.Type == api.HistoryEventExecutionStarted { + started := event.ExecutionStarted + if started == nil || started.ExecutionID == "" { + return errors.New("history ExecutionStarted event is missing an execution ID") + } + if normalized.ExecutionID != "" && started.ExecutionID != normalized.ExecutionID { + return fmt.Errorf("history execution ID %q does not match requested execution %q", + started.ExecutionID, normalized.ExecutionID) + } + if result.ExecutionID != "" && started.ExecutionID != result.ExecutionID { + return fmt.Errorf("history contains conflicting execution IDs %q and %q", + result.ExecutionID, started.ExecutionID) + } + result.ExecutionID = started.ExecutionID + } + totalBytes = addSize(totalBytes, ApproximateEventSize(event)) + if totalBytes > normalized.MaxBytes { + return fmt.Errorf("%w: byte limit %d", api.ErrHistoryLimitExceeded, normalized.MaxBytes) + } + result.Events = append(result.Events, event) + return nil + }) + if err != nil { + return nil, err + } + if normalized.ExecutionID != "" && result.ExecutionID == "" { + return nil, fmt.Errorf("history is missing an ExecutionStarted event for requested execution %q", + normalized.ExecutionID) + } + return result, nil +} diff --git a/internal/historyconv/query_test.go b/internal/historyconv/query_test.go new file mode 100644 index 00000000..ca5630c9 --- /dev/null +++ b/internal/historyconv/query_test.go @@ -0,0 +1,329 @@ +package historyconv + +import ( + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/require" +) + +func collectEvents(query api.HistoryQuery, events ...*api.HistoryEvent) (*api.OrchestrationHistory, error) { + return Collect("instance", query, func(handler api.HistoryEventHandler) error { + for _, event := range events { + if err := handler(event); err != nil { + return err + } + } + return nil + }) +} + +func executionStarted(executionID string) *api.HistoryEvent { + return &api.HistoryEvent{ + Type: api.HistoryEventExecutionStarted, + ExecutionStarted: &api.HistoryExecutionStartedEvent{ + InstanceID: "instance", + ExecutionID: executionID, + }, + } +} + +func TestCollectFailureContentByteLimit(t *testing.T) { + large := strings.Repeat("x", 4096) + failures := []struct { + name string + details *api.FailureDetails + }{ + {"error type", &api.FailureDetails{ErrorType: api.ErrorType(large)}}, + {"message", &api.FailureDetails{ErrorMessage: large}}, + {"stack", &api.FailureDetails{StackTrace: large}}, + {"inner failure", &api.FailureDetails{InnerFailure: &api.FailureDetails{StackTrace: large}}}, + {"property key", &api.FailureDetails{Properties: map[string]any{large: nil}}}, + {"nested properties", &api.FailureDetails{Properties: map[string]any{ + "nested": []any{nil, true, float64(1), map[string]any{"value": large}}, + }}}, + {"typed properties", &api.FailureDetails{Properties: map[string]any{ + "nested": []map[string]string{{"value": large}}, + }}}, + {"byte property", &api.FailureDetails{Properties: map[string]any{"value": []byte(large)}}}, + {"struct property", &api.FailureDetails{Properties: map[string]any{ + "value": &struct{ Message string }{Message: large}, + }}}, + {"array property", &api.FailureDetails{Properties: map[string]any{"value": [1]string{large}}}}, + {"retained byte capacity", &api.FailureDetails{Properties: map[string]any{"value": make([]byte, 0, 4096)}}}, + {"retained list capacity", &api.FailureDetails{Properties: map[string]any{"value": []string{large}[:0]}}}, + } + variants := []struct { + name string + event func(*api.FailureDetails) *api.HistoryEvent + }{ + {"execution", func(details *api.FailureDetails) *api.HistoryEvent { + return &api.HistoryEvent{ExecutionCompleted: &api.HistoryExecutionCompletedEvent{FailureDetails: details}} + }}, + {"task", func(details *api.FailureDetails) *api.HistoryEvent { + return &api.HistoryEvent{TaskFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}} + }}, + {"sub-orchestration", func(details *api.FailureDetails) *api.HistoryEvent { + return &api.HistoryEvent{SubOrchestrationInstanceFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}} + }}, + {"entity", func(details *api.FailureDetails) *api.HistoryEvent { + return &api.HistoryEvent{Entity: &api.HistoryEntityEvent{FailureDetails: details}} + }}, + {"history state", func(details *api.FailureDetails) *api.HistoryEvent { + return &api.HistoryEvent{HistoryState: &api.HistoryStateEvent{ + State: &api.OrchestrationMetadata{FailureDetails: details}, + }} + }}, + } + for _, variant := range variants { + for _, failure := range failures { + t.Run(variant.name+"/"+failure.name, func(t *testing.T) { + event := variant.event(failure.details) + result, err := collectEvents(api.HistoryQuery{MaxBytes: 1024}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + require.Greater(t, ApproximateEventSize(event), 1024) + }) + } + } +} + +func TestCollectHistoryStateContentByteLimit(t *testing.T) { + large := strings.Repeat("x", 4096) + states := []struct { + name string + state *api.OrchestrationMetadata + }{ + {"input", &api.OrchestrationMetadata{SerializedInput: large}}, + {"output", &api.OrchestrationMetadata{SerializedOutput: large}}, + {"custom status", &api.OrchestrationMetadata{SerializedCustomStatus: large}}, + {"name", &api.OrchestrationMetadata{Name: large}}, + {"version", &api.OrchestrationMetadata{Version: large}}, + {"instance", &api.OrchestrationMetadata{InstanceID: api.InstanceID(large)}}, + {"execution", &api.OrchestrationMetadata{ExecutionID: large}}, + {"parent instance", &api.OrchestrationMetadata{ParentInstanceID: api.InstanceID(large)}}, + {"tag key", &api.OrchestrationMetadata{Tags: map[string]string{large: "value"}}}, + {"tag value", &api.OrchestrationMetadata{Tags: map[string]string{"key": large}}}, + } + for _, test := range states { + t.Run(test.name, func(t *testing.T) { + event := &api.HistoryEvent{HistoryState: &api.HistoryStateEvent{State: test.state}} + result, err := collectEvents(api.HistoryQuery{MaxBytes: 1024}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + }) + } +} + +func TestCollectParentAndTraceContentByteLimit(t *testing.T) { + large := strings.Repeat("x", 4096) + parents := []struct { + name string + parent *api.HistoryParentInstanceInfo + }{ + {"name", &api.HistoryParentInstanceInfo{Name: large}}, + {"version", &api.HistoryParentInstanceInfo{Version: large}}, + {"instance", &api.HistoryParentInstanceInfo{InstanceID: api.InstanceID(large)}}, + {"execution", &api.HistoryParentInstanceInfo{ExecutionID: large}}, + } + for _, test := range parents { + started := executionStarted("execution") + started.ExecutionStarted.Parent = test.parent + for name, event := range map[string]*api.HistoryEvent{ + "started": started, + "rewound": {ExecutionRewound: &api.HistoryExecutionRewoundEvent{Parent: test.parent}}, + } { + t.Run(name+"/"+test.name, func(t *testing.T) { + result, err := collectEvents(api.HistoryQuery{MaxBytes: 1024}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + }) + } + } + traces := []struct { + name string + trace *api.HistoryTraceContext + }{ + {"parent", &api.HistoryTraceContext{TraceParent: large}}, + {"state", &api.HistoryTraceContext{TraceState: large}}, + {"span", &api.HistoryTraceContext{SpanID: large}}, + } + for _, test := range traces { + started := executionStarted("execution") + started.ExecutionStarted.ParentTraceContext = test.trace + for name, event := range map[string]*api.HistoryEvent{ + "started": started, + "task": {TaskScheduled: &api.HistoryTaskScheduledEvent{ParentTraceContext: test.trace}}, + "sub-orchestration": {SubOrchestrationInstanceCreated: &api.HistorySubOrchestrationInstanceCreatedEvent{ParentTraceContext: test.trace}}, + "rewound": {ExecutionRewound: &api.HistoryExecutionRewoundEvent{ParentTraceContext: test.trace}}, + } { + t.Run(name+"/"+test.name, func(t *testing.T) { + result, err := collectEvents(api.HistoryQuery{MaxBytes: 1024}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + }) + } + } +} + +func TestCollectRetainedContentFailsClosed(t *testing.T) { + cyclicFailure := &api.FailureDetails{} + cyclicFailure.InnerFailure = cyclicFailure + cyclicMap := map[string]any{} + cyclicMap["self"] = cyclicMap + cyclicList := make([]any, 1) + cyclicList[0] = cyclicList + var cyclicPointer any + cyclicPointer = &cyclicPointer + deepFailure := &api.FailureDetails{} + for range api.MaxFailureDetailsDepth { + deepFailure = &api.FailureDetails{InnerFailure: deepFailure} + } + var deepProperty any = "value" + for range 128 { + deepProperty = map[string]any{"nested": []any{deepProperty}} + } + var repeatedProperty any = strings.Repeat("x", 1024*1024) + for range 12 { + repeatedProperty = []any{repeatedProperty, repeatedProperty} + } + for name, details := range map[string]*api.FailureDetails{ + "cyclic failure": cyclicFailure, + "cyclic map": {Properties: cyclicMap}, + "cyclic list": {Properties: map[string]any{"value": cyclicList}}, + "cyclic pointer": {Properties: map[string]any{"value": cyclicPointer}}, + "deep failure": deepFailure, + "deep property": {Properties: map[string]any{"value": deepProperty}}, + "saturated size": {Properties: map[string]any{"value": repeatedProperty}}, + } { + t.Run(name, func(t *testing.T) { + event := &api.HistoryEvent{TaskFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}} + require.Equal(t, api.MaxHistoryMaxBytes+1, ApproximateEventSize(event)) + result, err := collectEvents(api.HistoryQuery{MaxBytes: api.MaxHistoryMaxBytes}, + &api.HistoryEvent{Generic: &api.HistoryPayloadEvent{SerializedInput: "small"}}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + }) + } +} + +func TestRetainedSizeArithmeticSaturates(t *testing.T) { + maxInt := int(^uint(0) >> 1) + require.Equal(t, maxEstimatedSize, addSize(maxEstimatedSize, maxInt)) + require.Equal(t, maxEstimatedSize, addSize(maxInt, maxInt)) + require.Equal(t, maxEstimatedSize, addSize(512, maxInt)) + require.Equal(t, maxEstimatedSize, addSize(api.MaxHistoryMaxBytes, 1)) + require.Equal(t, api.MaxHistoryMaxBytes, addSize(api.MaxHistoryMaxBytes-1, 1)) + require.Equal(t, maxEstimatedSize, multiplySize(maxInt, maxInt)) + require.Equal(t, maxEstimatedSize, multiplySize(maxInt, 16)) + require.Equal(t, 0, multiplySize(maxInt, 0)) + require.Equal(t, 0, multiplySize(0, maxInt)) + require.Equal(t, 64, multiplySize(4, 16)) +} + +func TestCollectAllowsMaximumFailureDepth(t *testing.T) { + details := &api.FailureDetails{} + for range api.MaxFailureDetailsDepth - 1 { + details = &api.FailureDetails{InnerFailure: details} + } + event := &api.HistoryEvent{TaskFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}} + result, err := collectEvents(api.HistoryQuery{MaxBytes: 4096}, event) + require.NoError(t, err) + require.Equal(t, []*api.HistoryEvent{event}, result.Events) +} + +func TestApproximateEventSizeDoesNotAllocateForPayloads(t *testing.T) { + event := &api.HistoryEvent{Generic: &api.HistoryPayloadEvent{SerializedInput: strings.Repeat("x", 4096)}} + var size int + require.Zero(t, testing.AllocsPerRun(100, func() { + size = ApproximateEventSize(event) + })) + require.Greater(t, size, len(event.Generic.SerializedInput)) +} + +func TestCollectRetainsSmallHistoriesWithinTheBudget(t *testing.T) { + details := &api.FailureDetails{ + ErrorType: "failure", + ErrorMessage: "message", + InnerFailure: &api.FailureDetails{StackTrace: "stack"}, + Properties: map[string]any{"nested": []any{nil, true, float64(1), map[string]any{"key": "value"}}}, + } + events := []*api.HistoryEvent{ + executionStarted("execution"), + {ExecutionCompleted: &api.HistoryExecutionCompletedEvent{FailureDetails: details}}, + {HistoryState: &api.HistoryStateEvent{State: &api.OrchestrationMetadata{ + SerializedInput: "input", SerializedOutput: "output", SerializedCustomStatus: "status", + }}}, + } + size := 0 + for _, event := range events { + size += ApproximateEventSize(event) + } + require.Less(t, size, 4096) + result, err := collectEvents(api.HistoryQuery{MaxBytes: size}, events...) + require.NoError(t, err) + require.Equal(t, events, result.Events) + require.Equal(t, "execution", result.ExecutionID) + + result, err = collectEvents(api.HistoryQuery{MaxBytes: size - 1}, events...) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) + result, err = collectEvents(api.HistoryQuery{MaxEvents: len(events) - 1}, events...) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) +} + +func TestCollectCountsAllRetainedDetails(t *testing.T) { + event := executionStarted("execution") + event.TaskFailed = &api.HistoryTaskFailureEvent{FailureDetails: &api.FailureDetails{ + ErrorMessage: strings.Repeat("x", 4096), + }} + result, err := collectEvents(api.HistoryQuery{MaxBytes: 1024}, event) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.Nil(t, result) +} + +func TestCollectExecutionIdentity(t *testing.T) { + generic := &api.HistoryEvent{Type: api.HistoryEventGeneric, Generic: &api.HistoryPayloadEvent{SerializedInput: "value"}} + for _, test := range []struct { + name string + requested string + events []*api.HistoryEvent + observed string + wantError bool + }{ + {"pinned match", "A", []*api.HistoryEvent{executionStarted("A")}, "A", false}, + {"pinned mismatch", "A", []*api.HistoryEvent{executionStarted("B")}, "", true}, + {"pinned mixed", "A", []*api.HistoryEvent{executionStarted("A"), executionStarted("B")}, "", true}, + {"unpinned mixed", "", []*api.HistoryEvent{executionStarted("A"), executionStarted("B")}, "", true}, + {"pinned repeated match", "A", []*api.HistoryEvent{executionStarted("A"), executionStarted("A")}, "A", false}, + {"pinned empty", "A", nil, "", true}, + {"pinned generic", "A", []*api.HistoryEvent{generic}, "", true}, + {"pinned missing ID", "A", []*api.HistoryEvent{executionStarted("")}, "", true}, + {"unpinned missing ID", "", []*api.HistoryEvent{executionStarted("")}, "", true}, + {"pinned missing details", "A", []*api.HistoryEvent{{Type: api.HistoryEventExecutionStarted}}, "", true}, + {"missing repeated ID", "A", []*api.HistoryEvent{executionStarted("A"), executionStarted("")}, "", true}, + {"unpinned empty", "", nil, "", false}, + {"unpinned generic", "", []*api.HistoryEvent{generic}, "", false}, + {"unpinned observed", "", []*api.HistoryEvent{generic, executionStarted("A")}, "A", false}, + } { + t.Run(test.name, func(t *testing.T) { + result, err := collectEvents(api.HistoryQuery{ExecutionID: test.requested}, test.events...) + if test.wantError { + require.ErrorContains(t, err, "execution") + require.Nil(t, result) + } else { + require.NoError(t, err) + require.Equal(t, test.observed, result.ExecutionID) + require.Equal(t, test.events, result.Events) + } + }) + } +} + +func TestCollectRejectsNilEvents(t *testing.T) { + result, err := collectEvents(api.HistoryQuery{}, nil) + require.ErrorContains(t, err, "nil") + require.Nil(t, result) +} diff --git a/internal/historyconv/size.go b/internal/historyconv/size.go new file mode 100644 index 00000000..ed0c60b3 --- /dev/null +++ b/internal/historyconv/size.go @@ -0,0 +1,221 @@ +package historyconv + +import ( + "reflect" + + "github.com/microsoft/durabletask-go/api" +) + +// Saturating above every valid query budget also works on 32-bit platforms. +const maxEstimatedSize = api.MaxHistoryMaxBytes + 1 +const maxPropertyDepth = 64 + +func addSize(size int, amounts ...int) int { + if size >= maxEstimatedSize { + return maxEstimatedSize + } + for _, amount := range amounts { + if amount >= maxEstimatedSize-size { + return maxEstimatedSize + } + size += amount + } + return size +} + +func multiplySize(count, size int) int { + if size == 0 { + return 0 + } + if count > maxEstimatedSize/size { + return maxEstimatedSize + } + return count * size +} + +// ApproximateEventSize estimates retained history content without serializing +// it. Shared converters are excluded; event/detail structs use fixed allowances. +// Cyclic or excessively nested values fail closed above every valid byte limit. +func ApproximateEventSize(event *api.HistoryEvent) int { + if event == nil { + return 0 + } + size := addSize(512, len(event.Type), len(event.UnknownType)) + if value := event.ExecutionStarted; value != nil { + size = addSize(size, len(value.Name), len(value.Version), len(value.InstanceID), + len(value.ExecutionID), len(value.SerializedInput), len(value.OrchestrationSpanID), + mapSize(value.Tags), mapSize(value.ContextFields), parentSize(value.Parent), traceSize(value.ParentTraceContext)) + } + if value := event.ExecutionCompleted; value != nil { + size = addSize(size, len(value.SerializedResult), failureSize(value.FailureDetails)) + } + if value := event.ExecutionTerminated; value != nil { + size = addSize(size, len(value.SerializedInput)) + } + if value := event.TaskScheduled; value != nil { + size = addSize(size, len(value.Name), len(value.Version), len(value.SerializedInput), + mapSize(value.Tags), mapSize(value.ContextFields), traceSize(value.ParentTraceContext)) + } + if value := event.TaskCompleted; value != nil { + size = addSize(size, len(value.SerializedResult)) + } + if value := event.TaskFailed; value != nil { + size = addSize(size, failureSize(value.FailureDetails)) + } + if value := event.SubOrchestrationInstanceCreated; value != nil { + size = addSize(size, len(value.InstanceID), len(value.Name), len(value.Version), len(value.SerializedInput), + mapSize(value.Tags), mapSize(value.ContextFields), traceSize(value.ParentTraceContext)) + } + if value := event.SubOrchestrationInstanceCompleted; value != nil { + size = addSize(size, len(value.SerializedResult)) + } + if value := event.SubOrchestrationInstanceFailed; value != nil { + size = addSize(size, failureSize(value.FailureDetails)) + } + if value := event.EventSent; value != nil { + size = addSize(size, len(value.InstanceID), len(value.Name), len(value.SerializedInput)) + } + if value := event.EventRaised; value != nil { + size = addSize(size, len(value.InstanceID), len(value.Name), len(value.SerializedInput)) + } + if value := event.Generic; value != nil { + size = addSize(size, len(value.SerializedInput)) + } + if value := event.HistoryState; value != nil { + size = addSize(size, metadataSize(value.State)) + } + if value := event.ContinueAsNew; value != nil { + size = addSize(size, len(value.SerializedInput)) + } + if value := event.ExecutionSuspended; value != nil { + size = addSize(size, len(value.SerializedInput)) + } + if value := event.ExecutionResumed; value != nil { + size = addSize(size, len(value.SerializedInput)) + } + if value := event.Entity; value != nil { + size = addSize(size, len(value.RequestID), len(value.Operation), len(value.TargetInstanceID), + len(value.ParentInstanceID), len(value.ParentExecutionID), len(value.CriticalSectionID), + len(value.SerializedInput), len(value.SerializedOutput), failureSize(value.FailureDetails), + propertySize(reflect.ValueOf(value.LockSet), 0)) + } + if value := event.ExecutionRewound; value != nil { + size = addSize(size, len(value.Reason), len(value.Name), len(value.Version), len(value.InstanceID), + len(value.ParentExecutionID), len(value.SerializedInput), mapSize(value.Tags), mapSize(value.ContextFields), + parentSize(value.Parent), traceSize(value.ParentTraceContext)) + } + return size +} + +func mapSize(values map[string]string) int { + if values == nil { + return 0 + } + size := 64 + for key, value := range values { + size = addSize(size, 32, len(key), len(value)) + if size == maxEstimatedSize { + break + } + } + return size +} + +func parentSize(value *api.HistoryParentInstanceInfo) int { + if value == nil { + return 0 + } + return addSize(96, len(value.Name), len(value.Version), len(value.InstanceID), len(value.ExecutionID)) +} + +func traceSize(value *api.HistoryTraceContext) int { + if value == nil { + return 0 + } + return addSize(64, len(value.TraceParent), len(value.TraceState), len(value.SpanID)) +} + +func metadataSize(value *api.OrchestrationMetadata) int { + if value == nil { + return 0 + } + return addSize(384, len(value.InstanceID), len(value.Name), len(value.Version), len(value.ExecutionID), + len(value.ParentInstanceID), len(value.SerializedInput), len(value.SerializedOutput), + len(value.SerializedCustomStatus), mapSize(value.Tags), failureSize(value.FailureDetails)) +} + +func failureSize(value *api.FailureDetails) int { + size := 0 + for depth := 0; value != nil; depth, value = depth+1, value.InnerFailure { + if depth >= api.MaxFailureDetailsDepth { + return maxEstimatedSize + } + size = addSize(size, 96, len(value.ErrorType), len(value.ErrorMessage), len(value.StackTrace), + propertySize(reflect.ValueOf(value.Properties), 0)) + if size == maxEstimatedSize { + return size + } + } + return size +} + +// The depth limit rejects cycles without allocating a visited set. Reflection +// also covers typed maps, slices, and structs supplied directly through the API. +func propertySize(value reflect.Value, depth int) int { + if !value.IsValid() { + return 0 + } + if depth >= maxPropertyDepth { + return maxEstimatedSize + } + size := 0 + switch value.Kind() { + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: + return int(value.Type().Size()) + case reflect.String: + return addSize(16, value.Len()) + case reflect.Interface, reflect.Pointer: + size = int(value.Type().Size()) + if !value.IsNil() { + size = addSize(size, propertySize(value.Elem(), depth+1)) + } + case reflect.Map: + if value.IsNil() { + return 0 + } + size = 64 + iterator := value.MapRange() + for size < maxEstimatedSize && iterator.Next() { + size = addSize(size, 32, propertySize(iterator.Key(), depth+1), propertySize(iterator.Value(), depth+1)) + } + case reflect.Slice, reflect.Array: + if value.Kind() == reflect.Slice { + if value.IsNil() { + return 0 + } + size = 24 + // The backing array, including elements beyond len, remains retained. + value = value.Slice(0, value.Cap()) + } + switch value.Type().Elem().Kind() { + case reflect.Bool, reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, + reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr, + reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128: + return addSize(size, multiplySize(value.Len(), int(value.Type().Elem().Size()))) + } + for i := 0; i < value.Len() && size < maxEstimatedSize; i++ { + size = addSize(size, propertySize(value.Index(i), depth+1)) + } + case reflect.Struct: + size = 32 + for i := 0; i < value.NumField() && size < maxEstimatedSize; i++ { + size = addSize(size, propertySize(value.Field(i), depth+1)) + } + default: + // Closures, channels, and unsafe pointers cannot be safely inspected. + return maxEstimatedSize + } + return size +} diff --git a/internal/largepayload/bounded.go b/internal/largepayload/bounded.go new file mode 100644 index 00000000..b95b9a8e --- /dev/null +++ b/internal/largepayload/bounded.go @@ -0,0 +1,183 @@ +package largepayload + +import ( + "context" + "sync" + + "github.com/microsoft/durabletask-go/api" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// maxConcurrentPayloadOperations caps the simultaneous payload store requests +// issued for a single message, matching the .NET Azure Blob interceptor. It +// overlaps independent payload I/O without emitting an unbounded burst. +const maxConcurrentPayloadOperations = 8 + +// runBounded runs operations concurrently, capped at +// maxConcurrentPayloadOperations, while preserving everything a caller could +// observe from a sequential loop over the same operations: +// +// - An empty operation list does no work and never reports cancellation. +// - The failure of the earliest operation in message order is returned even +// when a later operation fails first or cancellation follows. +// - No operation is dispatched after a failure is recorded. An operation +// already dispatched may begin afterwards, but every dispatched operation +// finishes before runBounded returns, so no operation mutates its target +// after the caller regains control. +// - Cancellation is reported only when it actually stopped an operation from +// starting, so already-completed work is never turned into an error. +func runBounded(ctx context.Context, operations []func(context.Context) error) error { + return runBoundedWithHook(ctx, operations, nil) +} + +// runBoundedWithHook is runBounded with an ordering point for tests. +// failureRecorded, which is nil in production, runs after an operation's +// failure has been recorded and before its concurrency slot is released. +func runBoundedWithHook( + ctx context.Context, + operations []func(context.Context) error, + failureRecorded func(), +) error { + if len(operations) == 0 { + return nil + } + // A single operation cannot overlap with anything, so it runs inline. The + // explicit ctx.Err check keeps the cancellation contract identical to the + // concurrent path, which reports cancellation before dispatching. + if len(operations) == 1 { + if err := ctx.Err(); err != nil { + return err + } + return operations[0](ctx) + } + + limit := min(len(operations), maxConcurrentPayloadOperations) + var ( + semaphore = make(chan struct{}, limit) + workers sync.WaitGroup + mu sync.Mutex + failure error + failedOrdinal = len(operations) + cancelStopped bool + ) + + // canDispatch reports whether the operation may start. It is the dispatch + // linearization point: a failure recorded afterwards never revokes a claim, + // and no operation can claim dispatch once a failure has been recorded. + canDispatch := func() bool { + mu.Lock() + defer mu.Unlock() + if failure != nil { + return false + } + if ctx.Err() != nil { + cancelStopped = true + return false + } + return true + } + + for ordinal, operation := range operations { + if !canDispatch() { + break + } + // Acquiring a slot is deliberately not cancellable: every dispatched + // operation observes ctx itself, so a slot always frees up, and an + // unwind here would abandon operations that are already tracked. + semaphore <- struct{}{} + if !canDispatch() { + <-semaphore + break + } + workers.Add(1) + go func() { + defer workers.Done() + defer func() { <-semaphore }() + err := operation(ctx) + if err == nil { + return + } + mu.Lock() + if ordinal < failedOrdinal { + failedOrdinal, failure = ordinal, err + } + mu.Unlock() + if failureRecorded != nil { + failureRecorded() + } + }() + } + workers.Wait() + + mu.Lock() + err, cancelled := failure, cancelStopped + mu.Unlock() + if err != nil { + return err + } + if cancelled { + return ctx.Err() + } + return nil +} + +type payloadTransform func( + context.Context, + *api.LargePayloadOptions, + *wrapperspb.StringValue, +) (*wrapperspb.StringValue, error) + +// transformPlan collects the payload fields of one message so they can be +// transformed with bounded concurrency in a stable, message-defined order. +type transformPlan struct { + options *api.LargePayloadOptions + transform payloadTransform + operations []func(context.Context) error + // queued records the fields already planned. A message may reach the same + // field twice, for example when one *protos.HistoryEvent appears in both + // the past and new event lists of an orchestrator request. Transforming + // such a field twice would externalize an already externalized token, so + // each distinct field is planned exactly once. + queued map[**wrapperspb.StringValue]struct{} +} + +func newTransformPlan(options *api.LargePayloadOptions, transform payloadTransform) *transformPlan { + return &transformPlan{options: options, transform: transform} +} + +// add queues target for transformation. Fields without a payload contribute no +// operation, so a message with nothing to transform stays zero-work. The field +// value is snapshotted now rather than read when the operation runs, so a +// concurrently transformed sibling field can never be observed mid-update. +func (p *transformPlan) add(target **wrapperspb.StringValue) { + if target == nil || *target == nil { + return + } + // With no options, ordinary payloads are already final. Reserved references + // must still run through the transform so missing configuration is rejected. + if p.options == nil && !api.IsLargePayloadReference((*target).GetValue()) { + return + } + if _, duplicate := p.queued[target]; duplicate { + return + } + if p.queued == nil { + p.queued = make(map[**wrapperspb.StringValue]struct{}) + } + p.queued[target] = struct{}{} + transform, options, value := p.transform, p.options, *target + p.operations = append(p.operations, func(ctx context.Context) error { + transformed, err := transform(ctx, options, value) + if err != nil { + return err + } + // Distinct fields are written by distinct operations, so concurrent + // writes never target the same memory. + *target = transformed + return nil + }) +} + +func (p *transformPlan) run(ctx context.Context) error { + return runBounded(ctx, p.operations) +} diff --git a/internal/largepayload/bounded_test.go b/internal/largepayload/bounded_test.go new file mode 100644 index 00000000..f25ff76a --- /dev/null +++ b/internal/largepayload/bounded_test.go @@ -0,0 +1,486 @@ +package largepayload + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +var errOperation = errors.New("operation failed") + +func cancelledContext() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return ctx +} + +func memoryStoreOptions(store *payload.MemoryStore) *api.LargePayloadOptions { + return &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } +} + +func externalizedValue( + t *testing.T, + options *api.LargePayloadOptions, + value string, +) *wrapperspb.StringValue { + t.Helper() + externalized, err := Externalize(context.Background(), options, wrapperspb.String(value)) + require.NoError(t, err) + return externalized +} + +func TestRunBoundedDispatchAndCancellation(t *testing.T) { + tests := []struct { + name string + ctx context.Context + results []error + wantErr error + wantStarted int64 + }{ + { + name: "no operations never reports cancellation", + ctx: cancelledContext(), + }, + { + name: "single operation runs", + ctx: context.Background(), + results: []error{nil}, + wantStarted: 1, + }, + { + name: "single operation is skipped when already cancelled", + ctx: cancelledContext(), + results: []error{nil}, + wantErr: context.Canceled, + wantStarted: 0, + }, + { + name: "many operations are skipped when already cancelled", + ctx: cancelledContext(), + results: []error{nil, nil, nil, nil}, + wantErr: context.Canceled, + wantStarted: 0, + }, + { + name: "many operations all run", + ctx: context.Background(), + results: []error{nil, nil, nil, nil}, + wantStarted: 4, + }, + { + name: "operation failure is returned", + ctx: context.Background(), + results: []error{nil, errOperation}, + wantErr: errOperation, + wantStarted: 2, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var started atomic.Int64 + operations := make([]func(context.Context) error, 0, len(test.results)) + for _, result := range test.results { + operations = append(operations, func(context.Context) error { + started.Add(1) + return result + }) + } + err := runBounded(test.ctx, operations) + if test.wantErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, test.wantErr) + } + require.Equal(t, test.wantStarted, started.Load()) + }) + } +} + +// TestRunBoundedReturnsLowestOrdinalFailure pins message order as the tie +// breaker rather than whichever concurrent operation happens to fail first. +func TestRunBoundedReturnsLowestOrdinalFailure(t *testing.T) { + errEarly := errors.New("early field") + errLate := errors.New("late field") + lateFailed := make(chan struct{}) + + operations := []func(context.Context) error{ + func(context.Context) error { return nil }, + func(context.Context) error { + <-lateFailed + return errEarly + }, + func(context.Context) error { return nil }, + func(context.Context) error { + close(lateFailed) + return errLate + }, + } + + require.ErrorIs(t, runBounded(context.Background(), operations), errEarly) +} + +// TestRunBoundedStopsDispatchAndDrainsClaimedOperations verifies that a failure +// prevents any further operation from starting while every operation that did +// start runs to completion before runBounded returns. +func TestRunBoundedStopsDispatchAndDrainsClaimedOperations(t *testing.T) { + const operationCount = 4 * maxConcurrentPayloadOperations + var ( + started atomic.Int64 + finished atomic.Int64 + ) + release := make(chan struct{}) + failureRecorded := make(chan struct{}) + + operations := make([]func(context.Context) error, 0, operationCount) + // The first slots stay occupied so the dispatch loop can only advance past + // the concurrency bound once the failing operation frees its slot, which it + // does strictly after recording the failure. + for i := 0; i < maxConcurrentPayloadOperations-1; i++ { + operations = append(operations, func(context.Context) error { + started.Add(1) + <-release + finished.Add(1) + return nil + }) + } + operations = append(operations, func(context.Context) error { + started.Add(1) + finished.Add(1) + return errOperation + }) + for i := maxConcurrentPayloadOperations; i < operationCount; i++ { + operations = append(operations, func(context.Context) error { + started.Add(1) + finished.Add(1) + return nil + }) + } + + result := make(chan error, 1) + go func() { + result <- runBoundedWithHook(context.Background(), operations, func() { close(failureRecorded) }) + }() + + <-failureRecorded + close(release) + require.ErrorIs(t, <-result, errOperation) + require.Equal(t, int64(maxConcurrentPayloadOperations), started.Load(), + "no operation may start after a failure is recorded") + require.Equal(t, started.Load(), finished.Load(), "claimed operations must drain") +} + +// TestRunBoundedFailureWinsOverCancellation pins failure precedence: a real +// error is never masked by cancellation that arrives alongside it. +func TestRunBoundedFailureWinsOverCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + operations := []func(context.Context) error{ + func(context.Context) error { + cancel() + return errOperation + }, + func(context.Context) error { return nil }, + func(context.Context) error { return nil }, + } + + err := runBounded(ctx, operations) + require.ErrorIs(t, err, errOperation) + require.NotErrorIs(t, err, context.Canceled) +} + +// TestRunBoundedCancellationAfterFullDispatchSucceeds verifies that +// cancellation observed once every operation has started does not turn +// completed work into an error. +func TestRunBoundedCancellationAfterFullDispatchSucceeds(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + operations := []func(context.Context) error{ + func(context.Context) error { return nil }, + func(context.Context) error { + // The final operation only runs once dispatch is complete. + cancel() + return nil + }, + } + + require.NoError(t, runBounded(ctx, operations)) +} + +// TestTransformPlanUsesFieldSnapshot verifies that a queued field is +// transformed from the value observed when the plan was built. +func TestTransformPlanUsesFieldSnapshot(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + + target := wrapperspb.String("planned") + plan := newTransformPlan(options, Externalize) + plan.add(&target) + target = wrapperspb.String("replaced after planning") + + require.NoError(t, plan.run(context.Background())) + hydrated, err := Hydrate(context.Background(), options, target) + require.NoError(t, err) + require.Equal(t, "planned", hydrated.GetValue()) +} + +func TestTransformPlanSkipsAbsentFields(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + state := &protos.OrchestrationState{} + + // A message with no payloads is zero-work, so a cancelled context has + // nothing to cancel. + require.NoError(t, TransformOrchestrationState(cancelledContext(), options, state)) + require.NoError(t, TransformOrchestrationState(cancelledContext(), options, nil)) + require.NoError(t, TransformOrchestratorResponse(cancelledContext(), options, &protos.OrchestratorResponse{})) + require.NoError(t, TransformEntityBatchRequest(cancelledContext(), options, &protos.EntityBatchRequest{})) +} + +// TestTransformResolvesFieldsConcurrently deadlocks unless every payload field +// of a message is resolved concurrently under the shared bound. +func TestTransformResolvesFieldsConcurrently(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + + values := []string{"state", "first", "second", "third"} + request := &protos.EntityBatchRequest{Operations: make([]*protos.OperationRequest, 0, len(values)-1)} + for i, value := range values { + externalized := externalizedValue(t, options, value) + if i == 0 { + request.EntityState = externalized + continue + } + request.Operations = append(request.Operations, &protos.OperationRequest{Input: externalized}) + } + + var barrier sync.WaitGroup + barrier.Add(len(values)) + options.Resolver = barrierResolver{store: store, barrier: &barrier} + + require.NoError(t, TransformEntityBatchRequest(context.Background(), options, request)) + require.Equal(t, "state", request.EntityState.GetValue()) + for i, operation := range request.Operations { + require.Equal(t, values[i+1], operation.Input.GetValue()) + } +} + +// TestTransformPlansFieldsInMessageOrder pins the ordinals a message assigns to +// its payload fields. runBounded returns the lowest-ordinal failure, so failing +// a known set of fields makes the plan order observable without timing control. +func TestTransformPlansFieldsInMessageOrder(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + + externalized := make([]*wrapperspb.StringValue, 0, 3) + locations := make([]string, 0, 3) + for _, value := range []string{"state", "first", "second"} { + external := externalizedValue(t, options, value) + externalized = append(externalized, external) + locations = append(locations, referenceLocation(t, external)) + } + + errState := errors.New("entity state field") + errFirst := errors.New("first operation field") + errSecond := errors.New("second operation field") + + tests := []struct { + name string + failures map[string]error + want error + }{ + { + name: "entity state precedes the operations", + failures: map[string]error{locations[0]: errState, locations[1]: errFirst, locations[2]: errSecond}, + want: errState, + }, + { + name: "operations keep their slice order", + failures: map[string]error{locations[1]: errFirst, locations[2]: errSecond}, + want: errFirst, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + failing := &api.LargePayloadOptions{ + Store: store, + Resolver: failingResolver{store: store, failures: test.failures}, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + err := TransformEntityBatchRequest(context.Background(), failing, &protos.EntityBatchRequest{ + EntityState: externalized[0], + Operations: []*protos.OperationRequest{ + {Input: externalized[1]}, + {Input: externalized[2]}, + }, + }) + require.ErrorIs(t, err, test.want) + }) + } +} + +// TestTransformPlanQueuesEachFieldOnce verifies that a field a message reaches +// twice is transformed exactly once. Without deduplication the field would be +// externalized twice, leaving an orphaned payload behind in the store. +func TestTransformPlanQueuesEachFieldOnce(t *testing.T) { + store := payload.NewMemoryStore() + var stores atomic.Int64 + options := &api.LargePayloadOptions{ + Store: countingStore{store: store, calls: &stores}, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + + // A rewind action can list the same event object more than once, so both + // entries resolve to the same input field. + event := &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{Input: wrapperspb.String("payload")}, + }} + response := &protos.OrchestratorResponse{Actions: []*protos.OrchestratorAction{ + {OrchestratorActionType: &protos.OrchestratorAction_RewindOrchestration{ + RewindOrchestration: &protos.RewindOrchestrationAction{ + NewHistory: []*protos.HistoryEvent{event, event}, + }, + }}, + }} + + require.NoError(t, TransformOrchestratorResponse(context.Background(), options, response)) + require.Equal(t, int64(1), stores.Load(), "a field reached twice must be externalized once") + + hydrated, err := Hydrate(context.Background(), options, event.GetExecutionStarted().Input) + require.NoError(t, err) + require.Equal(t, "payload", hydrated.GetValue()) +} + +// TestTransformMessageShapesResolveEveryPayloadField walks the multi-field +// messages that use bounded concurrency and verifies every payload round-trips. +func TestTransformMessageShapesResolveEveryPayloadField(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + external := func(value string) *wrapperspb.StringValue { + externalized := externalizedValue(t, options, value) + require.NotEqual(t, value, externalized.GetValue()) + return externalized + } + startedEvent := func(input string) *protos.HistoryEvent { + return &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionStarted{ + ExecutionStarted: &protos.ExecutionStartedEvent{Input: external(input)}, + }} + } + + orchestratorRequest := &protos.OrchestratorRequest{ + PastEvents: []*protos.HistoryEvent{startedEvent("past-one"), startedEvent("past-two")}, + NewEvents: []*protos.HistoryEvent{startedEvent("new-one")}, + } + state := &protos.OrchestrationState{ + Input: external("state-input"), + Output: external("state-output"), + CustomStatus: external("state-status"), + } + entityRequest := &protos.EntityBatchRequest{ + EntityState: external("entity-state"), + Operations: []*protos.OperationRequest{ + {Input: external("entity-one")}, + {Input: external("entity-two")}, + }, + } + + require.NoError(t, TransformOrchestratorRequest(context.Background(), options, orchestratorRequest)) + require.NoError(t, TransformOrchestrationState(context.Background(), options, state)) + require.NoError(t, TransformEntityBatchRequest(context.Background(), options, entityRequest)) + + require.Equal(t, "past-one", orchestratorRequest.PastEvents[0].GetExecutionStarted().Input.GetValue()) + require.Equal(t, "past-two", orchestratorRequest.PastEvents[1].GetExecutionStarted().Input.GetValue()) + require.Equal(t, "new-one", orchestratorRequest.NewEvents[0].GetExecutionStarted().Input.GetValue()) + require.Equal(t, "state-input", state.Input.GetValue()) + require.Equal(t, "state-output", state.Output.GetValue()) + require.Equal(t, "state-status", state.CustomStatus.GetValue()) + require.Equal(t, "entity-state", entityRequest.EntityState.GetValue()) + require.Equal(t, "entity-one", entityRequest.Operations[0].Input.GetValue()) + require.Equal(t, "entity-two", entityRequest.Operations[1].Input.GetValue()) +} + +// TestTransformPreCancelledContextPerformsNoStoreCalls verifies that a message +// with payload fields reports cancellation without touching the payload store. +func TestTransformPreCancelledContextPerformsNoStoreCalls(t *testing.T) { + store := payload.NewMemoryStore() + options := memoryStoreOptions(store) + request := &protos.EntityBatchRequest{ + EntityState: externalizedValue(t, options, "state"), + Operations: []*protos.OperationRequest{{Input: externalizedValue(t, options, "input")}}, + } + + var calls atomic.Int64 + options.Resolver = countingResolver{store: store, calls: &calls} + require.ErrorIs(t, TransformEntityBatchRequest(cancelledContext(), options, request), context.Canceled) + require.Zero(t, calls.Load()) +} + +type countingResolver struct { + store *payload.MemoryStore + calls *atomic.Int64 +} + +func (r countingResolver) Resolve(ctx context.Context, location string) ([]byte, error) { + r.calls.Add(1) + return r.store.Resolve(ctx, location) +} + +type countingStore struct { + store *payload.MemoryStore + calls *atomic.Int64 +} + +func (s countingStore) Store(ctx context.Context, value []byte) (string, error) { + s.calls.Add(1) + return s.store.Store(ctx, value) +} + +func referenceLocation(t *testing.T, value *wrapperspb.StringValue) string { + t.Helper() + ref, ok, err := parseReference(value.GetValue(), api.DefaultLargePayloadMaxBytes) + require.NoError(t, err) + require.True(t, ok) + return ref.Location +} + +type barrierResolver struct { + store *payload.MemoryStore + barrier *sync.WaitGroup +} + +func (r barrierResolver) Resolve(ctx context.Context, location string) ([]byte, error) { + r.barrier.Done() + r.barrier.Wait() + return r.store.Resolve(ctx, location) +} + +// failingResolver fails the configured locations and resolves everything else, +// which makes the ordinal a message assigned to a field observable. +type failingResolver struct { + store *payload.MemoryStore + failures map[string]error +} + +func (r failingResolver) Resolve(ctx context.Context, location string) ([]byte, error) { + if failure, ok := r.failures[location]; ok { + return nil, failure + } + return r.store.Resolve(ctx, location) +} diff --git a/internal/largepayload/codec.go b/internal/largepayload/codec.go new file mode 100644 index 00000000..e401193c --- /dev/null +++ b/internal/largepayload/codec.go @@ -0,0 +1,407 @@ +package largepayload + +import ( + "context" + "crypto/sha256" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const referencePrefix = api.DurableTaskPayloadReferencePrefix + +type reference struct { + Location string `json:"location"` + Size int `json:"size"` + SHA256 string `json:"sha256"` +} + +func Externalize(ctx context.Context, options *api.LargePayloadOptions, value *wrapperspb.StringValue) (*wrapperspb.StringValue, error) { + if value == nil { + return value, nil + } + if options == nil { + if isRecognizedReference(value.GetValue()) { + return nil, fmt.Errorf("%w: large payload reference requires a configured store and resolver", api.ErrFeatureNotSupported) + } + return value, nil + } + normalized, err := api.NormalizeLargePayloadOptions(options) + if err != nil { + return nil, err + } + isToken, err := isNativePayloadToken(normalized.TokenStore, value.GetValue()) + if err != nil { + return nil, err + } + if isToken { + return value, nil + } + if _, ok, err := parseReference(value.GetValue(), normalized.MaxPayloadBytes); ok || err != nil { + return value, err + } + payload := []byte(value.GetValue()) + if len(payload) > normalized.MaxPayloadBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), normalized.MaxPayloadBytes) + } + if !exceedsThreshold(normalized, len(payload)) { + return value, nil + } + if normalized.TokenStore != nil { + token, err := normalized.TokenStore.StoreToken(ctx, append([]byte(nil), payload...)) + if err != nil { + return nil, fmt.Errorf("failed to store large payload: %w", err) + } + if strings.TrimSpace(token) == "" { + return nil, errors.New("large payload store returned an empty token") + } + return wrapperspb.String(token), nil + } + location, err := normalized.Store.Store(ctx, append([]byte(nil), payload...)) + if err != nil { + return nil, fmt.Errorf("failed to store large payload: %w", err) + } + if strings.TrimSpace(location) == "" { + return nil, errors.New("large payload store returned an empty location") + } + digest := sha256.Sum256(payload) + descriptor, err := json.Marshal(reference{ + Location: location, + Size: len(payload), + SHA256: hex.EncodeToString(digest[:]), + }) + if err != nil { + return nil, fmt.Errorf("failed to encode large payload reference: %w", err) + } + return wrapperspb.String(referencePrefix + base64.RawURLEncoding.EncodeToString(descriptor)), nil +} + +func Hydrate(ctx context.Context, options *api.LargePayloadOptions, value *wrapperspb.StringValue) (*wrapperspb.StringValue, error) { + if value == nil { + return value, nil + } + if options == nil { + if isRecognizedReference(value.GetValue()) { + return nil, fmt.Errorf("%w: large payload reference requires a configured resolver", api.ErrFeatureNotSupported) + } + return value, nil + } + normalized, err := api.NormalizeLargePayloadOptions(options) + if err != nil { + return nil, err + } + isToken, err := isNativePayloadToken(normalized.TokenStore, value.GetValue()) + if err != nil { + return nil, err + } + if isToken { + payload, err := normalized.TokenStore.ResolveToken(ctx, value.GetValue()) + if err != nil { + return nil, fmt.Errorf("failed to resolve large payload: %w", err) + } + if len(payload) > normalized.MaxPayloadBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), normalized.MaxPayloadBytes) + } + return wrapperspb.String(string(payload)), nil + } + ref, ok, err := parseReference(value.GetValue(), normalized.MaxPayloadBytes) + if err != nil || !ok { + return value, err + } + payload, err := normalized.Resolver.Resolve(ctx, ref.Location) + if err != nil { + return nil, fmt.Errorf("failed to resolve large payload: %w", err) + } + if len(payload) != ref.Size { + return nil, fmt.Errorf("%w: expected %d bytes, got %d", api.ErrLargePayloadIntegrity, ref.Size, len(payload)) + } + digest := sha256.Sum256(payload) + expectedDigest, err := hex.DecodeString(ref.SHA256) + if err != nil || len(expectedDigest) != sha256.Size { + return nil, fmt.Errorf("%w: invalid SHA-256 digest", api.ErrLargePayloadReference) + } + if subtle.ConstantTimeCompare(digest[:], expectedDigest) != 1 { + return nil, fmt.Errorf("%w: SHA-256 mismatch", api.ErrLargePayloadIntegrity) + } + return wrapperspb.String(string(payload)), nil +} + +func TransformHistoryEvent( + ctx context.Context, + options *api.LargePayloadOptions, + event *protos.HistoryEvent, + externalize bool, +) error { + transform := Hydrate + if externalize { + transform = Externalize + } + plan := newTransformPlan(options, transform) + planHistoryEvent(plan, event) + return plan.run(ctx) +} + +func planHistoryEvent(plan *transformPlan, event *protos.HistoryEvent) { + if event == nil { + return + } + var target **wrapperspb.StringValue + switch { + case event.GetExecutionStarted() != nil: + target = &event.GetExecutionStarted().Input + case event.GetExecutionCompleted() != nil: + target = &event.GetExecutionCompleted().Result + case event.GetExecutionTerminated() != nil: + target = &event.GetExecutionTerminated().Input + case event.GetTaskScheduled() != nil: + target = &event.GetTaskScheduled().Input + case event.GetTaskCompleted() != nil: + target = &event.GetTaskCompleted().Result + case event.GetSubOrchestrationInstanceCreated() != nil: + target = &event.GetSubOrchestrationInstanceCreated().Input + case event.GetSubOrchestrationInstanceCompleted() != nil: + target = &event.GetSubOrchestrationInstanceCompleted().Result + case event.GetEventSent() != nil: + target = &event.GetEventSent().Input + case event.GetEventRaised() != nil: + target = &event.GetEventRaised().Input + case event.GetGenericEvent() != nil: + target = &event.GetGenericEvent().Data + case event.GetHistoryState() != nil: + planOrchestrationState(plan, event.GetHistoryState().OrchestrationState) + return + case event.GetContinueAsNew() != nil: + target = &event.GetContinueAsNew().Input + case event.GetExecutionSuspended() != nil: + target = &event.GetExecutionSuspended().Input + case event.GetExecutionResumed() != nil: + target = &event.GetExecutionResumed().Input + case event.GetExecutionRewound() != nil: + target = &event.GetExecutionRewound().Input + case event.GetEntityOperationSignaled() != nil: + target = &event.GetEntityOperationSignaled().Input + case event.GetEntityOperationCalled() != nil: + target = &event.GetEntityOperationCalled().Input + case event.GetEntityOperationCompleted() != nil: + target = &event.GetEntityOperationCompleted().Output + default: + return + } + plan.add(target) +} + +func planOrchestrationState(plan *transformPlan, state *protos.OrchestrationState) { + if state == nil { + return + } + plan.add(&state.Input) + plan.add(&state.Output) + plan.add(&state.CustomStatus) +} + +func TransformOrchestratorRequest( + ctx context.Context, + options *api.LargePayloadOptions, + request *protos.OrchestratorRequest, +) error { + if request == nil { + return nil + } + plan := newTransformPlan(options, Hydrate) + for _, events := range [][]*protos.HistoryEvent{request.PastEvents, request.NewEvents} { + for _, event := range events { + planHistoryEvent(plan, event) + } + } + return plan.run(ctx) +} + +func TransformOrchestratorResponse( + ctx context.Context, + options *api.LargePayloadOptions, + response *protos.OrchestratorResponse, +) error { + if response == nil { + return nil + } + plan := newTransformPlan(options, Externalize) + plan.add(&response.CustomStatus) + for _, action := range response.Actions { + if action == nil { + continue + } + switch { + case action.GetScheduleTask() != nil: + plan.add(&action.GetScheduleTask().Input) + case action.GetCreateSubOrchestration() != nil: + plan.add(&action.GetCreateSubOrchestration().Input) + case action.GetSendEvent() != nil: + plan.add(&action.GetSendEvent().Data) + case action.GetCompleteOrchestration() != nil: + plan.add(&action.GetCompleteOrchestration().Result) + case action.GetTerminateOrchestration() != nil: + plan.add(&action.GetTerminateOrchestration().Reason) + case action.GetSendEntityMessage() != nil: + message := action.GetSendEntityMessage() + switch { + case message.GetEntityOperationSignaled() != nil: + plan.add(&message.GetEntityOperationSignaled().Input) + case message.GetEntityOperationCalled() != nil: + plan.add(&message.GetEntityOperationCalled().Input) + } + case action.GetRewindOrchestration() != nil: + for _, event := range action.GetRewindOrchestration().NewHistory { + planHistoryEvent(plan, event) + } + } + } + return plan.run(ctx) +} + +func TransformActivityRequest(ctx context.Context, options *api.LargePayloadOptions, request *protos.ActivityRequest) error { + if request == nil { + return nil + } + var err error + request.Input, err = Hydrate(ctx, options, request.Input) + return err +} + +func TransformActivityResponse(ctx context.Context, options *api.LargePayloadOptions, response *protos.ActivityResponse) error { + if response == nil { + return nil + } + var err error + response.Result, err = Externalize(ctx, options, response.Result) + return err +} + +func TransformEntityBatchRequest( + ctx context.Context, + options *api.LargePayloadOptions, + request *protos.EntityBatchRequest, +) error { + if request == nil { + return nil + } + plan := newTransformPlan(options, Hydrate) + plan.add(&request.EntityState) + for _, operation := range request.Operations { + if operation == nil { + continue + } + plan.add(&operation.Input) + } + return plan.run(ctx) +} + +func TransformEntityBatchResult( + ctx context.Context, + options *api.LargePayloadOptions, + result *protos.EntityBatchResult, +) error { + if result == nil { + return nil + } + plan := newTransformPlan(options, Externalize) + plan.add(&result.EntityState) + for _, operationResult := range result.Results { + if operationResult == nil || operationResult.GetSuccess() == nil { + continue + } + plan.add(&operationResult.GetSuccess().Result) + } + for _, action := range result.Actions { + if action == nil { + continue + } + switch { + case action.GetSendSignal() != nil: + plan.add(&action.GetSendSignal().Input) + case action.GetStartNewOrchestration() != nil: + plan.add(&action.GetStartNewOrchestration().Input) + } + } + return plan.run(ctx) +} + +func TransformOrchestrationState(ctx context.Context, options *api.LargePayloadOptions, state *protos.OrchestrationState) error { + plan := newTransformPlan(options, Hydrate) + planOrchestrationState(plan, state) + return plan.run(ctx) +} + +func parseReference(value string, maxPayloadBytes int) (reference, bool, error) { + if !strings.HasPrefix(value, referencePrefix) { + return reference{}, false, nil + } + encoded := strings.TrimPrefix(value, referencePrefix) + descriptor, err := base64.RawURLEncoding.DecodeString(encoded) + if err != nil { + return reference{}, true, fmt.Errorf("%w: invalid base64 descriptor", api.ErrLargePayloadReference) + } + var ref reference + if err := json.Unmarshal(descriptor, &ref); err != nil { + return reference{}, true, fmt.Errorf("%w: invalid JSON descriptor", api.ErrLargePayloadReference) + } + if strings.TrimSpace(ref.Location) == "" || ref.Size < 0 || ref.Size > maxPayloadBytes { + return reference{}, true, fmt.Errorf("%w: invalid location or size", api.ErrLargePayloadReference) + } + if len(ref.SHA256) != sha256.Size*2 { + return reference{}, true, fmt.Errorf("%w: invalid SHA-256 digest", api.ErrLargePayloadReference) + } + return ref, true, nil +} + +func isRecognizedReference(value string) bool { + return api.IsLargePayloadReference(value) +} + +func isBlobReference(value string) bool { + return strings.HasPrefix(value, api.AzureBlobPayloadReferencePrefixV1) || + strings.HasPrefix(value, api.AzureBlobPayloadReferencePrefixV2) +} + +// isNativePayloadToken reports whether value is a native token that the +// configured store can handle, validating it when the store supports +// validation. Azure Blob tokens are rejected when no token store can resolve +// them so they are never mistaken for opaque payload data. +func isNativePayloadToken(store api.LargePayloadTokenStore, value string) (bool, error) { + if store == nil || !store.IsLargePayloadToken(value) { + if isBlobReference(value) { + return false, fmt.Errorf( + "%w: Azure Blob payload token requires an Azure Blob token store", + api.ErrFeatureNotSupported, + ) + } + return false, nil + } + if validator, ok := store.(api.LargePayloadTokenValidator); ok { + if err := validator.ValidateLargePayloadToken(value); err != nil { + return false, err + } + } + return true, nil +} + +// exceedsThreshold reports whether a payload is large enough to externalize. +// Stores such as Azure Blob externalize payloads exactly at the threshold, +// while the built-in reference store externalizes only above it. +func exceedsThreshold(options *api.LargePayloadOptions, size int) bool { + if size > options.ThresholdBytes { + return true + } + if size < options.ThresholdBytes { + return false + } + policy, ok := options.Store.(api.InclusiveLargePayloadThreshold) + return ok && policy.UsesInclusiveLargePayloadThreshold() +} diff --git a/internal/largepayload/codec_test.go b/internal/largepayload/codec_test.go new file mode 100644 index 00000000..ae0c143b --- /dev/null +++ b/internal/largepayload/codec_test.go @@ -0,0 +1,465 @@ +package largepayload + +import ( + "context" + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestExternalizeAndHydrate(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 4, + MaxPayloadBytes: 1024, + } + externalized, err := Externalize(context.Background(), options, wrapperspb.String("large payload")) + require.NoError(t, err) + require.NotEqual(t, "large payload", externalized.GetValue()) + + hydrated, err := Hydrate(context.Background(), options, externalized) + require.NoError(t, err) + require.Equal(t, "large payload", hydrated.GetValue()) +} + +func TestLargePayloadIntegrityFailure(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + externalized, err := Externalize(context.Background(), options, wrapperspb.String("payload")) + require.NoError(t, err) + + options.Resolver = staticResolver{payload: []byte("tampered")} + _, err = Hydrate(context.Background(), options, externalized) + require.ErrorIs(t, err, api.ErrLargePayloadIntegrity) +} + +func TestLargePayloadReferenceRequiresConfiguration(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + externalized, err := Externalize(context.Background(), options, wrapperspb.String("payload")) + require.NoError(t, err) + _, err = Hydrate(context.Background(), nil, externalized) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) + _, err = Externalize(context.Background(), nil, externalized) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) + err = TransformActivityRequest(context.Background(), nil, &protos.ActivityRequest{Input: externalized}) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) + err = TransformOrchestrationState(context.Background(), nil, &protos.OrchestrationState{Output: externalized}) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) +} + +func TestLargePayloadLimitAndMalformedReference(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 4, + } + + _, err := Externalize(context.Background(), options, wrapperspb.String("oversized")) + require.ErrorIs(t, err, api.ErrLargePayloadTooLarge) + + _, err = Hydrate(context.Background(), options, wrapperspb.String(referencePrefix+"not-base64")) + require.ErrorIs(t, err, api.ErrLargePayloadReference) +} + +func TestNativeLargePayloadTokens(t *testing.T) { + store := nativeStore{} + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 4, + MaxPayloadBytes: 1024, + } + externalized, err := Externalize(context.Background(), options, wrapperspb.String("payload")) + require.NoError(t, err) + require.Equal(t, "blob:v2:https://account.example/payload", externalized.GetValue()) + + hydrated, err := Hydrate(context.Background(), options, externalized) + require.NoError(t, err) + require.Equal(t, "payload", hydrated.GetValue()) + _, err = Hydrate(context.Background(), options, wrapperspb.String("blob:v2:malformed")) + require.ErrorIs(t, err, api.ErrLargePayloadReference) +} + +func TestNativeLargePayloadThresholdIsInclusive(t *testing.T) { + store := inclusiveNativeStore{} + externalized, err := Externalize(context.Background(), &api.LargePayloadOptions{ + Store: store, Resolver: store, ThresholdBytes: len("payload"), MaxPayloadBytes: 1024, + }, wrapperspb.String("payload")) + require.NoError(t, err) + require.Equal(t, "blob:v2:https://account.example/payload", externalized.GetValue()) +} + +func TestTransformOrchestratorResponsePayloadFields(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + response := &protos.OrchestratorResponse{ + CustomStatus: wrapperspb.String("status"), + Actions: []*protos.OrchestratorAction{ + { + OrchestratorActionType: &protos.OrchestratorAction_ScheduleTask{ + ScheduleTask: &protos.ScheduleTaskAction{Input: wrapperspb.String("activity")}, + }, + }, + { + OrchestratorActionType: &protos.OrchestratorAction_CreateSubOrchestration{ + CreateSubOrchestration: &protos.CreateSubOrchestrationAction{Input: wrapperspb.String("child")}, + }, + }, + { + OrchestratorActionType: &protos.OrchestratorAction_CompleteOrchestration{ + CompleteOrchestration: &protos.CompleteOrchestrationAction{Result: wrapperspb.String("result")}, + }, + }, + { + OrchestratorActionType: &protos.OrchestratorAction_SendEntityMessage{ + SendEntityMessage: &protos.SendEntityMessageAction{ + EntityMessageType: &protos.SendEntityMessageAction_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{Input: wrapperspb.String("entity")}, + }, + }, + }, + }, + }, + } + require.NoError(t, TransformOrchestratorResponse(context.Background(), options, response)) + + for _, value := range []*wrapperspb.StringValue{ + response.CustomStatus, + response.Actions[0].GetScheduleTask().Input, + response.Actions[1].GetCreateSubOrchestration().Input, + response.Actions[2].GetCompleteOrchestration().Result, + response.Actions[3].GetSendEntityMessage().GetEntityOperationCalled().Input, + } { + hydrated, err := Hydrate(context.Background(), options, value) + require.NoError(t, err) + require.NotEqual(t, value.GetValue(), hydrated.GetValue()) + } +} + +func BenchmarkTransformOrchestratorResponseDisabled(b *testing.B) { + response := &protos.OrchestratorResponse{ + CustomStatus: wrapperspb.String("status"), + Actions: []*protos.OrchestratorAction{ + { + OrchestratorActionType: &protos.OrchestratorAction_ScheduleTask{ + ScheduleTask: &protos.ScheduleTaskAction{Input: wrapperspb.String("activity")}, + }, + }, + { + OrchestratorActionType: &protos.OrchestratorAction_CreateSubOrchestration{ + CreateSubOrchestration: &protos.CreateSubOrchestrationAction{Input: wrapperspb.String("child")}, + }, + }, + { + OrchestratorActionType: &protos.OrchestratorAction_CompleteOrchestration{ + CompleteOrchestration: &protos.CompleteOrchestrationAction{Result: wrapperspb.String("result")}, + }, + }, + }, + } + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + for range b.N { + if err := TransformOrchestratorResponse(ctx, nil, response); err != nil { + b.Fatal(err) + } + } +} + +func TestTransformOrchestratorResponseDisabledFastPath(t *testing.T) { + t.Run("ordinary payload ignores cancellation because no work is required", func(t *testing.T) { + value := wrapperspb.String("ordinary") + response := &protos.OrchestratorResponse{CustomStatus: value} + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + require.NoError(t, TransformOrchestratorResponse(ctx, nil, response)) + require.Same(t, value, response.CustomStatus) + }) + + t.Run("reserved reference still requires configuration", func(t *testing.T) { + response := &protos.OrchestratorResponse{ + CustomStatus: wrapperspb.String(api.AzureBlobPayloadReferencePrefixV2 + "https://account.example/payload"), + } + + err := TransformOrchestratorResponse(context.Background(), nil, response) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) + }) +} + +func TestTransformHistoryEventPayloadFields(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + tests := []struct { + name string + event *protos.HistoryEvent + value func(*protos.HistoryEvent) *wrapperspb.StringValue + }{ + { + name: "event raised", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_EventRaised{ + EventRaised: &protos.EventRaisedEvent{Input: wrapperspb.String("raised")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetEventRaised().Input + }, + }, + { + name: "generic data", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_GenericEvent{ + GenericEvent: &protos.GenericEvent{Data: wrapperspb.String("generic")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetGenericEvent().Data + }, + }, + { + name: "execution suspended", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionSuspended{ + ExecutionSuspended: &protos.ExecutionSuspendedEvent{Input: wrapperspb.String("suspended")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetExecutionSuspended().Input + }, + }, + { + name: "execution resumed", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionResumed{ + ExecutionResumed: &protos.ExecutionResumedEvent{Input: wrapperspb.String("resumed")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetExecutionResumed().Input + }, + }, + { + name: "event sent", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_EventSent{ + EventSent: &protos.EventSentEvent{Input: wrapperspb.String("sent")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetEventSent().Input + }, + }, + { + name: "termination", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionTerminated{ + ExecutionTerminated: &protos.ExecutionTerminatedEvent{Input: wrapperspb.String("terminated")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetExecutionTerminated().Input + }, + }, + { + name: "continue as new", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_ContinueAsNew{ + ContinueAsNew: &protos.ContinueAsNewEvent{Input: wrapperspb.String("continued")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetContinueAsNew().Input + }, + }, + { + name: "rewind", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_ExecutionRewound{ + ExecutionRewound: &protos.ExecutionRewoundEvent{Input: wrapperspb.String("rewound")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetExecutionRewound().Input + }, + }, + { + name: "entity signal", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{Input: wrapperspb.String("signal")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetEntityOperationSignaled().Input + }, + }, + { + name: "entity call", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: &protos.EntityOperationCalledEvent{Input: wrapperspb.String("call")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetEntityOperationCalled().Input + }, + }, + { + name: "entity result", + event: &protos.HistoryEvent{EventType: &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{Output: wrapperspb.String("result")}, + }}, + value: func(event *protos.HistoryEvent) *wrapperspb.StringValue { + return event.GetEntityOperationCompleted().Output + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + original := test.value(test.event).GetValue() + require.NoError(t, TransformHistoryEvent(context.Background(), options, test.event, true)) + require.NotEqual(t, original, test.value(test.event).GetValue()) + require.NoError(t, TransformHistoryEvent(context.Background(), options, test.event, false)) + require.Equal(t, original, test.value(test.event).GetValue()) + }) + } +} + +func TestTransformHistoryStateEventPayloadFields(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + state := &protos.OrchestrationState{ + Input: wrapperspb.String("input"), + Output: wrapperspb.String("output"), + CustomStatus: wrapperspb.String("status"), + } + event := &protos.HistoryEvent{EventType: &protos.HistoryEvent_HistoryState{ + HistoryState: &protos.HistoryStateEvent{OrchestrationState: state}, + }} + require.NoError(t, TransformHistoryEvent(context.Background(), options, event, true)) + require.NoError(t, TransformHistoryEvent(context.Background(), options, event, false)) + require.Equal(t, "input", state.Input.GetValue()) + require.Equal(t, "output", state.Output.GetValue()) + require.Equal(t, "status", state.CustomStatus.GetValue()) +} + +func TestTransformEntityBatchPayloadFields(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 1, + MaxPayloadBytes: 1024, + } + request := &protos.EntityBatchRequest{ + EntityState: wrapperspb.String("state"), + Operations: []*protos.OperationRequest{{ + Input: wrapperspb.String("input"), + }}, + } + var err error + request.EntityState, err = Externalize(context.Background(), options, request.EntityState) + require.NoError(t, err) + request.Operations[0].Input, err = Externalize(context.Background(), options, request.Operations[0].Input) + require.NoError(t, err) + require.ErrorIs(t, TransformEntityBatchRequest(context.Background(), nil, &protos.EntityBatchRequest{ + EntityState: request.EntityState, + Operations: []*protos.OperationRequest{{Input: request.Operations[0].Input}}, + }), api.ErrFeatureNotSupported) + require.ErrorIs(t, TransformEntityBatchResult(context.Background(), nil, &protos.EntityBatchResult{ + EntityState: request.EntityState, + }), api.ErrFeatureNotSupported) + require.NoError(t, TransformEntityBatchRequest(context.Background(), options, request)) + require.Equal(t, "state", request.EntityState.GetValue()) + require.Equal(t, "input", request.Operations[0].Input.GetValue()) + + result := &protos.EntityBatchResult{ + EntityState: wrapperspb.String("next-state"), + Results: []*protos.OperationResult{{ + ResultType: &protos.OperationResult_Success{ + Success: &protos.OperationResultSuccess{Result: wrapperspb.String("output")}, + }, + }}, + Actions: []*protos.OperationAction{ + { + OperationActionType: &protos.OperationAction_SendSignal{ + SendSignal: &protos.SendSignalAction{Input: wrapperspb.String("signal")}, + }, + }, + { + OperationActionType: &protos.OperationAction_StartNewOrchestration{ + StartNewOrchestration: &protos.StartNewOrchestrationAction{Input: wrapperspb.String("start")}, + }, + }, + }, + } + require.NoError(t, TransformEntityBatchResult(context.Background(), options, result)) + for _, value := range []*wrapperspb.StringValue{ + result.EntityState, + result.Results[0].GetSuccess().Result, + result.Actions[0].GetSendSignal().Input, + result.Actions[1].GetStartNewOrchestration().Input, + } { + hydrated, err := Hydrate(context.Background(), options, value) + require.NoError(t, err) + require.NotEqual(t, value.GetValue(), hydrated.GetValue()) + } +} + +type staticResolver struct { + payload []byte + err error +} + +func (r staticResolver) Resolve(context.Context, string) ([]byte, error) { + if r.err != nil { + return nil, r.err + } + return append([]byte(nil), r.payload...), nil +} + +func (staticResolver) Store(context.Context, []byte) (string, error) { + return "", errors.New("not implemented") +} + +type nativeStore struct{} + +func (nativeStore) Store(context.Context, []byte) (string, error) { return "", nil } +func (nativeStore) Resolve(context.Context, string) ([]byte, error) { return []byte("payload"), nil } +func (nativeStore) StoreToken(context.Context, []byte) (string, error) { + return "blob:v2:https://account.example/payload", nil +} +func (nativeStore) ResolveToken(context.Context, string) ([]byte, error) { + return []byte("payload"), nil +} +func (nativeStore) IsLargePayloadToken(value string) bool { + return len(value) >= len("blob:v2:") && value[:len("blob:v2:")] == "blob:v2:" +} +func (nativeStore) ValidateLargePayloadToken(value string) error { + if value != "blob:v2:https://account.example/payload" { + return api.ErrLargePayloadReference + } + return nil +} + +type inclusiveNativeStore struct{ nativeStore } + +func (inclusiveNativeStore) UsesInclusiveLargePayloadThreshold() bool { return true } diff --git a/internal/protos/orchestrator_service.pb.go b/internal/protos/orchestrator_service.pb.go index 52eb6d21..9aca4923 100644 --- a/internal/protos/orchestrator_service.pb.go +++ b/internal/protos/orchestrator_service.pb.go @@ -3,21 +3,23 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.31.0 -// protoc v3.12.4 +// protoc-gen-go v1.36.12 +// protoc v7.36.1 // source: orchestrator_service.proto package protos import ( - duration "google.golang.org/protobuf/types/known/durationpb" - empty "google.golang.org/protobuf/types/known/emptypb" - timestamp "google.golang.org/protobuf/types/known/timestamppb" - wrappers "google.golang.org/protobuf/types/known/wrapperspb" protoreflect "google.golang.org/protobuf/reflect/protoreflect" protoimpl "google.golang.org/protobuf/runtime/protoimpl" + durationpb "google.golang.org/protobuf/types/known/durationpb" + emptypb "google.golang.org/protobuf/types/known/emptypb" + structpb "google.golang.org/protobuf/types/known/structpb" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + wrapperspb "google.golang.org/protobuf/types/known/wrapperspb" reflect "reflect" sync "sync" + unsafe "unsafe" ) const ( @@ -91,71 +93,81 @@ func (OrchestrationStatus) EnumDescriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{0} } -type CreateOrchestrationAction int32 +type WorkerCapability int32 const ( - CreateOrchestrationAction_ERROR CreateOrchestrationAction = 0 - CreateOrchestrationAction_IGNORE CreateOrchestrationAction = 1 - CreateOrchestrationAction_TERMINATE CreateOrchestrationAction = 2 + WorkerCapability_WORKER_CAPABILITY_UNSPECIFIED WorkerCapability = 0 + // Indicates that the worker is capable of streaming instance history as a more optimized + // alternative to receiving the full history embedded in the orchestrator work-item. + // When set, the service may return work items without any history events as an optimization. + // It is strongly recommended that all SDKs support this capability. + WorkerCapability_WORKER_CAPABILITY_HISTORY_STREAMING WorkerCapability = 1 + // Indicates that the worker supports scheduled tasks. + // The service may send schedule-triggered orchestration work items, + // and the worker must handle them, including the scheduledTime field. + WorkerCapability_WORKER_CAPABILITY_SCHEDULED_TASKS WorkerCapability = 2 + // Signals that the worker can handle large payloads stored externally (e.g., Blob Storage). + // Work items may contain URI references instead of inline data, and the worker must fetch them. + // This avoids message size limits and reduces network overhead. + WorkerCapability_WORKER_CAPABILITY_LARGE_PAYLOADS WorkerCapability = 3 ) -// Enum value maps for CreateOrchestrationAction. +// Enum value maps for WorkerCapability. var ( - CreateOrchestrationAction_name = map[int32]string{ - 0: "ERROR", - 1: "IGNORE", - 2: "TERMINATE", - } - CreateOrchestrationAction_value = map[string]int32{ - "ERROR": 0, - "IGNORE": 1, - "TERMINATE": 2, + WorkerCapability_name = map[int32]string{ + 0: "WORKER_CAPABILITY_UNSPECIFIED", + 1: "WORKER_CAPABILITY_HISTORY_STREAMING", + 2: "WORKER_CAPABILITY_SCHEDULED_TASKS", + 3: "WORKER_CAPABILITY_LARGE_PAYLOADS", + } + WorkerCapability_value = map[string]int32{ + "WORKER_CAPABILITY_UNSPECIFIED": 0, + "WORKER_CAPABILITY_HISTORY_STREAMING": 1, + "WORKER_CAPABILITY_SCHEDULED_TASKS": 2, + "WORKER_CAPABILITY_LARGE_PAYLOADS": 3, } ) -func (x CreateOrchestrationAction) Enum() *CreateOrchestrationAction { - p := new(CreateOrchestrationAction) +func (x WorkerCapability) Enum() *WorkerCapability { + p := new(WorkerCapability) *p = x return p } -func (x CreateOrchestrationAction) String() string { +func (x WorkerCapability) String() string { return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (CreateOrchestrationAction) Descriptor() protoreflect.EnumDescriptor { +func (WorkerCapability) Descriptor() protoreflect.EnumDescriptor { return file_orchestrator_service_proto_enumTypes[1].Descriptor() } -func (CreateOrchestrationAction) Type() protoreflect.EnumType { +func (WorkerCapability) Type() protoreflect.EnumType { return &file_orchestrator_service_proto_enumTypes[1] } -func (x CreateOrchestrationAction) Number() protoreflect.EnumNumber { +func (x WorkerCapability) Number() protoreflect.EnumNumber { return protoreflect.EnumNumber(x) } -// Deprecated: Use CreateOrchestrationAction.Descriptor instead. -func (CreateOrchestrationAction) EnumDescriptor() ([]byte, []int) { +// Deprecated: Use WorkerCapability.Descriptor instead. +func (WorkerCapability) EnumDescriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{1} } type OrchestrationInstance struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + ExecutionId *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - ExecutionId *wrappers.StringValue `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` + sizeCache protoimpl.SizeCache } func (x *OrchestrationInstance) Reset() { *x = OrchestrationInstance{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[0] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *OrchestrationInstance) String() string { @@ -166,7 +178,7 @@ func (*OrchestrationInstance) ProtoMessage() {} func (x *OrchestrationInstance) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[0] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -188,7 +200,7 @@ func (x *OrchestrationInstance) GetInstanceId() string { return "" } -func (x *OrchestrationInstance) GetExecutionId() *wrappers.StringValue { +func (x *OrchestrationInstance) GetExecutionId() *wrapperspb.StringValue { if x != nil { return x.ExecutionId } @@ -196,24 +208,23 @@ func (x *OrchestrationInstance) GetExecutionId() *wrappers.StringValue { } type ActivityRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` - TaskId int32 `protobuf:"varint,5,opt,name=taskId,proto3" json:"taskId,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` + TaskId int32 `protobuf:"varint,5,opt,name=taskId,proto3" json:"taskId,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,6,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + Tags map[string]string `protobuf:"bytes,7,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActivityRequest) Reset() { *x = ActivityRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[1] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ActivityRequest) String() string { @@ -224,7 +235,7 @@ func (*ActivityRequest) ProtoMessage() {} func (x *ActivityRequest) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[1] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -246,14 +257,14 @@ func (x *ActivityRequest) GetName() string { return "" } -func (x *ActivityRequest) GetVersion() *wrappers.StringValue { +func (x *ActivityRequest) GetVersion() *wrapperspb.StringValue { if x != nil { return x.Version } return nil } -func (x *ActivityRequest) GetInput() *wrappers.StringValue { +func (x *ActivityRequest) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -274,24 +285,36 @@ func (x *ActivityRequest) GetTaskId() int32 { return 0 } -type ActivityResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *ActivityRequest) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil +} + +func (x *ActivityRequest) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - TaskId int32 `protobuf:"varint,2,opt,name=taskId,proto3" json:"taskId,omitempty"` - Result *wrappers.StringValue `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,4,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` +type ActivityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + TaskId int32 `protobuf:"varint,2,opt,name=taskId,proto3" json:"taskId,omitempty"` + Result *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=result,proto3" json:"result,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,4,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + CompletionToken string `protobuf:"bytes,5,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ActivityResponse) Reset() { *x = ActivityResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[2] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ActivityResponse) String() string { @@ -302,7 +325,7 @@ func (*ActivityResponse) ProtoMessage() {} func (x *ActivityResponse) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[2] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -331,7 +354,7 @@ func (x *ActivityResponse) GetTaskId() int32 { return 0 } -func (x *ActivityResponse) GetResult() *wrappers.StringValue { +func (x *ActivityResponse) GetResult() *wrapperspb.StringValue { if x != nil { return x.Result } @@ -345,25 +368,30 @@ func (x *ActivityResponse) GetFailureDetails() *TaskFailureDetails { return nil } -type TaskFailureDetails struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *ActivityResponse) GetCompletionToken() string { + if x != nil { + return x.CompletionToken + } + return "" +} - ErrorType string `protobuf:"bytes,1,opt,name=errorType,proto3" json:"errorType,omitempty"` - ErrorMessage string `protobuf:"bytes,2,opt,name=errorMessage,proto3" json:"errorMessage,omitempty"` - StackTrace *wrappers.StringValue `protobuf:"bytes,3,opt,name=stackTrace,proto3" json:"stackTrace,omitempty"` - InnerFailure *TaskFailureDetails `protobuf:"bytes,4,opt,name=innerFailure,proto3" json:"innerFailure,omitempty"` - IsNonRetriable bool `protobuf:"varint,5,opt,name=isNonRetriable,proto3" json:"isNonRetriable,omitempty"` +type TaskFailureDetails struct { + state protoimpl.MessageState `protogen:"open.v1"` + ErrorType string `protobuf:"bytes,1,opt,name=errorType,proto3" json:"errorType,omitempty"` + ErrorMessage string `protobuf:"bytes,2,opt,name=errorMessage,proto3" json:"errorMessage,omitempty"` + StackTrace *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=stackTrace,proto3" json:"stackTrace,omitempty"` + InnerFailure *TaskFailureDetails `protobuf:"bytes,4,opt,name=innerFailure,proto3" json:"innerFailure,omitempty"` + IsNonRetriable bool `protobuf:"varint,5,opt,name=isNonRetriable,proto3" json:"isNonRetriable,omitempty"` + Properties map[string]*structpb.Value `protobuf:"bytes,6,rep,name=properties,proto3" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskFailureDetails) Reset() { *x = TaskFailureDetails{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[3] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TaskFailureDetails) String() string { @@ -374,7 +402,7 @@ func (*TaskFailureDetails) ProtoMessage() {} func (x *TaskFailureDetails) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[3] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -403,7 +431,7 @@ func (x *TaskFailureDetails) GetErrorMessage() string { return "" } -func (x *TaskFailureDetails) GetStackTrace() *wrappers.StringValue { +func (x *TaskFailureDetails) GetStackTrace() *wrapperspb.StringValue { if x != nil { return x.StackTrace } @@ -424,24 +452,28 @@ func (x *TaskFailureDetails) GetIsNonRetriable() bool { return false } -type ParentInstanceInfo struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *TaskFailureDetails) GetProperties() map[string]*structpb.Value { + if x != nil { + return x.Properties + } + return nil +} - TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` - Name *wrappers.StringValue `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` +type ParentInstanceInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` + Name *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ParentInstanceInfo) Reset() { *x = ParentInstanceInfo{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[4] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ParentInstanceInfo) String() string { @@ -452,7 +484,7 @@ func (*ParentInstanceInfo) ProtoMessage() {} func (x *ParentInstanceInfo) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[4] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -474,14 +506,14 @@ func (x *ParentInstanceInfo) GetTaskScheduledId() int32 { return 0 } -func (x *ParentInstanceInfo) GetName() *wrappers.StringValue { +func (x *ParentInstanceInfo) GetName() *wrapperspb.StringValue { if x != nil { return x.Name } return nil } -func (x *ParentInstanceInfo) GetVersion() *wrappers.StringValue { +func (x *ParentInstanceInfo) GetVersion() *wrapperspb.StringValue { if x != nil { return x.Version } @@ -496,23 +528,20 @@ func (x *ParentInstanceInfo) GetOrchestrationInstance() *OrchestrationInstance { } type TraceContext struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - TraceParent string `protobuf:"bytes,1,opt,name=traceParent,proto3" json:"traceParent,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + TraceParent string `protobuf:"bytes,1,opt,name=traceParent,proto3" json:"traceParent,omitempty"` // Deprecated: Marked as deprecated in orchestrator_service.proto. - SpanID string `protobuf:"bytes,2,opt,name=spanID,proto3" json:"spanID,omitempty"` - TraceState *wrappers.StringValue `protobuf:"bytes,3,opt,name=traceState,proto3" json:"traceState,omitempty"` + SpanID string `protobuf:"bytes,2,opt,name=spanID,proto3" json:"spanID,omitempty"` + TraceState *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=traceState,proto3" json:"traceState,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TraceContext) Reset() { *x = TraceContext{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[5] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TraceContext) String() string { @@ -523,7 +552,7 @@ func (*TraceContext) ProtoMessage() {} func (x *TraceContext) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[5] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -553,7 +582,7 @@ func (x *TraceContext) GetSpanID() string { return "" } -func (x *TraceContext) GetTraceState() *wrappers.StringValue { +func (x *TraceContext) GetTraceState() *wrapperspb.StringValue { if x != nil { return x.TraceState } @@ -561,27 +590,25 @@ func (x *TraceContext) GetTraceState() *wrappers.StringValue { } type ExecutionStartedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` - ParentInstance *ParentInstanceInfo `protobuf:"bytes,5,opt,name=parentInstance,proto3" json:"parentInstance,omitempty"` - ScheduledStartTimestamp *timestamp.Timestamp `protobuf:"bytes,6,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` - ParentTraceContext *TraceContext `protobuf:"bytes,7,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` - OrchestrationSpanID *wrappers.StringValue `protobuf:"bytes,8,opt,name=orchestrationSpanID,proto3" json:"orchestrationSpanID,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + OrchestrationInstance *OrchestrationInstance `protobuf:"bytes,4,opt,name=orchestrationInstance,proto3" json:"orchestrationInstance,omitempty"` + ParentInstance *ParentInstanceInfo `protobuf:"bytes,5,opt,name=parentInstance,proto3" json:"parentInstance,omitempty"` + ScheduledStartTimestamp *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,7,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + OrchestrationSpanID *wrapperspb.StringValue `protobuf:"bytes,8,opt,name=orchestrationSpanID,proto3" json:"orchestrationSpanID,omitempty"` + Tags map[string]string `protobuf:"bytes,9,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExecutionStartedEvent) Reset() { *x = ExecutionStartedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[6] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecutionStartedEvent) String() string { @@ -592,7 +619,7 @@ func (*ExecutionStartedEvent) ProtoMessage() {} func (x *ExecutionStartedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[6] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -614,14 +641,14 @@ func (x *ExecutionStartedEvent) GetName() string { return "" } -func (x *ExecutionStartedEvent) GetVersion() *wrappers.StringValue { +func (x *ExecutionStartedEvent) GetVersion() *wrapperspb.StringValue { if x != nil { return x.Version } return nil } -func (x *ExecutionStartedEvent) GetInput() *wrappers.StringValue { +func (x *ExecutionStartedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -642,7 +669,7 @@ func (x *ExecutionStartedEvent) GetParentInstance() *ParentInstanceInfo { return nil } -func (x *ExecutionStartedEvent) GetScheduledStartTimestamp() *timestamp.Timestamp { +func (x *ExecutionStartedEvent) GetScheduledStartTimestamp() *timestamppb.Timestamp { if x != nil { return x.ScheduledStartTimestamp } @@ -656,30 +683,34 @@ func (x *ExecutionStartedEvent) GetParentTraceContext() *TraceContext { return nil } -func (x *ExecutionStartedEvent) GetOrchestrationSpanID() *wrappers.StringValue { +func (x *ExecutionStartedEvent) GetOrchestrationSpanID() *wrapperspb.StringValue { if x != nil { return x.OrchestrationSpanID } return nil } -type ExecutionCompletedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *ExecutionStartedEvent) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} - OrchestrationStatus OrchestrationStatus `protobuf:"varint,1,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` - Result *wrappers.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,3,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` +type ExecutionCompletedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + OrchestrationStatus OrchestrationStatus `protobuf:"varint,1,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` + Result *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,3,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ExecutionCompletedEvent) Reset() { *x = ExecutionCompletedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[7] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecutionCompletedEvent) String() string { @@ -690,7 +721,7 @@ func (*ExecutionCompletedEvent) ProtoMessage() {} func (x *ExecutionCompletedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[7] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -712,7 +743,7 @@ func (x *ExecutionCompletedEvent) GetOrchestrationStatus() OrchestrationStatus { return OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING } -func (x *ExecutionCompletedEvent) GetResult() *wrappers.StringValue { +func (x *ExecutionCompletedEvent) GetResult() *wrapperspb.StringValue { if x != nil { return x.Result } @@ -727,21 +758,18 @@ func (x *ExecutionCompletedEvent) GetFailureDetails() *TaskFailureDetails { } type ExecutionTerminatedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Input *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` + Recurse bool `protobuf:"varint,2,opt,name=recurse,proto3" json:"recurse,omitempty"` unknownFields protoimpl.UnknownFields - - Input *wrappers.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` - Recurse bool `protobuf:"varint,2,opt,name=recurse,proto3" json:"recurse,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ExecutionTerminatedEvent) Reset() { *x = ExecutionTerminatedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[8] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecutionTerminatedEvent) String() string { @@ -752,7 +780,7 @@ func (*ExecutionTerminatedEvent) ProtoMessage() {} func (x *ExecutionTerminatedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[8] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -767,7 +795,7 @@ func (*ExecutionTerminatedEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{8} } -func (x *ExecutionTerminatedEvent) GetInput() *wrappers.StringValue { +func (x *ExecutionTerminatedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -782,23 +810,21 @@ func (x *ExecutionTerminatedEvent) GetRecurse() bool { } type TaskScheduledEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - ParentTraceContext *TraceContext `protobuf:"bytes,4,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,4,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + Tags map[string]string `protobuf:"bytes,5,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskScheduledEvent) Reset() { *x = TaskScheduledEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[9] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TaskScheduledEvent) String() string { @@ -809,7 +835,7 @@ func (*TaskScheduledEvent) ProtoMessage() {} func (x *TaskScheduledEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[9] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -831,14 +857,14 @@ func (x *TaskScheduledEvent) GetName() string { return "" } -func (x *TaskScheduledEvent) GetVersion() *wrappers.StringValue { +func (x *TaskScheduledEvent) GetVersion() *wrapperspb.StringValue { if x != nil { return x.Version } return nil } -func (x *TaskScheduledEvent) GetInput() *wrappers.StringValue { +func (x *TaskScheduledEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -852,22 +878,26 @@ func (x *TaskScheduledEvent) GetParentTraceContext() *TraceContext { return nil } -type TaskCompletedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *TaskScheduledEvent) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} - TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` - Result *wrappers.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` +type TaskCompletedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` + Result *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskCompletedEvent) Reset() { *x = TaskCompletedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[10] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TaskCompletedEvent) String() string { @@ -878,7 +908,7 @@ func (*TaskCompletedEvent) ProtoMessage() {} func (x *TaskCompletedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[10] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -900,7 +930,7 @@ func (x *TaskCompletedEvent) GetTaskScheduledId() int32 { return 0 } -func (x *TaskCompletedEvent) GetResult() *wrappers.StringValue { +func (x *TaskCompletedEvent) GetResult() *wrapperspb.StringValue { if x != nil { return x.Result } @@ -908,21 +938,18 @@ func (x *TaskCompletedEvent) GetResult() *wrappers.StringValue { } type TaskFailedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,2,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,2,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *TaskFailedEvent) Reset() { *x = TaskFailedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[11] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TaskFailedEvent) String() string { @@ -933,7 +960,7 @@ func (*TaskFailedEvent) ProtoMessage() {} func (x *TaskFailedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[11] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -963,24 +990,22 @@ func (x *TaskFailedEvent) GetFailureDetails() *TaskFailureDetails { } type SubOrchestrationInstanceCreatedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` - ParentTraceContext *TraceContext `protobuf:"bytes,5,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,5,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + Tags map[string]string `protobuf:"bytes,6,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SubOrchestrationInstanceCreatedEvent) Reset() { *x = SubOrchestrationInstanceCreatedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[12] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SubOrchestrationInstanceCreatedEvent) String() string { @@ -991,7 +1016,7 @@ func (*SubOrchestrationInstanceCreatedEvent) ProtoMessage() {} func (x *SubOrchestrationInstanceCreatedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[12] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1020,14 +1045,14 @@ func (x *SubOrchestrationInstanceCreatedEvent) GetName() string { return "" } -func (x *SubOrchestrationInstanceCreatedEvent) GetVersion() *wrappers.StringValue { +func (x *SubOrchestrationInstanceCreatedEvent) GetVersion() *wrapperspb.StringValue { if x != nil { return x.Version } return nil } -func (x *SubOrchestrationInstanceCreatedEvent) GetInput() *wrappers.StringValue { +func (x *SubOrchestrationInstanceCreatedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -1041,22 +1066,26 @@ func (x *SubOrchestrationInstanceCreatedEvent) GetParentTraceContext() *TraceCon return nil } -type SubOrchestrationInstanceCompletedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *SubOrchestrationInstanceCreatedEvent) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} - TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` - Result *wrappers.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` +type SubOrchestrationInstanceCompletedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` + Result *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SubOrchestrationInstanceCompletedEvent) Reset() { *x = SubOrchestrationInstanceCompletedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[13] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SubOrchestrationInstanceCompletedEvent) String() string { @@ -1067,7 +1096,7 @@ func (*SubOrchestrationInstanceCompletedEvent) ProtoMessage() {} func (x *SubOrchestrationInstanceCompletedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[13] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1089,7 +1118,7 @@ func (x *SubOrchestrationInstanceCompletedEvent) GetTaskScheduledId() int32 { return 0 } -func (x *SubOrchestrationInstanceCompletedEvent) GetResult() *wrappers.StringValue { +func (x *SubOrchestrationInstanceCompletedEvent) GetResult() *wrapperspb.StringValue { if x != nil { return x.Result } @@ -1097,21 +1126,18 @@ func (x *SubOrchestrationInstanceCompletedEvent) GetResult() *wrappers.StringVal } type SubOrchestrationInstanceFailedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,2,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + TaskScheduledId int32 `protobuf:"varint,1,opt,name=taskScheduledId,proto3" json:"taskScheduledId,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,2,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SubOrchestrationInstanceFailedEvent) Reset() { *x = SubOrchestrationInstanceFailedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[14] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *SubOrchestrationInstanceFailedEvent) String() string { @@ -1122,7 +1148,7 @@ func (*SubOrchestrationInstanceFailedEvent) ProtoMessage() {} func (x *SubOrchestrationInstanceFailedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[14] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1152,20 +1178,17 @@ func (x *SubOrchestrationInstanceFailedEvent) GetFailureDetails() *TaskFailureDe } type TimerCreatedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + FireAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` unknownFields protoimpl.UnknownFields - - FireAt *timestamp.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` + sizeCache protoimpl.SizeCache } func (x *TimerCreatedEvent) Reset() { *x = TimerCreatedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[15] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TimerCreatedEvent) String() string { @@ -1176,7 +1199,7 @@ func (*TimerCreatedEvent) ProtoMessage() {} func (x *TimerCreatedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[15] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1191,7 +1214,7 @@ func (*TimerCreatedEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{15} } -func (x *TimerCreatedEvent) GetFireAt() *timestamp.Timestamp { +func (x *TimerCreatedEvent) GetFireAt() *timestamppb.Timestamp { if x != nil { return x.FireAt } @@ -1199,21 +1222,18 @@ func (x *TimerCreatedEvent) GetFireAt() *timestamp.Timestamp { } type TimerFiredEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + FireAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` + TimerId int32 `protobuf:"varint,2,opt,name=timerId,proto3" json:"timerId,omitempty"` unknownFields protoimpl.UnknownFields - - FireAt *timestamp.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` - TimerId int32 `protobuf:"varint,2,opt,name=timerId,proto3" json:"timerId,omitempty"` + sizeCache protoimpl.SizeCache } func (x *TimerFiredEvent) Reset() { *x = TimerFiredEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[16] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *TimerFiredEvent) String() string { @@ -1224,7 +1244,7 @@ func (*TimerFiredEvent) ProtoMessage() {} func (x *TimerFiredEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[16] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1239,7 +1259,7 @@ func (*TimerFiredEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{16} } -func (x *TimerFiredEvent) GetFireAt() *timestamp.Timestamp { +func (x *TimerFiredEvent) GetFireAt() *timestamppb.Timestamp { if x != nil { return x.FireAt } @@ -1254,18 +1274,16 @@ func (x *TimerFiredEvent) GetTimerId() int32 { } type OrchestratorStartedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *OrchestratorStartedEvent) Reset() { *x = OrchestratorStartedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[17] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *OrchestratorStartedEvent) String() string { @@ -1276,7 +1294,7 @@ func (*OrchestratorStartedEvent) ProtoMessage() {} func (x *OrchestratorStartedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[17] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1292,18 +1310,16 @@ func (*OrchestratorStartedEvent) Descriptor() ([]byte, []int) { } type OrchestratorCompletedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *OrchestratorCompletedEvent) Reset() { *x = OrchestratorCompletedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[18] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *OrchestratorCompletedEvent) String() string { @@ -1314,7 +1330,7 @@ func (*OrchestratorCompletedEvent) ProtoMessage() {} func (x *OrchestratorCompletedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[18] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1330,22 +1346,19 @@ func (*OrchestratorCompletedEvent) Descriptor() ([]byte, []int) { } type EventSentEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + sizeCache protoimpl.SizeCache } func (x *EventSentEvent) Reset() { *x = EventSentEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[19] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *EventSentEvent) String() string { @@ -1356,7 +1369,7 @@ func (*EventSentEvent) ProtoMessage() {} func (x *EventSentEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[19] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1385,7 +1398,7 @@ func (x *EventSentEvent) GetName() string { return "" } -func (x *EventSentEvent) GetInput() *wrappers.StringValue { +func (x *EventSentEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -1393,21 +1406,18 @@ func (x *EventSentEvent) GetInput() *wrappers.StringValue { } type EventRaisedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=input,proto3" json:"input,omitempty"` unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,2,opt,name=input,proto3" json:"input,omitempty"` + sizeCache protoimpl.SizeCache } func (x *EventRaisedEvent) Reset() { *x = EventRaisedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[20] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *EventRaisedEvent) String() string { @@ -1418,7 +1428,7 @@ func (*EventRaisedEvent) ProtoMessage() {} func (x *EventRaisedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[20] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1440,7 +1450,7 @@ func (x *EventRaisedEvent) GetName() string { return "" } -func (x *EventRaisedEvent) GetInput() *wrappers.StringValue { +func (x *EventRaisedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -1448,20 +1458,17 @@ func (x *EventRaisedEvent) GetInput() *wrappers.StringValue { } type GenericEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Data *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` unknownFields protoimpl.UnknownFields - - Data *wrappers.StringValue `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + sizeCache protoimpl.SizeCache } func (x *GenericEvent) Reset() { *x = GenericEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[21] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *GenericEvent) String() string { @@ -1472,7 +1479,7 @@ func (*GenericEvent) ProtoMessage() {} func (x *GenericEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[21] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1487,7 +1494,7 @@ func (*GenericEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{21} } -func (x *GenericEvent) GetData() *wrappers.StringValue { +func (x *GenericEvent) GetData() *wrapperspb.StringValue { if x != nil { return x.Data } @@ -1495,20 +1502,17 @@ func (x *GenericEvent) GetData() *wrappers.StringValue { } type HistoryStateEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - OrchestrationState *OrchestrationState `protobuf:"bytes,1,opt,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + OrchestrationState *OrchestrationState `protobuf:"bytes,1,opt,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *HistoryStateEvent) Reset() { *x = HistoryStateEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[22] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *HistoryStateEvent) String() string { @@ -1519,7 +1523,7 @@ func (*HistoryStateEvent) ProtoMessage() {} func (x *HistoryStateEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[22] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1542,20 +1546,17 @@ func (x *HistoryStateEvent) GetOrchestrationState() *OrchestrationState { } type ContinueAsNewEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Input *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` unknownFields protoimpl.UnknownFields - - Input *wrappers.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ContinueAsNewEvent) Reset() { *x = ContinueAsNewEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[23] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ContinueAsNewEvent) String() string { @@ -1566,7 +1567,7 @@ func (*ContinueAsNewEvent) ProtoMessage() {} func (x *ContinueAsNewEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[23] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1581,7 +1582,7 @@ func (*ContinueAsNewEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{23} } -func (x *ContinueAsNewEvent) GetInput() *wrappers.StringValue { +func (x *ContinueAsNewEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -1589,20 +1590,17 @@ func (x *ContinueAsNewEvent) GetInput() *wrappers.StringValue { } type ExecutionSuspendedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Input *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` unknownFields protoimpl.UnknownFields - - Input *wrappers.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ExecutionSuspendedEvent) Reset() { *x = ExecutionSuspendedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[24] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecutionSuspendedEvent) String() string { @@ -1613,7 +1611,7 @@ func (*ExecutionSuspendedEvent) ProtoMessage() {} func (x *ExecutionSuspendedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[24] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1628,7 +1626,7 @@ func (*ExecutionSuspendedEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{24} } -func (x *ExecutionSuspendedEvent) GetInput() *wrappers.StringValue { +func (x *ExecutionSuspendedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } @@ -1636,20 +1634,17 @@ func (x *ExecutionSuspendedEvent) GetInput() *wrappers.StringValue { } type ExecutionResumedEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` + Input *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` unknownFields protoimpl.UnknownFields - - Input *wrappers.StringValue `protobuf:"bytes,1,opt,name=input,proto3" json:"input,omitempty"` + sizeCache protoimpl.SizeCache } func (x *ExecutionResumedEvent) Reset() { *x = ExecutionResumedEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[25] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *ExecutionResumedEvent) String() string { @@ -1660,7 +1655,7 @@ func (*ExecutionResumedEvent) ProtoMessage() {} func (x *ExecutionResumedEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[25] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1675,63 +1670,40 @@ func (*ExecutionResumedEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{25} } -func (x *ExecutionResumedEvent) GetInput() *wrappers.StringValue { +func (x *ExecutionResumedEvent) GetInput() *wrapperspb.StringValue { if x != nil { return x.Input } return nil } -type HistoryEvent struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - EventId int32 `protobuf:"varint,1,opt,name=eventId,proto3" json:"eventId,omitempty"` - Timestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` - // Types that are assignable to EventType: - // - // *HistoryEvent_ExecutionStarted - // *HistoryEvent_ExecutionCompleted - // *HistoryEvent_ExecutionTerminated - // *HistoryEvent_TaskScheduled - // *HistoryEvent_TaskCompleted - // *HistoryEvent_TaskFailed - // *HistoryEvent_SubOrchestrationInstanceCreated - // *HistoryEvent_SubOrchestrationInstanceCompleted - // *HistoryEvent_SubOrchestrationInstanceFailed - // *HistoryEvent_TimerCreated - // *HistoryEvent_TimerFired - // *HistoryEvent_OrchestratorStarted - // *HistoryEvent_OrchestratorCompleted - // *HistoryEvent_EventSent - // *HistoryEvent_EventRaised - // *HistoryEvent_GenericEvent - // *HistoryEvent_HistoryState - // *HistoryEvent_ContinueAsNew - // *HistoryEvent_ExecutionSuspended - // *HistoryEvent_ExecutionResumed - EventType isHistoryEvent_EventType `protobuf_oneof:"eventType"` +type EntityOperationSignaledEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=requestId,proto3" json:"requestId,omitempty"` + Operation string `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` + ScheduledTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + TargetInstanceId *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=targetInstanceId,proto3" json:"targetInstanceId,omitempty"` // used only within histories, null in messages + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *HistoryEvent) Reset() { - *x = HistoryEvent{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[26] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityOperationSignaledEvent) Reset() { + *x = EntityOperationSignaledEvent{} + mi := &file_orchestrator_service_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *HistoryEvent) String() string { +func (x *EntityOperationSignaledEvent) String() string { return protoimpl.X.MessageStringOf(x) } -func (*HistoryEvent) ProtoMessage() {} +func (*EntityOperationSignaledEvent) ProtoMessage() {} -func (x *HistoryEvent) ProtoReflect() protoreflect.Message { +func (x *EntityOperationSignaledEvent) ProtoReflect() protoreflect.Message { mi := &file_orchestrator_service_proto_msgTypes[26] - if protoimpl.UnsafeEnabled && x != nil { + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -1741,324 +1713,335 @@ func (x *HistoryEvent) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use HistoryEvent.ProtoReflect.Descriptor instead. -func (*HistoryEvent) Descriptor() ([]byte, []int) { +// Deprecated: Use EntityOperationSignaledEvent.ProtoReflect.Descriptor instead. +func (*EntityOperationSignaledEvent) Descriptor() ([]byte, []int) { return file_orchestrator_service_proto_rawDescGZIP(), []int{26} } -func (x *HistoryEvent) GetEventId() int32 { +func (x *EntityOperationSignaledEvent) GetRequestId() string { if x != nil { - return x.EventId + return x.RequestId } - return 0 + return "" } -func (x *HistoryEvent) GetTimestamp() *timestamp.Timestamp { +func (x *EntityOperationSignaledEvent) GetOperation() string { if x != nil { - return x.Timestamp + return x.Operation } - return nil + return "" } -func (m *HistoryEvent) GetEventType() isHistoryEvent_EventType { - if m != nil { - return m.EventType +func (x *EntityOperationSignaledEvent) GetScheduledTime() *timestamppb.Timestamp { + if x != nil { + return x.ScheduledTime } return nil } -func (x *HistoryEvent) GetExecutionStarted() *ExecutionStartedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ExecutionStarted); ok { - return x.ExecutionStarted +func (x *EntityOperationSignaledEvent) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input } return nil } -func (x *HistoryEvent) GetExecutionCompleted() *ExecutionCompletedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ExecutionCompleted); ok { - return x.ExecutionCompleted +func (x *EntityOperationSignaledEvent) GetTargetInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.TargetInstanceId } return nil } -func (x *HistoryEvent) GetExecutionTerminated() *ExecutionTerminatedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ExecutionTerminated); ok { - return x.ExecutionTerminated - } - return nil +type EntityOperationCalledEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=requestId,proto3" json:"requestId,omitempty"` + Operation string `protobuf:"bytes,2,opt,name=operation,proto3" json:"operation,omitempty"` + ScheduledTime *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + ParentInstanceId *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=parentInstanceId,proto3" json:"parentInstanceId,omitempty"` // used only within messages, null in histories + ParentExecutionId *wrapperspb.StringValue `protobuf:"bytes,6,opt,name=parentExecutionId,proto3" json:"parentExecutionId,omitempty"` // used only within messages, null in histories + TargetInstanceId *wrapperspb.StringValue `protobuf:"bytes,7,opt,name=targetInstanceId,proto3" json:"targetInstanceId,omitempty"` // used only within histories, null in messages + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *HistoryEvent) GetTaskScheduled() *TaskScheduledEvent { - if x, ok := x.GetEventType().(*HistoryEvent_TaskScheduled); ok { - return x.TaskScheduled - } - return nil +func (x *EntityOperationCalledEvent) Reset() { + *x = EntityOperationCalledEvent{} + mi := &file_orchestrator_service_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *HistoryEvent) GetTaskCompleted() *TaskCompletedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_TaskCompleted); ok { - return x.TaskCompleted - } - return nil +func (x *EntityOperationCalledEvent) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *HistoryEvent) GetTaskFailed() *TaskFailedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_TaskFailed); ok { - return x.TaskFailed - } - return nil -} +func (*EntityOperationCalledEvent) ProtoMessage() {} -func (x *HistoryEvent) GetSubOrchestrationInstanceCreated() *SubOrchestrationInstanceCreatedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_SubOrchestrationInstanceCreated); ok { - return x.SubOrchestrationInstanceCreated +func (x *EntityOperationCalledEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[27] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -func (x *HistoryEvent) GetSubOrchestrationInstanceCompleted() *SubOrchestrationInstanceCompletedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_SubOrchestrationInstanceCompleted); ok { - return x.SubOrchestrationInstanceCompleted - } - return nil +// Deprecated: Use EntityOperationCalledEvent.ProtoReflect.Descriptor instead. +func (*EntityOperationCalledEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{27} } -func (x *HistoryEvent) GetSubOrchestrationInstanceFailed() *SubOrchestrationInstanceFailedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_SubOrchestrationInstanceFailed); ok { - return x.SubOrchestrationInstanceFailed +func (x *EntityOperationCalledEvent) GetRequestId() string { + if x != nil { + return x.RequestId } - return nil + return "" } -func (x *HistoryEvent) GetTimerCreated() *TimerCreatedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_TimerCreated); ok { - return x.TimerCreated +func (x *EntityOperationCalledEvent) GetOperation() string { + if x != nil { + return x.Operation } - return nil + return "" } -func (x *HistoryEvent) GetTimerFired() *TimerFiredEvent { - if x, ok := x.GetEventType().(*HistoryEvent_TimerFired); ok { - return x.TimerFired +func (x *EntityOperationCalledEvent) GetScheduledTime() *timestamppb.Timestamp { + if x != nil { + return x.ScheduledTime } return nil } -func (x *HistoryEvent) GetOrchestratorStarted() *OrchestratorStartedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_OrchestratorStarted); ok { - return x.OrchestratorStarted +func (x *EntityOperationCalledEvent) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input } return nil } -func (x *HistoryEvent) GetOrchestratorCompleted() *OrchestratorCompletedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_OrchestratorCompleted); ok { - return x.OrchestratorCompleted +func (x *EntityOperationCalledEvent) GetParentInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.ParentInstanceId } return nil } -func (x *HistoryEvent) GetEventSent() *EventSentEvent { - if x, ok := x.GetEventType().(*HistoryEvent_EventSent); ok { - return x.EventSent +func (x *EntityOperationCalledEvent) GetParentExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ParentExecutionId } return nil } -func (x *HistoryEvent) GetEventRaised() *EventRaisedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_EventRaised); ok { - return x.EventRaised +func (x *EntityOperationCalledEvent) GetTargetInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.TargetInstanceId } return nil } -func (x *HistoryEvent) GetGenericEvent() *GenericEvent { - if x, ok := x.GetEventType().(*HistoryEvent_GenericEvent); ok { - return x.GenericEvent - } - return nil +type EntityLockRequestedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + CriticalSectionId string `protobuf:"bytes,1,opt,name=criticalSectionId,proto3" json:"criticalSectionId,omitempty"` + LockSet []string `protobuf:"bytes,2,rep,name=lockSet,proto3" json:"lockSet,omitempty"` + Position int32 `protobuf:"varint,3,opt,name=position,proto3" json:"position,omitempty"` + ParentInstanceId *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=parentInstanceId,proto3" json:"parentInstanceId,omitempty"` // used only within messages, null in histories + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *HistoryEvent) GetHistoryState() *HistoryStateEvent { - if x, ok := x.GetEventType().(*HistoryEvent_HistoryState); ok { - return x.HistoryState - } - return nil +func (x *EntityLockRequestedEvent) Reset() { + *x = EntityLockRequestedEvent{} + mi := &file_orchestrator_service_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *HistoryEvent) GetContinueAsNew() *ContinueAsNewEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ContinueAsNew); ok { - return x.ContinueAsNew - } - return nil +func (x *EntityLockRequestedEvent) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *HistoryEvent) GetExecutionSuspended() *ExecutionSuspendedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ExecutionSuspended); ok { - return x.ExecutionSuspended - } - return nil -} +func (*EntityLockRequestedEvent) ProtoMessage() {} -func (x *HistoryEvent) GetExecutionResumed() *ExecutionResumedEvent { - if x, ok := x.GetEventType().(*HistoryEvent_ExecutionResumed); ok { - return x.ExecutionResumed +func (x *EntityLockRequestedEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[28] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -type isHistoryEvent_EventType interface { - isHistoryEvent_EventType() +// Deprecated: Use EntityLockRequestedEvent.ProtoReflect.Descriptor instead. +func (*EntityLockRequestedEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{28} } -type HistoryEvent_ExecutionStarted struct { - ExecutionStarted *ExecutionStartedEvent `protobuf:"bytes,3,opt,name=executionStarted,proto3,oneof"` +func (x *EntityLockRequestedEvent) GetCriticalSectionId() string { + if x != nil { + return x.CriticalSectionId + } + return "" } -type HistoryEvent_ExecutionCompleted struct { - ExecutionCompleted *ExecutionCompletedEvent `protobuf:"bytes,4,opt,name=executionCompleted,proto3,oneof"` +func (x *EntityLockRequestedEvent) GetLockSet() []string { + if x != nil { + return x.LockSet + } + return nil } -type HistoryEvent_ExecutionTerminated struct { - ExecutionTerminated *ExecutionTerminatedEvent `protobuf:"bytes,5,opt,name=executionTerminated,proto3,oneof"` +func (x *EntityLockRequestedEvent) GetPosition() int32 { + if x != nil { + return x.Position + } + return 0 } -type HistoryEvent_TaskScheduled struct { - TaskScheduled *TaskScheduledEvent `protobuf:"bytes,6,opt,name=taskScheduled,proto3,oneof"` +func (x *EntityLockRequestedEvent) GetParentInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.ParentInstanceId + } + return nil } -type HistoryEvent_TaskCompleted struct { - TaskCompleted *TaskCompletedEvent `protobuf:"bytes,7,opt,name=taskCompleted,proto3,oneof"` +type EntityOperationCompletedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=requestId,proto3" json:"requestId,omitempty"` + Output *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -type HistoryEvent_TaskFailed struct { - TaskFailed *TaskFailedEvent `protobuf:"bytes,8,opt,name=taskFailed,proto3,oneof"` +func (x *EntityOperationCompletedEvent) Reset() { + *x = EntityOperationCompletedEvent{} + mi := &file_orchestrator_service_proto_msgTypes[29] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -type HistoryEvent_SubOrchestrationInstanceCreated struct { - SubOrchestrationInstanceCreated *SubOrchestrationInstanceCreatedEvent `protobuf:"bytes,9,opt,name=subOrchestrationInstanceCreated,proto3,oneof"` +func (x *EntityOperationCompletedEvent) String() string { + return protoimpl.X.MessageStringOf(x) } -type HistoryEvent_SubOrchestrationInstanceCompleted struct { - SubOrchestrationInstanceCompleted *SubOrchestrationInstanceCompletedEvent `protobuf:"bytes,10,opt,name=subOrchestrationInstanceCompleted,proto3,oneof"` -} +func (*EntityOperationCompletedEvent) ProtoMessage() {} -type HistoryEvent_SubOrchestrationInstanceFailed struct { - SubOrchestrationInstanceFailed *SubOrchestrationInstanceFailedEvent `protobuf:"bytes,11,opt,name=subOrchestrationInstanceFailed,proto3,oneof"` +func (x *EntityOperationCompletedEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[29] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type HistoryEvent_TimerCreated struct { - TimerCreated *TimerCreatedEvent `protobuf:"bytes,12,opt,name=timerCreated,proto3,oneof"` +// Deprecated: Use EntityOperationCompletedEvent.ProtoReflect.Descriptor instead. +func (*EntityOperationCompletedEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{29} } -type HistoryEvent_TimerFired struct { - TimerFired *TimerFiredEvent `protobuf:"bytes,13,opt,name=timerFired,proto3,oneof"` +func (x *EntityOperationCompletedEvent) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" } -type HistoryEvent_OrchestratorStarted struct { - OrchestratorStarted *OrchestratorStartedEvent `protobuf:"bytes,14,opt,name=orchestratorStarted,proto3,oneof"` +func (x *EntityOperationCompletedEvent) GetOutput() *wrapperspb.StringValue { + if x != nil { + return x.Output + } + return nil } -type HistoryEvent_OrchestratorCompleted struct { - OrchestratorCompleted *OrchestratorCompletedEvent `protobuf:"bytes,15,opt,name=orchestratorCompleted,proto3,oneof"` +type EntityOperationFailedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=requestId,proto3" json:"requestId,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,2,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -type HistoryEvent_EventSent struct { - EventSent *EventSentEvent `protobuf:"bytes,16,opt,name=eventSent,proto3,oneof"` +func (x *EntityOperationFailedEvent) Reset() { + *x = EntityOperationFailedEvent{} + mi := &file_orchestrator_service_proto_msgTypes[30] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -type HistoryEvent_EventRaised struct { - EventRaised *EventRaisedEvent `protobuf:"bytes,17,opt,name=eventRaised,proto3,oneof"` +func (x *EntityOperationFailedEvent) String() string { + return protoimpl.X.MessageStringOf(x) } -type HistoryEvent_GenericEvent struct { - GenericEvent *GenericEvent `protobuf:"bytes,18,opt,name=genericEvent,proto3,oneof"` -} +func (*EntityOperationFailedEvent) ProtoMessage() {} -type HistoryEvent_HistoryState struct { - HistoryState *HistoryStateEvent `protobuf:"bytes,19,opt,name=historyState,proto3,oneof"` +func (x *EntityOperationFailedEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[30] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -type HistoryEvent_ContinueAsNew struct { - ContinueAsNew *ContinueAsNewEvent `protobuf:"bytes,20,opt,name=continueAsNew,proto3,oneof"` +// Deprecated: Use EntityOperationFailedEvent.ProtoReflect.Descriptor instead. +func (*EntityOperationFailedEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{30} } -type HistoryEvent_ExecutionSuspended struct { - ExecutionSuspended *ExecutionSuspendedEvent `protobuf:"bytes,21,opt,name=executionSuspended,proto3,oneof"` +func (x *EntityOperationFailedEvent) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" } -type HistoryEvent_ExecutionResumed struct { - ExecutionResumed *ExecutionResumedEvent `protobuf:"bytes,22,opt,name=executionResumed,proto3,oneof"` +func (x *EntityOperationFailedEvent) GetFailureDetails() *TaskFailureDetails { + if x != nil { + return x.FailureDetails + } + return nil } -func (*HistoryEvent_ExecutionStarted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_ExecutionCompleted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_ExecutionTerminated) isHistoryEvent_EventType() {} - -func (*HistoryEvent_TaskScheduled) isHistoryEvent_EventType() {} - -func (*HistoryEvent_TaskCompleted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_TaskFailed) isHistoryEvent_EventType() {} - -func (*HistoryEvent_SubOrchestrationInstanceCreated) isHistoryEvent_EventType() {} - -func (*HistoryEvent_SubOrchestrationInstanceCompleted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_SubOrchestrationInstanceFailed) isHistoryEvent_EventType() {} - -func (*HistoryEvent_TimerCreated) isHistoryEvent_EventType() {} - -func (*HistoryEvent_TimerFired) isHistoryEvent_EventType() {} - -func (*HistoryEvent_OrchestratorStarted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_OrchestratorCompleted) isHistoryEvent_EventType() {} - -func (*HistoryEvent_EventSent) isHistoryEvent_EventType() {} - -func (*HistoryEvent_EventRaised) isHistoryEvent_EventType() {} - -func (*HistoryEvent_GenericEvent) isHistoryEvent_EventType() {} - -func (*HistoryEvent_HistoryState) isHistoryEvent_EventType() {} - -func (*HistoryEvent_ContinueAsNew) isHistoryEvent_EventType() {} - -func (*HistoryEvent_ExecutionSuspended) isHistoryEvent_EventType() {} - -func (*HistoryEvent_ExecutionResumed) isHistoryEvent_EventType() {} - -type ScheduleTaskAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` +type EntityUnlockSentEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + CriticalSectionId string `protobuf:"bytes,1,opt,name=criticalSectionId,proto3" json:"criticalSectionId,omitempty"` + ParentInstanceId *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=parentInstanceId,proto3" json:"parentInstanceId,omitempty"` // used only within messages, null in histories + TargetInstanceId *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=targetInstanceId,proto3" json:"targetInstanceId,omitempty"` // used only within histories, null in messages + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ScheduleTaskAction) Reset() { - *x = ScheduleTaskAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[27] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityUnlockSentEvent) Reset() { + *x = EntityUnlockSentEvent{} + mi := &file_orchestrator_service_proto_msgTypes[31] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *ScheduleTaskAction) String() string { +func (x *EntityUnlockSentEvent) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ScheduleTaskAction) ProtoMessage() {} +func (*EntityUnlockSentEvent) ProtoMessage() {} -func (x *ScheduleTaskAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[27] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityUnlockSentEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[31] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2068,61 +2051,55 @@ func (x *ScheduleTaskAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ScheduleTaskAction.ProtoReflect.Descriptor instead. -func (*ScheduleTaskAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{27} +// Deprecated: Use EntityUnlockSentEvent.ProtoReflect.Descriptor instead. +func (*EntityUnlockSentEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{31} } -func (x *ScheduleTaskAction) GetName() string { +func (x *EntityUnlockSentEvent) GetCriticalSectionId() string { if x != nil { - return x.Name + return x.CriticalSectionId } return "" } -func (x *ScheduleTaskAction) GetVersion() *wrappers.StringValue { +func (x *EntityUnlockSentEvent) GetParentInstanceId() *wrapperspb.StringValue { if x != nil { - return x.Version + return x.ParentInstanceId } return nil } -func (x *ScheduleTaskAction) GetInput() *wrappers.StringValue { +func (x *EntityUnlockSentEvent) GetTargetInstanceId() *wrapperspb.StringValue { if x != nil { - return x.Input + return x.TargetInstanceId } return nil } -type CreateSubOrchestrationAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` +type EntityLockGrantedEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + CriticalSectionId string `protobuf:"bytes,1,opt,name=criticalSectionId,proto3" json:"criticalSectionId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateSubOrchestrationAction) Reset() { - *x = CreateSubOrchestrationAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[28] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityLockGrantedEvent) Reset() { + *x = EntityLockGrantedEvent{} + mi := &file_orchestrator_service_proto_msgTypes[32] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateSubOrchestrationAction) String() string { +func (x *EntityLockGrantedEvent) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateSubOrchestrationAction) ProtoMessage() {} +func (*EntityLockGrantedEvent) ProtoMessage() {} -func (x *CreateSubOrchestrationAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[28] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityLockGrantedEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[32] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2132,65 +2109,49 @@ func (x *CreateSubOrchestrationAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateSubOrchestrationAction.ProtoReflect.Descriptor instead. -func (*CreateSubOrchestrationAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{28} -} - -func (x *CreateSubOrchestrationAction) GetInstanceId() string { - if x != nil { - return x.InstanceId - } - return "" +// Deprecated: Use EntityLockGrantedEvent.ProtoReflect.Descriptor instead. +func (*EntityLockGrantedEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{32} } -func (x *CreateSubOrchestrationAction) GetName() string { +func (x *EntityLockGrantedEvent) GetCriticalSectionId() string { if x != nil { - return x.Name + return x.CriticalSectionId } return "" } -func (x *CreateSubOrchestrationAction) GetVersion() *wrappers.StringValue { - if x != nil { - return x.Version - } - return nil -} - -func (x *CreateSubOrchestrationAction) GetInput() *wrappers.StringValue { - if x != nil { - return x.Input - } - return nil -} - -type CreateTimerAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - FireAt *timestamp.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` +type ExecutionRewoundEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + Reason *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=reason,proto3" json:"reason,omitempty"` + ParentExecutionId *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=parentExecutionId,proto3" json:"parentExecutionId,omitempty"` // used only for rewinding suborchestrations, null otherwise + InstanceId *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=instanceId,proto3" json:"instanceId,omitempty"` // used only for rewinding suborchestrations, null otherwise + ParentTraceContext *TraceContext `protobuf:"bytes,4,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` // used only for rewinding suborchestrations, null otherwise + Name *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=name,proto3" json:"name,omitempty"` // used by DTS backend only + Version *wrapperspb.StringValue `protobuf:"bytes,6,opt,name=version,proto3" json:"version,omitempty"` // used by DTS backend only + Input *wrapperspb.StringValue `protobuf:"bytes,7,opt,name=input,proto3" json:"input,omitempty"` // used by DTS backend only + ParentInstance *ParentInstanceInfo `protobuf:"bytes,8,opt,name=parentInstance,proto3" json:"parentInstance,omitempty"` // used by DTS backend only + Tags map[string]string `protobuf:"bytes,9,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` // used by DTS backend only + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateTimerAction) Reset() { - *x = CreateTimerAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[29] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *ExecutionRewoundEvent) Reset() { + *x = ExecutionRewoundEvent{} + mi := &file_orchestrator_service_proto_msgTypes[33] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateTimerAction) String() string { +func (x *ExecutionRewoundEvent) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTimerAction) ProtoMessage() {} +func (*ExecutionRewoundEvent) ProtoMessage() {} -func (x *CreateTimerAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[29] - if protoimpl.UnsafeEnabled && x != nil { +func (x *ExecutionRewoundEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[33] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2200,112 +2161,129 @@ func (x *CreateTimerAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTimerAction.ProtoReflect.Descriptor instead. -func (*CreateTimerAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{29} +// Deprecated: Use ExecutionRewoundEvent.ProtoReflect.Descriptor instead. +func (*ExecutionRewoundEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{33} } -func (x *CreateTimerAction) GetFireAt() *timestamp.Timestamp { +func (x *ExecutionRewoundEvent) GetReason() *wrapperspb.StringValue { if x != nil { - return x.FireAt + return x.Reason } return nil } -type SendEventAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Instance *OrchestrationInstance `protobuf:"bytes,1,opt,name=instance,proto3" json:"instance,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Data *wrappers.StringValue `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` +func (x *ExecutionRewoundEvent) GetParentExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ParentExecutionId + } + return nil } -func (x *SendEventAction) Reset() { - *x = SendEventAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[30] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *ExecutionRewoundEvent) GetInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.InstanceId } + return nil } -func (x *SendEventAction) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *ExecutionRewoundEvent) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil } -func (*SendEventAction) ProtoMessage() {} - -func (x *SendEventAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[30] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *ExecutionRewoundEvent) GetName() *wrapperspb.StringValue { + if x != nil { + return x.Name } - return mi.MessageOf(x) + return nil } -// Deprecated: Use SendEventAction.ProtoReflect.Descriptor instead. -func (*SendEventAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{30} +func (x *ExecutionRewoundEvent) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version + } + return nil } -func (x *SendEventAction) GetInstance() *OrchestrationInstance { +func (x *ExecutionRewoundEvent) GetInput() *wrapperspb.StringValue { if x != nil { - return x.Instance + return x.Input } return nil } -func (x *SendEventAction) GetName() string { +func (x *ExecutionRewoundEvent) GetParentInstance() *ParentInstanceInfo { if x != nil { - return x.Name + return x.ParentInstance } - return "" + return nil } -func (x *SendEventAction) GetData() *wrappers.StringValue { +func (x *ExecutionRewoundEvent) GetTags() map[string]string { if x != nil { - return x.Data + return x.Tags } return nil } -type CompleteOrchestrationAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type HistoryEvent struct { + state protoimpl.MessageState `protogen:"open.v1"` + EventId int32 `protobuf:"varint,1,opt,name=eventId,proto3" json:"eventId,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` + // Types that are valid to be assigned to EventType: + // + // *HistoryEvent_ExecutionStarted + // *HistoryEvent_ExecutionCompleted + // *HistoryEvent_ExecutionTerminated + // *HistoryEvent_TaskScheduled + // *HistoryEvent_TaskCompleted + // *HistoryEvent_TaskFailed + // *HistoryEvent_SubOrchestrationInstanceCreated + // *HistoryEvent_SubOrchestrationInstanceCompleted + // *HistoryEvent_SubOrchestrationInstanceFailed + // *HistoryEvent_TimerCreated + // *HistoryEvent_TimerFired + // *HistoryEvent_OrchestratorStarted + // *HistoryEvent_OrchestratorCompleted + // *HistoryEvent_EventSent + // *HistoryEvent_EventRaised + // *HistoryEvent_GenericEvent + // *HistoryEvent_HistoryState + // *HistoryEvent_ContinueAsNew + // *HistoryEvent_ExecutionSuspended + // *HistoryEvent_ExecutionResumed + // *HistoryEvent_EntityOperationSignaled + // *HistoryEvent_EntityOperationCalled + // *HistoryEvent_EntityOperationCompleted + // *HistoryEvent_EntityOperationFailed + // *HistoryEvent_EntityLockRequested + // *HistoryEvent_EntityLockGranted + // *HistoryEvent_EntityUnlockSent + // *HistoryEvent_ExecutionRewound + EventType isHistoryEvent_EventType `protobuf_oneof:"eventType"` unknownFields protoimpl.UnknownFields - - OrchestrationStatus OrchestrationStatus `protobuf:"varint,1,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` - Result *wrappers.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` - Details *wrappers.StringValue `protobuf:"bytes,3,opt,name=details,proto3" json:"details,omitempty"` - NewVersion *wrappers.StringValue `protobuf:"bytes,4,opt,name=newVersion,proto3" json:"newVersion,omitempty"` - CarryoverEvents []*HistoryEvent `protobuf:"bytes,5,rep,name=carryoverEvents,proto3" json:"carryoverEvents,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,6,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *CompleteOrchestrationAction) Reset() { - *x = CompleteOrchestrationAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[31] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *HistoryEvent) Reset() { + *x = HistoryEvent{} + mi := &file_orchestrator_service_proto_msgTypes[34] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CompleteOrchestrationAction) String() string { +func (x *HistoryEvent) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CompleteOrchestrationAction) ProtoMessage() {} +func (*HistoryEvent) ProtoMessage() {} -func (x *CompleteOrchestrationAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[31] - if protoimpl.UnsafeEnabled && x != nil { +func (x *HistoryEvent) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[34] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2315,151 +2293,1922 @@ func (x *CompleteOrchestrationAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CompleteOrchestrationAction.ProtoReflect.Descriptor instead. -func (*CompleteOrchestrationAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{31} +// Deprecated: Use HistoryEvent.ProtoReflect.Descriptor instead. +func (*HistoryEvent) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{34} } -func (x *CompleteOrchestrationAction) GetOrchestrationStatus() OrchestrationStatus { +func (x *HistoryEvent) GetEventId() int32 { if x != nil { - return x.OrchestrationStatus + return x.EventId } - return OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING + return 0 } -func (x *CompleteOrchestrationAction) GetResult() *wrappers.StringValue { +func (x *HistoryEvent) GetTimestamp() *timestamppb.Timestamp { if x != nil { - return x.Result + return x.Timestamp } return nil } -func (x *CompleteOrchestrationAction) GetDetails() *wrappers.StringValue { +func (x *HistoryEvent) GetEventType() isHistoryEvent_EventType { if x != nil { - return x.Details + return x.EventType } return nil } -func (x *CompleteOrchestrationAction) GetNewVersion() *wrappers.StringValue { +func (x *HistoryEvent) GetExecutionStarted() *ExecutionStartedEvent { if x != nil { - return x.NewVersion + if x, ok := x.EventType.(*HistoryEvent_ExecutionStarted); ok { + return x.ExecutionStarted + } } return nil } -func (x *CompleteOrchestrationAction) GetCarryoverEvents() []*HistoryEvent { +func (x *HistoryEvent) GetExecutionCompleted() *ExecutionCompletedEvent { if x != nil { - return x.CarryoverEvents + if x, ok := x.EventType.(*HistoryEvent_ExecutionCompleted); ok { + return x.ExecutionCompleted + } } return nil } -func (x *CompleteOrchestrationAction) GetFailureDetails() *TaskFailureDetails { +func (x *HistoryEvent) GetExecutionTerminated() *ExecutionTerminatedEvent { if x != nil { - return x.FailureDetails + if x, ok := x.EventType.(*HistoryEvent_ExecutionTerminated); ok { + return x.ExecutionTerminated + } } return nil } -type TerminateOrchestrationAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Reason *wrappers.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` - Recurse bool `protobuf:"varint,3,opt,name=recurse,proto3" json:"recurse,omitempty"` +func (x *HistoryEvent) GetTaskScheduled() *TaskScheduledEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_TaskScheduled); ok { + return x.TaskScheduled + } + } + return nil } -func (x *TerminateOrchestrationAction) Reset() { - *x = TerminateOrchestrationAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[32] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *HistoryEvent) GetTaskCompleted() *TaskCompletedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_TaskCompleted); ok { + return x.TaskCompleted + } } + return nil } -func (x *TerminateOrchestrationAction) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *HistoryEvent) GetTaskFailed() *TaskFailedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_TaskFailed); ok { + return x.TaskFailed + } + } + return nil } -func (*TerminateOrchestrationAction) ProtoMessage() {} - -func (x *TerminateOrchestrationAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[32] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) +func (x *HistoryEvent) GetSubOrchestrationInstanceCreated() *SubOrchestrationInstanceCreatedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_SubOrchestrationInstanceCreated); ok { + return x.SubOrchestrationInstanceCreated } - return ms } - return mi.MessageOf(x) + return nil } -// Deprecated: Use TerminateOrchestrationAction.ProtoReflect.Descriptor instead. -func (*TerminateOrchestrationAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{32} +func (x *HistoryEvent) GetSubOrchestrationInstanceCompleted() *SubOrchestrationInstanceCompletedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_SubOrchestrationInstanceCompleted); ok { + return x.SubOrchestrationInstanceCompleted + } + } + return nil } -func (x *TerminateOrchestrationAction) GetInstanceId() string { +func (x *HistoryEvent) GetSubOrchestrationInstanceFailed() *SubOrchestrationInstanceFailedEvent { if x != nil { - return x.InstanceId + if x, ok := x.EventType.(*HistoryEvent_SubOrchestrationInstanceFailed); ok { + return x.SubOrchestrationInstanceFailed + } } - return "" + return nil } -func (x *TerminateOrchestrationAction) GetReason() *wrappers.StringValue { +func (x *HistoryEvent) GetTimerCreated() *TimerCreatedEvent { if x != nil { - return x.Reason + if x, ok := x.EventType.(*HistoryEvent_TimerCreated); ok { + return x.TimerCreated + } } return nil } -func (x *TerminateOrchestrationAction) GetRecurse() bool { +func (x *HistoryEvent) GetTimerFired() *TimerFiredEvent { if x != nil { - return x.Recurse + if x, ok := x.EventType.(*HistoryEvent_TimerFired); ok { + return x.TimerFired + } } - return false + return nil } -type OrchestratorAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - // Types that are assignable to OrchestratorActionType: - // - // *OrchestratorAction_ScheduleTask - // *OrchestratorAction_CreateSubOrchestration - // *OrchestratorAction_CreateTimer - // *OrchestratorAction_SendEvent - // *OrchestratorAction_CompleteOrchestration - // *OrchestratorAction_TerminateOrchestration - OrchestratorActionType isOrchestratorAction_OrchestratorActionType `protobuf_oneof:"orchestratorActionType"` +func (x *HistoryEvent) GetOrchestratorStarted() *OrchestratorStartedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_OrchestratorStarted); ok { + return x.OrchestratorStarted + } + } + return nil } -func (x *OrchestratorAction) Reset() { - *x = OrchestratorAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[33] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *HistoryEvent) GetOrchestratorCompleted() *OrchestratorCompletedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_OrchestratorCompleted); ok { + return x.OrchestratorCompleted + } } + return nil +} + +func (x *HistoryEvent) GetEventSent() *EventSentEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EventSent); ok { + return x.EventSent + } + } + return nil +} + +func (x *HistoryEvent) GetEventRaised() *EventRaisedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EventRaised); ok { + return x.EventRaised + } + } + return nil +} + +func (x *HistoryEvent) GetGenericEvent() *GenericEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_GenericEvent); ok { + return x.GenericEvent + } + } + return nil +} + +func (x *HistoryEvent) GetHistoryState() *HistoryStateEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_HistoryState); ok { + return x.HistoryState + } + } + return nil +} + +func (x *HistoryEvent) GetContinueAsNew() *ContinueAsNewEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_ContinueAsNew); ok { + return x.ContinueAsNew + } + } + return nil +} + +func (x *HistoryEvent) GetExecutionSuspended() *ExecutionSuspendedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_ExecutionSuspended); ok { + return x.ExecutionSuspended + } + } + return nil +} + +func (x *HistoryEvent) GetExecutionResumed() *ExecutionResumedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_ExecutionResumed); ok { + return x.ExecutionResumed + } + } + return nil +} + +func (x *HistoryEvent) GetEntityOperationSignaled() *EntityOperationSignaledEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityOperationSignaled); ok { + return x.EntityOperationSignaled + } + } + return nil +} + +func (x *HistoryEvent) GetEntityOperationCalled() *EntityOperationCalledEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityOperationCalled); ok { + return x.EntityOperationCalled + } + } + return nil +} + +func (x *HistoryEvent) GetEntityOperationCompleted() *EntityOperationCompletedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityOperationCompleted); ok { + return x.EntityOperationCompleted + } + } + return nil +} + +func (x *HistoryEvent) GetEntityOperationFailed() *EntityOperationFailedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityOperationFailed); ok { + return x.EntityOperationFailed + } + } + return nil +} + +func (x *HistoryEvent) GetEntityLockRequested() *EntityLockRequestedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityLockRequested); ok { + return x.EntityLockRequested + } + } + return nil +} + +func (x *HistoryEvent) GetEntityLockGranted() *EntityLockGrantedEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityLockGranted); ok { + return x.EntityLockGranted + } + } + return nil +} + +func (x *HistoryEvent) GetEntityUnlockSent() *EntityUnlockSentEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_EntityUnlockSent); ok { + return x.EntityUnlockSent + } + } + return nil +} + +func (x *HistoryEvent) GetExecutionRewound() *ExecutionRewoundEvent { + if x != nil { + if x, ok := x.EventType.(*HistoryEvent_ExecutionRewound); ok { + return x.ExecutionRewound + } + } + return nil +} + +type isHistoryEvent_EventType interface { + isHistoryEvent_EventType() +} + +type HistoryEvent_ExecutionStarted struct { + ExecutionStarted *ExecutionStartedEvent `protobuf:"bytes,3,opt,name=executionStarted,proto3,oneof"` +} + +type HistoryEvent_ExecutionCompleted struct { + ExecutionCompleted *ExecutionCompletedEvent `protobuf:"bytes,4,opt,name=executionCompleted,proto3,oneof"` +} + +type HistoryEvent_ExecutionTerminated struct { + ExecutionTerminated *ExecutionTerminatedEvent `protobuf:"bytes,5,opt,name=executionTerminated,proto3,oneof"` +} + +type HistoryEvent_TaskScheduled struct { + TaskScheduled *TaskScheduledEvent `protobuf:"bytes,6,opt,name=taskScheduled,proto3,oneof"` +} + +type HistoryEvent_TaskCompleted struct { + TaskCompleted *TaskCompletedEvent `protobuf:"bytes,7,opt,name=taskCompleted,proto3,oneof"` +} + +type HistoryEvent_TaskFailed struct { + TaskFailed *TaskFailedEvent `protobuf:"bytes,8,opt,name=taskFailed,proto3,oneof"` +} + +type HistoryEvent_SubOrchestrationInstanceCreated struct { + SubOrchestrationInstanceCreated *SubOrchestrationInstanceCreatedEvent `protobuf:"bytes,9,opt,name=subOrchestrationInstanceCreated,proto3,oneof"` +} + +type HistoryEvent_SubOrchestrationInstanceCompleted struct { + SubOrchestrationInstanceCompleted *SubOrchestrationInstanceCompletedEvent `protobuf:"bytes,10,opt,name=subOrchestrationInstanceCompleted,proto3,oneof"` +} + +type HistoryEvent_SubOrchestrationInstanceFailed struct { + SubOrchestrationInstanceFailed *SubOrchestrationInstanceFailedEvent `protobuf:"bytes,11,opt,name=subOrchestrationInstanceFailed,proto3,oneof"` +} + +type HistoryEvent_TimerCreated struct { + TimerCreated *TimerCreatedEvent `protobuf:"bytes,12,opt,name=timerCreated,proto3,oneof"` +} + +type HistoryEvent_TimerFired struct { + TimerFired *TimerFiredEvent `protobuf:"bytes,13,opt,name=timerFired,proto3,oneof"` +} + +type HistoryEvent_OrchestratorStarted struct { + OrchestratorStarted *OrchestratorStartedEvent `protobuf:"bytes,14,opt,name=orchestratorStarted,proto3,oneof"` +} + +type HistoryEvent_OrchestratorCompleted struct { + OrchestratorCompleted *OrchestratorCompletedEvent `protobuf:"bytes,15,opt,name=orchestratorCompleted,proto3,oneof"` +} + +type HistoryEvent_EventSent struct { + EventSent *EventSentEvent `protobuf:"bytes,16,opt,name=eventSent,proto3,oneof"` +} + +type HistoryEvent_EventRaised struct { + EventRaised *EventRaisedEvent `protobuf:"bytes,17,opt,name=eventRaised,proto3,oneof"` +} + +type HistoryEvent_GenericEvent struct { + GenericEvent *GenericEvent `protobuf:"bytes,18,opt,name=genericEvent,proto3,oneof"` +} + +type HistoryEvent_HistoryState struct { + HistoryState *HistoryStateEvent `protobuf:"bytes,19,opt,name=historyState,proto3,oneof"` +} + +type HistoryEvent_ContinueAsNew struct { + ContinueAsNew *ContinueAsNewEvent `protobuf:"bytes,20,opt,name=continueAsNew,proto3,oneof"` +} + +type HistoryEvent_ExecutionSuspended struct { + ExecutionSuspended *ExecutionSuspendedEvent `protobuf:"bytes,21,opt,name=executionSuspended,proto3,oneof"` +} + +type HistoryEvent_ExecutionResumed struct { + ExecutionResumed *ExecutionResumedEvent `protobuf:"bytes,22,opt,name=executionResumed,proto3,oneof"` +} + +type HistoryEvent_EntityOperationSignaled struct { + EntityOperationSignaled *EntityOperationSignaledEvent `protobuf:"bytes,23,opt,name=entityOperationSignaled,proto3,oneof"` +} + +type HistoryEvent_EntityOperationCalled struct { + EntityOperationCalled *EntityOperationCalledEvent `protobuf:"bytes,24,opt,name=entityOperationCalled,proto3,oneof"` +} + +type HistoryEvent_EntityOperationCompleted struct { + EntityOperationCompleted *EntityOperationCompletedEvent `protobuf:"bytes,25,opt,name=entityOperationCompleted,proto3,oneof"` +} + +type HistoryEvent_EntityOperationFailed struct { + EntityOperationFailed *EntityOperationFailedEvent `protobuf:"bytes,26,opt,name=entityOperationFailed,proto3,oneof"` +} + +type HistoryEvent_EntityLockRequested struct { + EntityLockRequested *EntityLockRequestedEvent `protobuf:"bytes,27,opt,name=entityLockRequested,proto3,oneof"` +} + +type HistoryEvent_EntityLockGranted struct { + EntityLockGranted *EntityLockGrantedEvent `protobuf:"bytes,28,opt,name=entityLockGranted,proto3,oneof"` +} + +type HistoryEvent_EntityUnlockSent struct { + EntityUnlockSent *EntityUnlockSentEvent `protobuf:"bytes,29,opt,name=entityUnlockSent,proto3,oneof"` +} + +type HistoryEvent_ExecutionRewound struct { + ExecutionRewound *ExecutionRewoundEvent `protobuf:"bytes,30,opt,name=executionRewound,proto3,oneof"` +} + +func (*HistoryEvent_ExecutionStarted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ExecutionCompleted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ExecutionTerminated) isHistoryEvent_EventType() {} + +func (*HistoryEvent_TaskScheduled) isHistoryEvent_EventType() {} + +func (*HistoryEvent_TaskCompleted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_TaskFailed) isHistoryEvent_EventType() {} + +func (*HistoryEvent_SubOrchestrationInstanceCreated) isHistoryEvent_EventType() {} + +func (*HistoryEvent_SubOrchestrationInstanceCompleted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_SubOrchestrationInstanceFailed) isHistoryEvent_EventType() {} + +func (*HistoryEvent_TimerCreated) isHistoryEvent_EventType() {} + +func (*HistoryEvent_TimerFired) isHistoryEvent_EventType() {} + +func (*HistoryEvent_OrchestratorStarted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_OrchestratorCompleted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EventSent) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EventRaised) isHistoryEvent_EventType() {} + +func (*HistoryEvent_GenericEvent) isHistoryEvent_EventType() {} + +func (*HistoryEvent_HistoryState) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ContinueAsNew) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ExecutionSuspended) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ExecutionResumed) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityOperationSignaled) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityOperationCalled) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityOperationCompleted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityOperationFailed) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityLockRequested) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityLockGranted) isHistoryEvent_EventType() {} + +func (*HistoryEvent_EntityUnlockSent) isHistoryEvent_EventType() {} + +func (*HistoryEvent_ExecutionRewound) isHistoryEvent_EventType() {} + +type ScheduleTaskAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + Tags map[string]string `protobuf:"bytes,4,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ParentTraceContext *TraceContext `protobuf:"bytes,5,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ScheduleTaskAction) Reset() { + *x = ScheduleTaskAction{} + mi := &file_orchestrator_service_proto_msgTypes[35] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ScheduleTaskAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ScheduleTaskAction) ProtoMessage() {} + +func (x *ScheduleTaskAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[35] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ScheduleTaskAction.ProtoReflect.Descriptor instead. +func (*ScheduleTaskAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{35} +} + +func (x *ScheduleTaskAction) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ScheduleTaskAction) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version + } + return nil +} + +func (x *ScheduleTaskAction) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input + } + return nil +} + +func (x *ScheduleTaskAction) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *ScheduleTaskAction) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil +} + +type CreateSubOrchestrationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,5,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + Tags map[string]string `protobuf:"bytes,6,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateSubOrchestrationAction) Reset() { + *x = CreateSubOrchestrationAction{} + mi := &file_orchestrator_service_proto_msgTypes[36] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateSubOrchestrationAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateSubOrchestrationAction) ProtoMessage() {} + +func (x *CreateSubOrchestrationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[36] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateSubOrchestrationAction.ProtoReflect.Descriptor instead. +func (*CreateSubOrchestrationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{36} +} + +func (x *CreateSubOrchestrationAction) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *CreateSubOrchestrationAction) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateSubOrchestrationAction) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version + } + return nil +} + +func (x *CreateSubOrchestrationAction) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input + } + return nil +} + +func (x *CreateSubOrchestrationAction) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil +} + +func (x *CreateSubOrchestrationAction) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} + +type CreateTimerAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + FireAt *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=fireAt,proto3" json:"fireAt,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateTimerAction) Reset() { + *x = CreateTimerAction{} + mi := &file_orchestrator_service_proto_msgTypes[37] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateTimerAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateTimerAction) ProtoMessage() {} + +func (x *CreateTimerAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[37] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateTimerAction.ProtoReflect.Descriptor instead. +func (*CreateTimerAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{37} +} + +func (x *CreateTimerAction) GetFireAt() *timestamppb.Timestamp { + if x != nil { + return x.FireAt + } + return nil +} + +type SendEventAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + Instance *OrchestrationInstance `protobuf:"bytes,1,opt,name=instance,proto3" json:"instance,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Data *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SendEventAction) Reset() { + *x = SendEventAction{} + mi := &file_orchestrator_service_proto_msgTypes[38] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SendEventAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SendEventAction) ProtoMessage() {} + +func (x *SendEventAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[38] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SendEventAction.ProtoReflect.Descriptor instead. +func (*SendEventAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{38} +} + +func (x *SendEventAction) GetInstance() *OrchestrationInstance { + if x != nil { + return x.Instance + } + return nil +} + +func (x *SendEventAction) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SendEventAction) GetData() *wrapperspb.StringValue { + if x != nil { + return x.Data + } + return nil +} + +type CompleteOrchestrationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + OrchestrationStatus OrchestrationStatus `protobuf:"varint,1,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` + Result *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=result,proto3" json:"result,omitempty"` + Details *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=details,proto3" json:"details,omitempty"` + NewVersion *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=newVersion,proto3" json:"newVersion,omitempty"` + CarryoverEvents []*HistoryEvent `protobuf:"bytes,5,rep,name=carryoverEvents,proto3" json:"carryoverEvents,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,6,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + Tags map[string]string `protobuf:"bytes,7,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CompleteOrchestrationAction) Reset() { + *x = CompleteOrchestrationAction{} + mi := &file_orchestrator_service_proto_msgTypes[39] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CompleteOrchestrationAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CompleteOrchestrationAction) ProtoMessage() {} + +func (x *CompleteOrchestrationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[39] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CompleteOrchestrationAction.ProtoReflect.Descriptor instead. +func (*CompleteOrchestrationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{39} +} + +func (x *CompleteOrchestrationAction) GetOrchestrationStatus() OrchestrationStatus { + if x != nil { + return x.OrchestrationStatus + } + return OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING +} + +func (x *CompleteOrchestrationAction) GetResult() *wrapperspb.StringValue { + if x != nil { + return x.Result + } + return nil +} + +func (x *CompleteOrchestrationAction) GetDetails() *wrapperspb.StringValue { + if x != nil { + return x.Details + } + return nil +} + +func (x *CompleteOrchestrationAction) GetNewVersion() *wrapperspb.StringValue { + if x != nil { + return x.NewVersion + } + return nil +} + +func (x *CompleteOrchestrationAction) GetCarryoverEvents() []*HistoryEvent { + if x != nil { + return x.CarryoverEvents + } + return nil +} + +func (x *CompleteOrchestrationAction) GetFailureDetails() *TaskFailureDetails { + if x != nil { + return x.FailureDetails + } + return nil +} + +func (x *CompleteOrchestrationAction) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} + +type TerminateOrchestrationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Reason *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + Recurse bool `protobuf:"varint,3,opt,name=recurse,proto3" json:"recurse,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateOrchestrationAction) Reset() { + *x = TerminateOrchestrationAction{} + mi := &file_orchestrator_service_proto_msgTypes[40] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateOrchestrationAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateOrchestrationAction) ProtoMessage() {} + +func (x *TerminateOrchestrationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[40] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateOrchestrationAction.ProtoReflect.Descriptor instead. +func (*TerminateOrchestrationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{40} +} + +func (x *TerminateOrchestrationAction) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *TerminateOrchestrationAction) GetReason() *wrapperspb.StringValue { + if x != nil { + return x.Reason + } + return nil +} + +func (x *TerminateOrchestrationAction) GetRecurse() bool { + if x != nil { + return x.Recurse + } + return false +} + +type SendEntityMessageAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to EntityMessageType: + // + // *SendEntityMessageAction_EntityOperationSignaled + // *SendEntityMessageAction_EntityOperationCalled + // *SendEntityMessageAction_EntityLockRequested + // *SendEntityMessageAction_EntityUnlockSent + EntityMessageType isSendEntityMessageAction_EntityMessageType `protobuf_oneof:"EntityMessageType"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SendEntityMessageAction) Reset() { + *x = SendEntityMessageAction{} + mi := &file_orchestrator_service_proto_msgTypes[41] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SendEntityMessageAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SendEntityMessageAction) ProtoMessage() {} + +func (x *SendEntityMessageAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[41] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SendEntityMessageAction.ProtoReflect.Descriptor instead. +func (*SendEntityMessageAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{41} +} + +func (x *SendEntityMessageAction) GetEntityMessageType() isSendEntityMessageAction_EntityMessageType { + if x != nil { + return x.EntityMessageType + } + return nil +} + +func (x *SendEntityMessageAction) GetEntityOperationSignaled() *EntityOperationSignaledEvent { + if x != nil { + if x, ok := x.EntityMessageType.(*SendEntityMessageAction_EntityOperationSignaled); ok { + return x.EntityOperationSignaled + } + } + return nil +} + +func (x *SendEntityMessageAction) GetEntityOperationCalled() *EntityOperationCalledEvent { + if x != nil { + if x, ok := x.EntityMessageType.(*SendEntityMessageAction_EntityOperationCalled); ok { + return x.EntityOperationCalled + } + } + return nil +} + +func (x *SendEntityMessageAction) GetEntityLockRequested() *EntityLockRequestedEvent { + if x != nil { + if x, ok := x.EntityMessageType.(*SendEntityMessageAction_EntityLockRequested); ok { + return x.EntityLockRequested + } + } + return nil +} + +func (x *SendEntityMessageAction) GetEntityUnlockSent() *EntityUnlockSentEvent { + if x != nil { + if x, ok := x.EntityMessageType.(*SendEntityMessageAction_EntityUnlockSent); ok { + return x.EntityUnlockSent + } + } + return nil +} + +type isSendEntityMessageAction_EntityMessageType interface { + isSendEntityMessageAction_EntityMessageType() +} + +type SendEntityMessageAction_EntityOperationSignaled struct { + EntityOperationSignaled *EntityOperationSignaledEvent `protobuf:"bytes,1,opt,name=entityOperationSignaled,proto3,oneof"` +} + +type SendEntityMessageAction_EntityOperationCalled struct { + EntityOperationCalled *EntityOperationCalledEvent `protobuf:"bytes,2,opt,name=entityOperationCalled,proto3,oneof"` +} + +type SendEntityMessageAction_EntityLockRequested struct { + EntityLockRequested *EntityLockRequestedEvent `protobuf:"bytes,3,opt,name=entityLockRequested,proto3,oneof"` +} + +type SendEntityMessageAction_EntityUnlockSent struct { + EntityUnlockSent *EntityUnlockSentEvent `protobuf:"bytes,4,opt,name=entityUnlockSent,proto3,oneof"` +} + +func (*SendEntityMessageAction_EntityOperationSignaled) isSendEntityMessageAction_EntityMessageType() { +} + +func (*SendEntityMessageAction_EntityOperationCalled) isSendEntityMessageAction_EntityMessageType() {} + +func (*SendEntityMessageAction_EntityLockRequested) isSendEntityMessageAction_EntityMessageType() {} + +func (*SendEntityMessageAction_EntityUnlockSent) isSendEntityMessageAction_EntityMessageType() {} + +type RewindOrchestrationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + NewHistory []*HistoryEvent `protobuf:"bytes,1,rep,name=newHistory,proto3" json:"newHistory,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RewindOrchestrationAction) Reset() { + *x = RewindOrchestrationAction{} + mi := &file_orchestrator_service_proto_msgTypes[42] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RewindOrchestrationAction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RewindOrchestrationAction) ProtoMessage() {} + +func (x *RewindOrchestrationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[42] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RewindOrchestrationAction.ProtoReflect.Descriptor instead. +func (*RewindOrchestrationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{42} +} + +func (x *RewindOrchestrationAction) GetNewHistory() []*HistoryEvent { + if x != nil { + return x.NewHistory + } + return nil +} + +type OrchestratorAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // Types that are valid to be assigned to OrchestratorActionType: + // + // *OrchestratorAction_ScheduleTask + // *OrchestratorAction_CreateSubOrchestration + // *OrchestratorAction_CreateTimer + // *OrchestratorAction_SendEvent + // *OrchestratorAction_CompleteOrchestration + // *OrchestratorAction_TerminateOrchestration + // *OrchestratorAction_SendEntityMessage + // *OrchestratorAction_RewindOrchestration + OrchestratorActionType isOrchestratorAction_OrchestratorActionType `protobuf_oneof:"orchestratorActionType"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestratorAction) Reset() { + *x = OrchestratorAction{} + mi := &file_orchestrator_service_proto_msgTypes[43] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *OrchestratorAction) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestratorAction) ProtoMessage() {} +func (*OrchestratorAction) ProtoMessage() {} + +func (x *OrchestratorAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[43] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrchestratorAction.ProtoReflect.Descriptor instead. +func (*OrchestratorAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{43} +} + +func (x *OrchestratorAction) GetId() int32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *OrchestratorAction) GetOrchestratorActionType() isOrchestratorAction_OrchestratorActionType { + if x != nil { + return x.OrchestratorActionType + } + return nil +} + +func (x *OrchestratorAction) GetScheduleTask() *ScheduleTaskAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_ScheduleTask); ok { + return x.ScheduleTask + } + } + return nil +} + +func (x *OrchestratorAction) GetCreateSubOrchestration() *CreateSubOrchestrationAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_CreateSubOrchestration); ok { + return x.CreateSubOrchestration + } + } + return nil +} + +func (x *OrchestratorAction) GetCreateTimer() *CreateTimerAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_CreateTimer); ok { + return x.CreateTimer + } + } + return nil +} + +func (x *OrchestratorAction) GetSendEvent() *SendEventAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_SendEvent); ok { + return x.SendEvent + } + } + return nil +} + +func (x *OrchestratorAction) GetCompleteOrchestration() *CompleteOrchestrationAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_CompleteOrchestration); ok { + return x.CompleteOrchestration + } + } + return nil +} + +func (x *OrchestratorAction) GetTerminateOrchestration() *TerminateOrchestrationAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_TerminateOrchestration); ok { + return x.TerminateOrchestration + } + } + return nil +} + +func (x *OrchestratorAction) GetSendEntityMessage() *SendEntityMessageAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_SendEntityMessage); ok { + return x.SendEntityMessage + } + } + return nil +} + +func (x *OrchestratorAction) GetRewindOrchestration() *RewindOrchestrationAction { + if x != nil { + if x, ok := x.OrchestratorActionType.(*OrchestratorAction_RewindOrchestration); ok { + return x.RewindOrchestration + } + } + return nil +} + +type isOrchestratorAction_OrchestratorActionType interface { + isOrchestratorAction_OrchestratorActionType() +} + +type OrchestratorAction_ScheduleTask struct { + ScheduleTask *ScheduleTaskAction `protobuf:"bytes,2,opt,name=scheduleTask,proto3,oneof"` +} + +type OrchestratorAction_CreateSubOrchestration struct { + CreateSubOrchestration *CreateSubOrchestrationAction `protobuf:"bytes,3,opt,name=createSubOrchestration,proto3,oneof"` +} + +type OrchestratorAction_CreateTimer struct { + CreateTimer *CreateTimerAction `protobuf:"bytes,4,opt,name=createTimer,proto3,oneof"` +} + +type OrchestratorAction_SendEvent struct { + SendEvent *SendEventAction `protobuf:"bytes,5,opt,name=sendEvent,proto3,oneof"` +} + +type OrchestratorAction_CompleteOrchestration struct { + CompleteOrchestration *CompleteOrchestrationAction `protobuf:"bytes,6,opt,name=completeOrchestration,proto3,oneof"` +} + +type OrchestratorAction_TerminateOrchestration struct { + TerminateOrchestration *TerminateOrchestrationAction `protobuf:"bytes,7,opt,name=terminateOrchestration,proto3,oneof"` +} + +type OrchestratorAction_SendEntityMessage struct { + SendEntityMessage *SendEntityMessageAction `protobuf:"bytes,8,opt,name=sendEntityMessage,proto3,oneof"` +} + +type OrchestratorAction_RewindOrchestration struct { + RewindOrchestration *RewindOrchestrationAction `protobuf:"bytes,9,opt,name=rewindOrchestration,proto3,oneof"` +} + +func (*OrchestratorAction_ScheduleTask) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_CreateSubOrchestration) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_CreateTimer) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_SendEvent) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_CompleteOrchestration) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_TerminateOrchestration) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_SendEntityMessage) isOrchestratorAction_OrchestratorActionType() {} + +func (*OrchestratorAction_RewindOrchestration) isOrchestratorAction_OrchestratorActionType() {} + +type OrchestrationTraceContext struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpanID *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=spanID,proto3" json:"spanID,omitempty"` + SpanStartTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=spanStartTime,proto3" json:"spanStartTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestrationTraceContext) Reset() { + *x = OrchestrationTraceContext{} + mi := &file_orchestrator_service_proto_msgTypes[44] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrchestrationTraceContext) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrchestrationTraceContext) ProtoMessage() {} + +func (x *OrchestrationTraceContext) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[44] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrchestrationTraceContext.ProtoReflect.Descriptor instead. +func (*OrchestrationTraceContext) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{44} +} + +func (x *OrchestrationTraceContext) GetSpanID() *wrapperspb.StringValue { + if x != nil { + return x.SpanID + } + return nil +} + +func (x *OrchestrationTraceContext) GetSpanStartTime() *timestamppb.Timestamp { + if x != nil { + return x.SpanStartTime + } + return nil +} + +type OrchestratorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + ExecutionId *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` + PastEvents []*HistoryEvent `protobuf:"bytes,3,rep,name=pastEvents,proto3" json:"pastEvents,omitempty"` + NewEvents []*HistoryEvent `protobuf:"bytes,4,rep,name=newEvents,proto3" json:"newEvents,omitempty"` + EntityParameters *OrchestratorEntityParameters `protobuf:"bytes,5,opt,name=entityParameters,proto3" json:"entityParameters,omitempty"` + RequiresHistoryStreaming bool `protobuf:"varint,6,opt,name=requiresHistoryStreaming,proto3" json:"requiresHistoryStreaming,omitempty"` + Properties map[string]*structpb.Value `protobuf:"bytes,7,rep,name=properties,proto3" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + OrchestrationTraceContext *OrchestrationTraceContext `protobuf:"bytes,8,opt,name=orchestrationTraceContext,proto3" json:"orchestrationTraceContext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestratorRequest) Reset() { + *x = OrchestratorRequest{} + mi := &file_orchestrator_service_proto_msgTypes[45] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrchestratorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrchestratorRequest) ProtoMessage() {} + +func (x *OrchestratorRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[45] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrchestratorRequest.ProtoReflect.Descriptor instead. +func (*OrchestratorRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{45} +} + +func (x *OrchestratorRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *OrchestratorRequest) GetExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ExecutionId + } + return nil +} + +func (x *OrchestratorRequest) GetPastEvents() []*HistoryEvent { + if x != nil { + return x.PastEvents + } + return nil +} + +func (x *OrchestratorRequest) GetNewEvents() []*HistoryEvent { + if x != nil { + return x.NewEvents + } + return nil +} + +func (x *OrchestratorRequest) GetEntityParameters() *OrchestratorEntityParameters { + if x != nil { + return x.EntityParameters + } + return nil +} + +func (x *OrchestratorRequest) GetRequiresHistoryStreaming() bool { + if x != nil { + return x.RequiresHistoryStreaming + } + return false +} + +func (x *OrchestratorRequest) GetProperties() map[string]*structpb.Value { + if x != nil { + return x.Properties + } + return nil +} + +func (x *OrchestratorRequest) GetOrchestrationTraceContext() *OrchestrationTraceContext { + if x != nil { + return x.OrchestrationTraceContext + } + return nil +} + +type OrchestratorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Actions []*OrchestratorAction `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty"` + CustomStatus *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=customStatus,proto3" json:"customStatus,omitempty"` + CompletionToken string `protobuf:"bytes,4,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + // The number of work item events that were processed by the orchestrator. + // This field is optional. If not set, the service should assume that the orchestrator processed all events. + NumEventsProcessed *wrapperspb.Int32Value `protobuf:"bytes,5,opt,name=numEventsProcessed,proto3" json:"numEventsProcessed,omitempty"` + OrchestrationTraceContext *OrchestrationTraceContext `protobuf:"bytes,6,opt,name=orchestrationTraceContext,proto3" json:"orchestrationTraceContext,omitempty"` + // Whether or not a history is required to complete the original OrchestratorRequest and none was provided. + RequiresHistory bool `protobuf:"varint,7,opt,name=requiresHistory,proto3" json:"requiresHistory,omitempty"` + // True if this is a partial (chunked) completion. The backend must keep the work item open until the final chunk (isPartial=false). + // + // Deprecated: Marked as deprecated in orchestrator_service.proto. + IsPartial bool `protobuf:"varint,8,opt,name=isPartial,proto3" json:"isPartial,omitempty"` + // Zero-based position of the current chunk within a chunked completion sequence. + // This field is omitted for non-chunked completions. + // + // Deprecated: Marked as deprecated in orchestrator_service.proto. + ChunkIndex *wrapperspb.Int32Value `protobuf:"bytes,9,opt,name=chunkIndex,proto3" json:"chunkIndex,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestratorResponse) Reset() { + *x = OrchestratorResponse{} + mi := &file_orchestrator_service_proto_msgTypes[46] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrchestratorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrchestratorResponse) ProtoMessage() {} + +func (x *OrchestratorResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[46] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrchestratorResponse.ProtoReflect.Descriptor instead. +func (*OrchestratorResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{46} +} + +func (x *OrchestratorResponse) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *OrchestratorResponse) GetActions() []*OrchestratorAction { + if x != nil { + return x.Actions + } + return nil +} + +func (x *OrchestratorResponse) GetCustomStatus() *wrapperspb.StringValue { + if x != nil { + return x.CustomStatus + } + return nil +} + +func (x *OrchestratorResponse) GetCompletionToken() string { + if x != nil { + return x.CompletionToken + } + return "" +} + +func (x *OrchestratorResponse) GetNumEventsProcessed() *wrapperspb.Int32Value { + if x != nil { + return x.NumEventsProcessed + } + return nil +} + +func (x *OrchestratorResponse) GetOrchestrationTraceContext() *OrchestrationTraceContext { + if x != nil { + return x.OrchestrationTraceContext + } + return nil +} + +func (x *OrchestratorResponse) GetRequiresHistory() bool { + if x != nil { + return x.RequiresHistory + } + return false +} + +// Deprecated: Marked as deprecated in orchestrator_service.proto. +func (x *OrchestratorResponse) GetIsPartial() bool { + if x != nil { + return x.IsPartial + } + return false +} + +// Deprecated: Marked as deprecated in orchestrator_service.proto. +func (x *OrchestratorResponse) GetChunkIndex() *wrapperspb.Int32Value { + if x != nil { + return x.ChunkIndex + } + return nil +} + +type CreateInstanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + ScheduledStartTimestamp *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` + OrchestrationIdReusePolicy *OrchestrationIdReusePolicy `protobuf:"bytes,6,opt,name=orchestrationIdReusePolicy,proto3" json:"orchestrationIdReusePolicy,omitempty"` + ExecutionId *wrapperspb.StringValue `protobuf:"bytes,7,opt,name=executionId,proto3" json:"executionId,omitempty"` + Tags map[string]string `protobuf:"bytes,8,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + ParentTraceContext *TraceContext `protobuf:"bytes,9,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + RequestTime *timestamppb.Timestamp `protobuf:"bytes,10,opt,name=requestTime,proto3" json:"requestTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateInstanceRequest) Reset() { + *x = CreateInstanceRequest{} + mi := &file_orchestrator_service_proto_msgTypes[47] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateInstanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateInstanceRequest) ProtoMessage() {} + +func (x *CreateInstanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[47] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateInstanceRequest.ProtoReflect.Descriptor instead. +func (*CreateInstanceRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{47} +} + +func (x *CreateInstanceRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *CreateInstanceRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *CreateInstanceRequest) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version + } + return nil +} + +func (x *CreateInstanceRequest) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input + } + return nil +} + +func (x *CreateInstanceRequest) GetScheduledStartTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.ScheduledStartTimestamp + } + return nil +} + +func (x *CreateInstanceRequest) GetOrchestrationIdReusePolicy() *OrchestrationIdReusePolicy { + if x != nil { + return x.OrchestrationIdReusePolicy + } + return nil +} + +func (x *CreateInstanceRequest) GetExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ExecutionId + } + return nil +} + +func (x *CreateInstanceRequest) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil +} + +func (x *CreateInstanceRequest) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil +} + +func (x *CreateInstanceRequest) GetRequestTime() *timestamppb.Timestamp { + if x != nil { + return x.RequestTime + } + return nil +} + +type OrchestrationIdReusePolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReplaceableStatus []OrchestrationStatus `protobuf:"varint,1,rep,packed,name=replaceableStatus,proto3,enum=OrchestrationStatus" json:"replaceableStatus,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestrationIdReusePolicy) Reset() { + *x = OrchestrationIdReusePolicy{} + mi := &file_orchestrator_service_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrchestrationIdReusePolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrchestrationIdReusePolicy) ProtoMessage() {} + +func (x *OrchestrationIdReusePolicy) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[48] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use OrchestrationIdReusePolicy.ProtoReflect.Descriptor instead. +func (*OrchestrationIdReusePolicy) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{48} +} + +func (x *OrchestrationIdReusePolicy) GetReplaceableStatus() []OrchestrationStatus { + if x != nil { + return x.ReplaceableStatus + } + return nil +} + +type CreateInstanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateInstanceResponse) Reset() { + *x = CreateInstanceResponse{} + mi := &file_orchestrator_service_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateInstanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateInstanceResponse) ProtoMessage() {} + +func (x *CreateInstanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[49] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateInstanceResponse.ProtoReflect.Descriptor instead. +func (*CreateInstanceResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{49} +} + +func (x *CreateInstanceResponse) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +type GetInstanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + GetInputsAndOutputs bool `protobuf:"varint,2,opt,name=getInputsAndOutputs,proto3" json:"getInputsAndOutputs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetInstanceRequest) Reset() { + *x = GetInstanceRequest{} + mi := &file_orchestrator_service_proto_msgTypes[50] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetInstanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetInstanceRequest) ProtoMessage() {} + +func (x *GetInstanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[50] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetInstanceRequest.ProtoReflect.Descriptor instead. +func (*GetInstanceRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{50} +} + +func (x *GetInstanceRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *GetInstanceRequest) GetGetInputsAndOutputs() bool { + if x != nil { + return x.GetInputsAndOutputs + } + return false +} + +type GetInstanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` + OrchestrationState *OrchestrationState `protobuf:"bytes,2,opt,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetInstanceResponse) Reset() { + *x = GetInstanceResponse{} + mi := &file_orchestrator_service_proto_msgTypes[51] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetInstanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetInstanceResponse) ProtoMessage() {} + +func (x *GetInstanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[51] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetInstanceResponse.ProtoReflect.Descriptor instead. +func (*GetInstanceResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{51} +} + +func (x *GetInstanceResponse) GetExists() bool { + if x != nil { + return x.Exists + } + return false +} + +func (x *GetInstanceResponse) GetOrchestrationState() *OrchestrationState { + if x != nil { + return x.OrchestrationState + } + return nil +} + +type RewindInstanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Reason *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RewindInstanceRequest) Reset() { + *x = RewindInstanceRequest{} + mi := &file_orchestrator_service_proto_msgTypes[52] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RewindInstanceRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RewindInstanceRequest) ProtoMessage() {} + +func (x *RewindInstanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[52] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RewindInstanceRequest.ProtoReflect.Descriptor instead. +func (*RewindInstanceRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{52} +} + +func (x *RewindInstanceRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *RewindInstanceRequest) GetReason() *wrapperspb.StringValue { + if x != nil { + return x.Reason + } + return nil +} + +type RewindInstanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RewindInstanceResponse) Reset() { + *x = RewindInstanceResponse{} + mi := &file_orchestrator_service_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RewindInstanceResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RewindInstanceResponse) ProtoMessage() {} + +func (x *RewindInstanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[53] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RewindInstanceResponse.ProtoReflect.Descriptor instead. +func (*RewindInstanceResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{53} +} + +type OrchestrationState struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + OrchestrationStatus OrchestrationStatus `protobuf:"varint,4,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` + ScheduledStartTimestamp *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` + CreatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=createdTimestamp,proto3" json:"createdTimestamp,omitempty"` + LastUpdatedTimestamp *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=lastUpdatedTimestamp,proto3" json:"lastUpdatedTimestamp,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,8,opt,name=input,proto3" json:"input,omitempty"` + Output *wrapperspb.StringValue `protobuf:"bytes,9,opt,name=output,proto3" json:"output,omitempty"` + CustomStatus *wrapperspb.StringValue `protobuf:"bytes,10,opt,name=customStatus,proto3" json:"customStatus,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,11,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + ExecutionId *wrapperspb.StringValue `protobuf:"bytes,12,opt,name=executionId,proto3" json:"executionId,omitempty"` + CompletedTimestamp *timestamppb.Timestamp `protobuf:"bytes,13,opt,name=completedTimestamp,proto3" json:"completedTimestamp,omitempty"` + ParentInstanceId *wrapperspb.StringValue `protobuf:"bytes,14,opt,name=parentInstanceId,proto3" json:"parentInstanceId,omitempty"` + Tags map[string]string `protobuf:"bytes,15,rep,name=tags,proto3" json:"tags,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *OrchestrationState) Reset() { + *x = OrchestrationState{} + mi := &file_orchestrator_service_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *OrchestrationState) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*OrchestrationState) ProtoMessage() {} -func (x *OrchestratorAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[33] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OrchestrationState) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[54] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2469,137 +4218,332 @@ func (x *OrchestratorAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestratorAction.ProtoReflect.Descriptor instead. -func (*OrchestratorAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{33} +// Deprecated: Use OrchestrationState.ProtoReflect.Descriptor instead. +func (*OrchestrationState) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{54} } -func (x *OrchestratorAction) GetId() int32 { +func (x *OrchestrationState) GetInstanceId() string { if x != nil { - return x.Id + return x.InstanceId } - return 0 + return "" +} + +func (x *OrchestrationState) GetName() string { + if x != nil { + return x.Name + } + return "" } -func (m *OrchestratorAction) GetOrchestratorActionType() isOrchestratorAction_OrchestratorActionType { - if m != nil { - return m.OrchestratorActionType +func (x *OrchestrationState) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version } return nil } -func (x *OrchestratorAction) GetScheduleTask() *ScheduleTaskAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_ScheduleTask); ok { - return x.ScheduleTask +func (x *OrchestrationState) GetOrchestrationStatus() OrchestrationStatus { + if x != nil { + return x.OrchestrationStatus + } + return OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING +} + +func (x *OrchestrationState) GetScheduledStartTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.ScheduledStartTimestamp } return nil } -func (x *OrchestratorAction) GetCreateSubOrchestration() *CreateSubOrchestrationAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_CreateSubOrchestration); ok { - return x.CreateSubOrchestration +func (x *OrchestrationState) GetCreatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.CreatedTimestamp } return nil } -func (x *OrchestratorAction) GetCreateTimer() *CreateTimerAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_CreateTimer); ok { - return x.CreateTimer +func (x *OrchestrationState) GetLastUpdatedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.LastUpdatedTimestamp } return nil } -func (x *OrchestratorAction) GetSendEvent() *SendEventAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_SendEvent); ok { - return x.SendEvent +func (x *OrchestrationState) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input } return nil } -func (x *OrchestratorAction) GetCompleteOrchestration() *CompleteOrchestrationAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_CompleteOrchestration); ok { - return x.CompleteOrchestration +func (x *OrchestrationState) GetOutput() *wrapperspb.StringValue { + if x != nil { + return x.Output } return nil } -func (x *OrchestratorAction) GetTerminateOrchestration() *TerminateOrchestrationAction { - if x, ok := x.GetOrchestratorActionType().(*OrchestratorAction_TerminateOrchestration); ok { - return x.TerminateOrchestration +func (x *OrchestrationState) GetCustomStatus() *wrapperspb.StringValue { + if x != nil { + return x.CustomStatus } return nil } -type isOrchestratorAction_OrchestratorActionType interface { - isOrchestratorAction_OrchestratorActionType() +func (x *OrchestrationState) GetFailureDetails() *TaskFailureDetails { + if x != nil { + return x.FailureDetails + } + return nil } -type OrchestratorAction_ScheduleTask struct { - ScheduleTask *ScheduleTaskAction `protobuf:"bytes,2,opt,name=scheduleTask,proto3,oneof"` +func (x *OrchestrationState) GetExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ExecutionId + } + return nil } -type OrchestratorAction_CreateSubOrchestration struct { - CreateSubOrchestration *CreateSubOrchestrationAction `protobuf:"bytes,3,opt,name=createSubOrchestration,proto3,oneof"` +func (x *OrchestrationState) GetCompletedTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.CompletedTimestamp + } + return nil } -type OrchestratorAction_CreateTimer struct { - CreateTimer *CreateTimerAction `protobuf:"bytes,4,opt,name=createTimer,proto3,oneof"` +func (x *OrchestrationState) GetParentInstanceId() *wrapperspb.StringValue { + if x != nil { + return x.ParentInstanceId + } + return nil } -type OrchestratorAction_SendEvent struct { - SendEvent *SendEventAction `protobuf:"bytes,5,opt,name=sendEvent,proto3,oneof"` +func (x *OrchestrationState) GetTags() map[string]string { + if x != nil { + return x.Tags + } + return nil } -type OrchestratorAction_CompleteOrchestration struct { - CompleteOrchestration *CompleteOrchestrationAction `protobuf:"bytes,6,opt,name=completeOrchestration,proto3,oneof"` +type RaiseEventRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -type OrchestratorAction_TerminateOrchestration struct { - TerminateOrchestration *TerminateOrchestrationAction `protobuf:"bytes,7,opt,name=terminateOrchestration,proto3,oneof"` +func (x *RaiseEventRequest) Reset() { + *x = RaiseEventRequest{} + mi := &file_orchestrator_service_proto_msgTypes[55] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaiseEventRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaiseEventRequest) ProtoMessage() {} + +func (x *RaiseEventRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[55] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaiseEventRequest.ProtoReflect.Descriptor instead. +func (*RaiseEventRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{55} +} + +func (x *RaiseEventRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *RaiseEventRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *RaiseEventRequest) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input + } + return nil +} + +type RaiseEventResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RaiseEventResponse) Reset() { + *x = RaiseEventResponse{} + mi := &file_orchestrator_service_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RaiseEventResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RaiseEventResponse) ProtoMessage() {} + +func (x *RaiseEventResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RaiseEventResponse.ProtoReflect.Descriptor instead. +func (*RaiseEventResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{56} +} + +type TerminateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Output *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateRequest) Reset() { + *x = TerminateRequest{} + mi := &file_orchestrator_service_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateRequest) ProtoMessage() {} + +func (x *TerminateRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. +func (*TerminateRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{57} +} + +func (x *TerminateRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *TerminateRequest) GetOutput() *wrapperspb.StringValue { + if x != nil { + return x.Output + } + return nil +} + +func (x *TerminateRequest) GetRecursive() bool { + if x != nil { + return x.Recursive + } + return false +} + +type TerminateResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TerminateResponse) Reset() { + *x = TerminateResponse{} + mi := &file_orchestrator_service_proto_msgTypes[58] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TerminateResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TerminateResponse) ProtoMessage() {} + +func (x *TerminateResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[58] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (*OrchestratorAction_ScheduleTask) isOrchestratorAction_OrchestratorActionType() {} - -func (*OrchestratorAction_CreateSubOrchestration) isOrchestratorAction_OrchestratorActionType() {} - -func (*OrchestratorAction_CreateTimer) isOrchestratorAction_OrchestratorActionType() {} - -func (*OrchestratorAction_SendEvent) isOrchestratorAction_OrchestratorActionType() {} - -func (*OrchestratorAction_CompleteOrchestration) isOrchestratorAction_OrchestratorActionType() {} - -func (*OrchestratorAction_TerminateOrchestration) isOrchestratorAction_OrchestratorActionType() {} +// Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. +func (*TerminateResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{58} +} -type OrchestratorRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type SuspendRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Reason *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - ExecutionId *wrappers.StringValue `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` - PastEvents []*HistoryEvent `protobuf:"bytes,3,rep,name=pastEvents,proto3" json:"pastEvents,omitempty"` - NewEvents []*HistoryEvent `protobuf:"bytes,4,rep,name=newEvents,proto3" json:"newEvents,omitempty"` - EntityParameters *OrchestratorEntityParameters `protobuf:"bytes,5,opt,name=entityParameters,proto3" json:"entityParameters,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *OrchestratorRequest) Reset() { - *x = OrchestratorRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[34] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *SuspendRequest) Reset() { + *x = SuspendRequest{} + mi := &file_orchestrator_service_proto_msgTypes[59] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OrchestratorRequest) String() string { +func (x *SuspendRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestratorRequest) ProtoMessage() {} +func (*SuspendRequest) ProtoMessage() {} -func (x *OrchestratorRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[34] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SuspendRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[59] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2609,74 +4553,85 @@ func (x *OrchestratorRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestratorRequest.ProtoReflect.Descriptor instead. -func (*OrchestratorRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{34} +// Deprecated: Use SuspendRequest.ProtoReflect.Descriptor instead. +func (*SuspendRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{59} } -func (x *OrchestratorRequest) GetInstanceId() string { +func (x *SuspendRequest) GetInstanceId() string { if x != nil { return x.InstanceId } return "" } -func (x *OrchestratorRequest) GetExecutionId() *wrappers.StringValue { +func (x *SuspendRequest) GetReason() *wrapperspb.StringValue { if x != nil { - return x.ExecutionId + return x.Reason } return nil } -func (x *OrchestratorRequest) GetPastEvents() []*HistoryEvent { - if x != nil { - return x.PastEvents - } - return nil +type SuspendResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OrchestratorRequest) GetNewEvents() []*HistoryEvent { - if x != nil { - return x.NewEvents - } - return nil +func (x *SuspendResponse) Reset() { + *x = SuspendResponse{} + mi := &file_orchestrator_service_proto_msgTypes[60] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OrchestratorRequest) GetEntityParameters() *OrchestratorEntityParameters { +func (x *SuspendResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendResponse) ProtoMessage() {} + +func (x *SuspendResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[60] if x != nil { - return x.EntityParameters + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -type OrchestratorResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +// Deprecated: Use SuspendResponse.ProtoReflect.Descriptor instead. +func (*SuspendResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{60} +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Actions []*OrchestratorAction `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty"` - CustomStatus *wrappers.StringValue `protobuf:"bytes,3,opt,name=customStatus,proto3" json:"customStatus,omitempty"` +type ResumeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Reason *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OrchestratorResponse) Reset() { - *x = OrchestratorResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[35] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *ResumeRequest) Reset() { + *x = ResumeRequest{} + mi := &file_orchestrator_service_proto_msgTypes[61] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OrchestratorResponse) String() string { +func (x *ResumeRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestratorResponse) ProtoMessage() {} +func (*ResumeRequest) ProtoMessage() {} -func (x *OrchestratorResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[35] - if protoimpl.UnsafeEnabled && x != nil { +func (x *ResumeRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[61] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2686,63 +4641,47 @@ func (x *OrchestratorResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestratorResponse.ProtoReflect.Descriptor instead. -func (*OrchestratorResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{35} +// Deprecated: Use ResumeRequest.ProtoReflect.Descriptor instead. +func (*ResumeRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{61} } -func (x *OrchestratorResponse) GetInstanceId() string { +func (x *ResumeRequest) GetInstanceId() string { if x != nil { return x.InstanceId } return "" } -func (x *OrchestratorResponse) GetActions() []*OrchestratorAction { - if x != nil { - return x.Actions - } - return nil -} - -func (x *OrchestratorResponse) GetCustomStatus() *wrappers.StringValue { +func (x *ResumeRequest) GetReason() *wrapperspb.StringValue { if x != nil { - return x.CustomStatus + return x.Reason } return nil } -type CreateInstanceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type ResumeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` - ScheduledStartTimestamp *timestamp.Timestamp `protobuf:"bytes,5,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` - OrchestrationIdReusePolicy *OrchestrationIdReusePolicy `protobuf:"bytes,6,opt,name=orchestrationIdReusePolicy,proto3" json:"orchestrationIdReusePolicy,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *CreateInstanceRequest) Reset() { - *x = CreateInstanceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[36] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *ResumeResponse) Reset() { + *x = ResumeResponse{} + mi := &file_orchestrator_service_proto_msgTypes[62] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateInstanceRequest) String() string { +func (x *ResumeResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateInstanceRequest) ProtoMessage() {} +func (*ResumeResponse) ProtoMessage() {} -func (x *CreateInstanceRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[36] - if protoimpl.UnsafeEnabled && x != nil { +func (x *ResumeResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[62] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2752,80 +4691,85 @@ func (x *CreateInstanceRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateInstanceRequest.ProtoReflect.Descriptor instead. -func (*CreateInstanceRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{36} +// Deprecated: Use ResumeResponse.ProtoReflect.Descriptor instead. +func (*ResumeResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{62} } -func (x *CreateInstanceRequest) GetInstanceId() string { - if x != nil { - return x.InstanceId - } - return "" +type QueryInstancesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Query *InstanceQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateInstanceRequest) GetName() string { - if x != nil { - return x.Name - } - return "" +func (x *QueryInstancesRequest) Reset() { + *x = QueryInstancesRequest{} + mi := &file_orchestrator_service_proto_msgTypes[63] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateInstanceRequest) GetVersion() *wrappers.StringValue { - if x != nil { - return x.Version - } - return nil +func (x *QueryInstancesRequest) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *CreateInstanceRequest) GetInput() *wrappers.StringValue { +func (*QueryInstancesRequest) ProtoMessage() {} + +func (x *QueryInstancesRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[63] if x != nil { - return x.Input + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -func (x *CreateInstanceRequest) GetScheduledStartTimestamp() *timestamp.Timestamp { - if x != nil { - return x.ScheduledStartTimestamp - } - return nil +// Deprecated: Use QueryInstancesRequest.ProtoReflect.Descriptor instead. +func (*QueryInstancesRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{63} } -func (x *CreateInstanceRequest) GetOrchestrationIdReusePolicy() *OrchestrationIdReusePolicy { +func (x *QueryInstancesRequest) GetQuery() *InstanceQuery { if x != nil { - return x.OrchestrationIdReusePolicy + return x.Query } return nil } -type OrchestrationIdReusePolicy struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - OperationStatus []OrchestrationStatus `protobuf:"varint,1,rep,packed,name=operationStatus,proto3,enum=OrchestrationStatus" json:"operationStatus,omitempty"` - Action CreateOrchestrationAction `protobuf:"varint,2,opt,name=action,proto3,enum=CreateOrchestrationAction" json:"action,omitempty"` +type InstanceQuery struct { + state protoimpl.MessageState `protogen:"open.v1"` + RuntimeStatus []OrchestrationStatus `protobuf:"varint,1,rep,packed,name=runtimeStatus,proto3,enum=OrchestrationStatus" json:"runtimeStatus,omitempty"` + CreatedTimeFrom *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=createdTimeFrom,proto3" json:"createdTimeFrom,omitempty"` + CreatedTimeTo *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=createdTimeTo,proto3" json:"createdTimeTo,omitempty"` + TaskHubNames []*wrapperspb.StringValue `protobuf:"bytes,4,rep,name=taskHubNames,proto3" json:"taskHubNames,omitempty"` + MaxInstanceCount int32 `protobuf:"varint,5,opt,name=maxInstanceCount,proto3" json:"maxInstanceCount,omitempty"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,6,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + InstanceIdPrefix *wrapperspb.StringValue `protobuf:"bytes,7,opt,name=instanceIdPrefix,proto3" json:"instanceIdPrefix,omitempty"` + FetchInputsAndOutputs bool `protobuf:"varint,8,opt,name=fetchInputsAndOutputs,proto3" json:"fetchInputsAndOutputs,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OrchestrationIdReusePolicy) Reset() { - *x = OrchestrationIdReusePolicy{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[37] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *InstanceQuery) Reset() { + *x = InstanceQuery{} + mi := &file_orchestrator_service_proto_msgTypes[64] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OrchestrationIdReusePolicy) String() string { +func (x *InstanceQuery) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestrationIdReusePolicy) ProtoMessage() {} +func (*InstanceQuery) ProtoMessage() {} -func (x *OrchestrationIdReusePolicy) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[37] - if protoimpl.UnsafeEnabled && x != nil { +func (x *InstanceQuery) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[64] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2835,51 +4779,91 @@ func (x *OrchestrationIdReusePolicy) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestrationIdReusePolicy.ProtoReflect.Descriptor instead. -func (*OrchestrationIdReusePolicy) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{37} +// Deprecated: Use InstanceQuery.ProtoReflect.Descriptor instead. +func (*InstanceQuery) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{64} +} + +func (x *InstanceQuery) GetRuntimeStatus() []OrchestrationStatus { + if x != nil { + return x.RuntimeStatus + } + return nil +} + +func (x *InstanceQuery) GetCreatedTimeFrom() *timestamppb.Timestamp { + if x != nil { + return x.CreatedTimeFrom + } + return nil +} + +func (x *InstanceQuery) GetCreatedTimeTo() *timestamppb.Timestamp { + if x != nil { + return x.CreatedTimeTo + } + return nil +} + +func (x *InstanceQuery) GetTaskHubNames() []*wrapperspb.StringValue { + if x != nil { + return x.TaskHubNames + } + return nil +} + +func (x *InstanceQuery) GetMaxInstanceCount() int32 { + if x != nil { + return x.MaxInstanceCount + } + return 0 } -func (x *OrchestrationIdReusePolicy) GetOperationStatus() []OrchestrationStatus { +func (x *InstanceQuery) GetContinuationToken() *wrapperspb.StringValue { if x != nil { - return x.OperationStatus + return x.ContinuationToken } return nil } -func (x *OrchestrationIdReusePolicy) GetAction() CreateOrchestrationAction { +func (x *InstanceQuery) GetInstanceIdPrefix() *wrapperspb.StringValue { if x != nil { - return x.Action + return x.InstanceIdPrefix } - return CreateOrchestrationAction_ERROR + return nil } -type CreateInstanceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *InstanceQuery) GetFetchInputsAndOutputs() bool { + if x != nil { + return x.FetchInputsAndOutputs + } + return false +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` +type QueryInstancesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + OrchestrationState []*OrchestrationState `protobuf:"bytes,1,rep,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateInstanceResponse) Reset() { - *x = CreateInstanceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[38] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *QueryInstancesResponse) Reset() { + *x = QueryInstancesResponse{} + mi := &file_orchestrator_service_proto_msgTypes[65] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateInstanceResponse) String() string { +func (x *QueryInstancesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateInstanceResponse) ProtoMessage() {} +func (*QueryInstancesResponse) ProtoMessage() {} -func (x *CreateInstanceResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[38] - if protoimpl.UnsafeEnabled && x != nil { +func (x *QueryInstancesResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[65] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2889,45 +4873,52 @@ func (x *CreateInstanceResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateInstanceResponse.ProtoReflect.Descriptor instead. -func (*CreateInstanceResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{38} +// Deprecated: Use QueryInstancesResponse.ProtoReflect.Descriptor instead. +func (*QueryInstancesResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{65} } -func (x *CreateInstanceResponse) GetInstanceId() string { +func (x *QueryInstancesResponse) GetOrchestrationState() []*OrchestrationState { if x != nil { - return x.InstanceId + return x.OrchestrationState } - return "" + return nil } -type GetInstanceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *QueryInstancesResponse) GetContinuationToken() *wrapperspb.StringValue { + if x != nil { + return x.ContinuationToken + } + return nil +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - GetInputsAndOutputs bool `protobuf:"varint,2,opt,name=getInputsAndOutputs,proto3" json:"getInputsAndOutputs,omitempty"` +type ListInstanceIdsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RuntimeStatus []OrchestrationStatus `protobuf:"varint,1,rep,packed,name=runtimeStatus,proto3,enum=OrchestrationStatus" json:"runtimeStatus,omitempty"` + CompletedTimeFrom *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=completedTimeFrom,proto3" json:"completedTimeFrom,omitempty"` + CompletedTimeTo *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=completedTimeTo,proto3" json:"completedTimeTo,omitempty"` + PageSize int32 `protobuf:"varint,4,opt,name=pageSize,proto3" json:"pageSize,omitempty"` + LastInstanceKey *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=lastInstanceKey,proto3" json:"lastInstanceKey,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *GetInstanceRequest) Reset() { - *x = GetInstanceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[39] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *ListInstanceIdsRequest) Reset() { + *x = ListInstanceIdsRequest{} + mi := &file_orchestrator_service_proto_msgTypes[66] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *GetInstanceRequest) String() string { +func (x *ListInstanceIdsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetInstanceRequest) ProtoMessage() {} +func (*ListInstanceIdsRequest) ProtoMessage() {} -func (x *GetInstanceRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[39] - if protoimpl.UnsafeEnabled && x != nil { +func (x *ListInstanceIdsRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[66] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2937,52 +4928,70 @@ func (x *GetInstanceRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetInstanceRequest.ProtoReflect.Descriptor instead. -func (*GetInstanceRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{39} +// Deprecated: Use ListInstanceIdsRequest.ProtoReflect.Descriptor instead. +func (*ListInstanceIdsRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{66} } -func (x *GetInstanceRequest) GetInstanceId() string { +func (x *ListInstanceIdsRequest) GetRuntimeStatus() []OrchestrationStatus { if x != nil { - return x.InstanceId + return x.RuntimeStatus } - return "" + return nil } -func (x *GetInstanceRequest) GetGetInputsAndOutputs() bool { +func (x *ListInstanceIdsRequest) GetCompletedTimeFrom() *timestamppb.Timestamp { if x != nil { - return x.GetInputsAndOutputs + return x.CompletedTimeFrom } - return false + return nil } -type GetInstanceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *ListInstanceIdsRequest) GetCompletedTimeTo() *timestamppb.Timestamp { + if x != nil { + return x.CompletedTimeTo + } + return nil +} - Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` - OrchestrationState *OrchestrationState `protobuf:"bytes,2,opt,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` +func (x *ListInstanceIdsRequest) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 } -func (x *GetInstanceResponse) Reset() { - *x = GetInstanceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[40] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *ListInstanceIdsRequest) GetLastInstanceKey() *wrapperspb.StringValue { + if x != nil { + return x.LastInstanceKey } + return nil } -func (x *GetInstanceResponse) String() string { +type ListInstanceIdsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceIds []string `protobuf:"bytes,1,rep,name=instanceIds,proto3" json:"instanceIds,omitempty"` + LastInstanceKey *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=lastInstanceKey,proto3" json:"lastInstanceKey,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListInstanceIdsResponse) Reset() { + *x = ListInstanceIdsResponse{} + mi := &file_orchestrator_service_proto_msgTypes[67] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListInstanceIdsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetInstanceResponse) ProtoMessage() {} +func (*ListInstanceIdsResponse) ProtoMessage() {} -func (x *GetInstanceResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[40] - if protoimpl.UnsafeEnabled && x != nil { +func (x *ListInstanceIdsResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[67] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -2992,52 +5001,56 @@ func (x *GetInstanceResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetInstanceResponse.ProtoReflect.Descriptor instead. -func (*GetInstanceResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{40} +// Deprecated: Use ListInstanceIdsResponse.ProtoReflect.Descriptor instead. +func (*ListInstanceIdsResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{67} } -func (x *GetInstanceResponse) GetExists() bool { +func (x *ListInstanceIdsResponse) GetInstanceIds() []string { if x != nil { - return x.Exists + return x.InstanceIds } - return false + return nil } -func (x *GetInstanceResponse) GetOrchestrationState() *OrchestrationState { +func (x *ListInstanceIdsResponse) GetLastInstanceKey() *wrapperspb.StringValue { if x != nil { - return x.OrchestrationState + return x.LastInstanceKey } return nil } -type RewindInstanceRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Reason *wrappers.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` +type PurgeInstancesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Request: + // + // *PurgeInstancesRequest_InstanceId + // *PurgeInstancesRequest_PurgeInstanceFilter + // *PurgeInstancesRequest_InstanceBatch + Request isPurgeInstancesRequest_Request `protobuf_oneof:"request"` + Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` + // used in the case when an instanceId is specified to determine if the purge request is for an orchestration (as opposed to an entity) + IsOrchestration bool `protobuf:"varint,5,opt,name=isOrchestration,proto3" json:"isOrchestration,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *RewindInstanceRequest) Reset() { - *x = RewindInstanceRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[41] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *PurgeInstancesRequest) Reset() { + *x = PurgeInstancesRequest{} + mi := &file_orchestrator_service_proto_msgTypes[68] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *RewindInstanceRequest) String() string { +func (x *PurgeInstancesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RewindInstanceRequest) ProtoMessage() {} +func (*PurgeInstancesRequest) ProtoMessage() {} -func (x *RewindInstanceRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[41] - if protoimpl.UnsafeEnabled && x != nil { +func (x *PurgeInstancesRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[68] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3047,99 +5060,107 @@ func (x *RewindInstanceRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RewindInstanceRequest.ProtoReflect.Descriptor instead. -func (*RewindInstanceRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{41} +// Deprecated: Use PurgeInstancesRequest.ProtoReflect.Descriptor instead. +func (*PurgeInstancesRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{68} } -func (x *RewindInstanceRequest) GetInstanceId() string { +func (x *PurgeInstancesRequest) GetRequest() isPurgeInstancesRequest_Request { if x != nil { - return x.InstanceId + return x.Request + } + return nil +} + +func (x *PurgeInstancesRequest) GetInstanceId() string { + if x != nil { + if x, ok := x.Request.(*PurgeInstancesRequest_InstanceId); ok { + return x.InstanceId + } } return "" } -func (x *RewindInstanceRequest) GetReason() *wrappers.StringValue { +func (x *PurgeInstancesRequest) GetPurgeInstanceFilter() *PurgeInstanceFilter { if x != nil { - return x.Reason + if x, ok := x.Request.(*PurgeInstancesRequest_PurgeInstanceFilter); ok { + return x.PurgeInstanceFilter + } } return nil } -type RewindInstanceResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *PurgeInstancesRequest) GetInstanceBatch() *InstanceBatch { + if x != nil { + if x, ok := x.Request.(*PurgeInstancesRequest_InstanceBatch); ok { + return x.InstanceBatch + } + } + return nil } -func (x *RewindInstanceResponse) Reset() { - *x = RewindInstanceResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[42] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *PurgeInstancesRequest) GetRecursive() bool { + if x != nil { + return x.Recursive } + return false } -func (x *RewindInstanceResponse) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *PurgeInstancesRequest) GetIsOrchestration() bool { + if x != nil { + return x.IsOrchestration + } + return false } -func (*RewindInstanceResponse) ProtoMessage() {} +type isPurgeInstancesRequest_Request interface { + isPurgeInstancesRequest_Request() +} -func (x *RewindInstanceResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[42] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms - } - return mi.MessageOf(x) +type PurgeInstancesRequest_InstanceId struct { + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3,oneof"` } -// Deprecated: Use RewindInstanceResponse.ProtoReflect.Descriptor instead. -func (*RewindInstanceResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{42} +type PurgeInstancesRequest_PurgeInstanceFilter struct { + PurgeInstanceFilter *PurgeInstanceFilter `protobuf:"bytes,2,opt,name=purgeInstanceFilter,proto3,oneof"` } -type OrchestrationState struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +type PurgeInstancesRequest_InstanceBatch struct { + InstanceBatch *InstanceBatch `protobuf:"bytes,4,opt,name=instanceBatch,proto3,oneof"` +} + +func (*PurgeInstancesRequest_InstanceId) isPurgeInstancesRequest_Request() {} + +func (*PurgeInstancesRequest_PurgeInstanceFilter) isPurgeInstancesRequest_Request() {} + +func (*PurgeInstancesRequest_InstanceBatch) isPurgeInstancesRequest_Request() {} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - OrchestrationStatus OrchestrationStatus `protobuf:"varint,4,opt,name=orchestrationStatus,proto3,enum=OrchestrationStatus" json:"orchestrationStatus,omitempty"` - ScheduledStartTimestamp *timestamp.Timestamp `protobuf:"bytes,5,opt,name=scheduledStartTimestamp,proto3" json:"scheduledStartTimestamp,omitempty"` - CreatedTimestamp *timestamp.Timestamp `protobuf:"bytes,6,opt,name=createdTimestamp,proto3" json:"createdTimestamp,omitempty"` - LastUpdatedTimestamp *timestamp.Timestamp `protobuf:"bytes,7,opt,name=lastUpdatedTimestamp,proto3" json:"lastUpdatedTimestamp,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,8,opt,name=input,proto3" json:"input,omitempty"` - Output *wrappers.StringValue `protobuf:"bytes,9,opt,name=output,proto3" json:"output,omitempty"` - CustomStatus *wrappers.StringValue `protobuf:"bytes,10,opt,name=customStatus,proto3" json:"customStatus,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,11,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` +type PurgeInstanceFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + CreatedTimeFrom *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=createdTimeFrom,proto3" json:"createdTimeFrom,omitempty"` + CreatedTimeTo *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=createdTimeTo,proto3" json:"createdTimeTo,omitempty"` + RuntimeStatus []OrchestrationStatus `protobuf:"varint,3,rep,packed,name=runtimeStatus,proto3,enum=OrchestrationStatus" json:"runtimeStatus,omitempty"` + Timeout *durationpb.Duration `protobuf:"bytes,4,opt,name=timeout,proto3" json:"timeout,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OrchestrationState) Reset() { - *x = OrchestrationState{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[43] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *PurgeInstanceFilter) Reset() { + *x = PurgeInstanceFilter{} + mi := &file_orchestrator_service_proto_msgTypes[69] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OrchestrationState) String() string { +func (x *PurgeInstanceFilter) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestrationState) ProtoMessage() {} +func (*PurgeInstanceFilter) ProtoMessage() {} -func (x *OrchestrationState) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[43] - if protoimpl.UnsafeEnabled && x != nil { +func (x *PurgeInstanceFilter) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[69] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3149,116 +5170,115 @@ func (x *OrchestrationState) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestrationState.ProtoReflect.Descriptor instead. -func (*OrchestrationState) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{43} -} - -func (x *OrchestrationState) GetInstanceId() string { - if x != nil { - return x.InstanceId - } - return "" -} - -func (x *OrchestrationState) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *OrchestrationState) GetVersion() *wrappers.StringValue { - if x != nil { - return x.Version - } - return nil -} - -func (x *OrchestrationState) GetOrchestrationStatus() OrchestrationStatus { - if x != nil { - return x.OrchestrationStatus - } - return OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING +// Deprecated: Use PurgeInstanceFilter.ProtoReflect.Descriptor instead. +func (*PurgeInstanceFilter) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{69} } -func (x *OrchestrationState) GetScheduledStartTimestamp() *timestamp.Timestamp { +func (x *PurgeInstanceFilter) GetCreatedTimeFrom() *timestamppb.Timestamp { if x != nil { - return x.ScheduledStartTimestamp + return x.CreatedTimeFrom } return nil } -func (x *OrchestrationState) GetCreatedTimestamp() *timestamp.Timestamp { +func (x *PurgeInstanceFilter) GetCreatedTimeTo() *timestamppb.Timestamp { if x != nil { - return x.CreatedTimestamp + return x.CreatedTimeTo } return nil } -func (x *OrchestrationState) GetLastUpdatedTimestamp() *timestamp.Timestamp { +func (x *PurgeInstanceFilter) GetRuntimeStatus() []OrchestrationStatus { if x != nil { - return x.LastUpdatedTimestamp + return x.RuntimeStatus } return nil } -func (x *OrchestrationState) GetInput() *wrappers.StringValue { +func (x *PurgeInstanceFilter) GetTimeout() *durationpb.Duration { if x != nil { - return x.Input + return x.Timeout } return nil } -func (x *OrchestrationState) GetOutput() *wrappers.StringValue { +type PurgeInstancesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + DeletedInstanceCount int32 `protobuf:"varint,1,opt,name=deletedInstanceCount,proto3" json:"deletedInstanceCount,omitempty"` + IsComplete *wrapperspb.BoolValue `protobuf:"bytes,2,opt,name=isComplete,proto3" json:"isComplete,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PurgeInstancesResponse) Reset() { + *x = PurgeInstancesResponse{} + mi := &file_orchestrator_service_proto_msgTypes[70] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PurgeInstancesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PurgeInstancesResponse) ProtoMessage() {} + +func (x *PurgeInstancesResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[70] if x != nil { - return x.Output + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) +} + +// Deprecated: Use PurgeInstancesResponse.ProtoReflect.Descriptor instead. +func (*PurgeInstancesResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{70} } -func (x *OrchestrationState) GetCustomStatus() *wrappers.StringValue { +func (x *PurgeInstancesResponse) GetDeletedInstanceCount() int32 { if x != nil { - return x.CustomStatus + return x.DeletedInstanceCount } - return nil + return 0 } -func (x *OrchestrationState) GetFailureDetails() *TaskFailureDetails { +func (x *PurgeInstancesResponse) GetIsComplete() *wrapperspb.BoolValue { if x != nil { - return x.FailureDetails + return x.IsComplete } return nil } -type RaiseEventRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` +type RestartInstanceRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + RestartWithNewInstanceId bool `protobuf:"varint,2,opt,name=restartWithNewInstanceId,proto3" json:"restartWithNewInstanceId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *RaiseEventRequest) Reset() { - *x = RaiseEventRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[44] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *RestartInstanceRequest) Reset() { + *x = RestartInstanceRequest{} + mi := &file_orchestrator_service_proto_msgTypes[71] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *RaiseEventRequest) String() string { +func (x *RestartInstanceRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RaiseEventRequest) ProtoMessage() {} +func (*RestartInstanceRequest) ProtoMessage() {} -func (x *RaiseEventRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[44] - if protoimpl.UnsafeEnabled && x != nil { +func (x *RestartInstanceRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[71] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3268,56 +5288,48 @@ func (x *RaiseEventRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RaiseEventRequest.ProtoReflect.Descriptor instead. -func (*RaiseEventRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{44} +// Deprecated: Use RestartInstanceRequest.ProtoReflect.Descriptor instead. +func (*RestartInstanceRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{71} } -func (x *RaiseEventRequest) GetInstanceId() string { +func (x *RestartInstanceRequest) GetInstanceId() string { if x != nil { return x.InstanceId } return "" } -func (x *RaiseEventRequest) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *RaiseEventRequest) GetInput() *wrappers.StringValue { +func (x *RestartInstanceRequest) GetRestartWithNewInstanceId() bool { if x != nil { - return x.Input + return x.RestartWithNewInstanceId } - return nil + return false } -type RaiseEventResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type RestartInstanceResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *RaiseEventResponse) Reset() { - *x = RaiseEventResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[45] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *RestartInstanceResponse) Reset() { + *x = RestartInstanceResponse{} + mi := &file_orchestrator_service_proto_msgTypes[72] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *RaiseEventResponse) String() string { +func (x *RestartInstanceResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*RaiseEventResponse) ProtoMessage() {} +func (*RestartInstanceResponse) ProtoMessage() {} -func (x *RaiseEventResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[45] - if protoimpl.UnsafeEnabled && x != nil { +func (x *RestartInstanceResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[72] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3327,39 +5339,41 @@ func (x *RaiseEventResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use RaiseEventResponse.ProtoReflect.Descriptor instead. -func (*RaiseEventResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{45} +// Deprecated: Use RestartInstanceResponse.ProtoReflect.Descriptor instead. +func (*RestartInstanceResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{72} } -type TerminateRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *RestartInstanceResponse) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Output *wrappers.StringValue `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` - Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` +type CreateTaskHubRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + RecreateIfExists bool `protobuf:"varint,1,opt,name=recreateIfExists,proto3" json:"recreateIfExists,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *TerminateRequest) Reset() { - *x = TerminateRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[46] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *CreateTaskHubRequest) Reset() { + *x = CreateTaskHubRequest{} + mi := &file_orchestrator_service_proto_msgTypes[73] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *TerminateRequest) String() string { +func (x *CreateTaskHubRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*TerminateRequest) ProtoMessage() {} +func (*CreateTaskHubRequest) ProtoMessage() {} -func (x *TerminateRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[46] - if protoimpl.UnsafeEnabled && x != nil { +func (x *CreateTaskHubRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[73] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3369,56 +5383,76 @@ func (x *TerminateRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use TerminateRequest.ProtoReflect.Descriptor instead. -func (*TerminateRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{46} +// Deprecated: Use CreateTaskHubRequest.ProtoReflect.Descriptor instead. +func (*CreateTaskHubRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{73} } -func (x *TerminateRequest) GetInstanceId() string { +func (x *CreateTaskHubRequest) GetRecreateIfExists() bool { if x != nil { - return x.InstanceId + return x.RecreateIfExists } - return "" + return false } -func (x *TerminateRequest) GetOutput() *wrappers.StringValue { - if x != nil { - return x.Output - } - return nil +type CreateTaskHubResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *TerminateRequest) GetRecursive() bool { +func (x *CreateTaskHubResponse) Reset() { + *x = CreateTaskHubResponse{} + mi := &file_orchestrator_service_proto_msgTypes[74] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateTaskHubResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateTaskHubResponse) ProtoMessage() {} + +func (x *CreateTaskHubResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[74] if x != nil { - return x.Recursive + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return false + return mi.MessageOf(x) } -type TerminateResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +// Deprecated: Use CreateTaskHubResponse.ProtoReflect.Descriptor instead. +func (*CreateTaskHubResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{74} +} + +type DeleteTaskHubRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *TerminateResponse) Reset() { - *x = TerminateResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[47] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *DeleteTaskHubRequest) Reset() { + *x = DeleteTaskHubRequest{} + mi := &file_orchestrator_service_proto_msgTypes[75] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *TerminateResponse) String() string { +func (x *DeleteTaskHubRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*TerminateResponse) ProtoMessage() {} +func (*DeleteTaskHubRequest) ProtoMessage() {} -func (x *TerminateResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[47] - if protoimpl.UnsafeEnabled && x != nil { +func (x *DeleteTaskHubRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[75] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3428,90 +5462,161 @@ func (x *TerminateResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use TerminateResponse.ProtoReflect.Descriptor instead. -func (*TerminateResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{47} +// Deprecated: Use DeleteTaskHubRequest.ProtoReflect.Descriptor instead. +func (*DeleteTaskHubRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{75} } -type SuspendRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type DeleteTaskHubResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteTaskHubResponse) Reset() { + *x = DeleteTaskHubResponse{} + mi := &file_orchestrator_service_proto_msgTypes[76] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Reason *wrappers.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` +func (x *DeleteTaskHubResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *SuspendRequest) Reset() { - *x = SuspendRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[48] +func (*DeleteTaskHubResponse) ProtoMessage() {} + +func (x *DeleteTaskHubResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[76] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } + return mi.MessageOf(x) } -func (x *SuspendRequest) String() string { +// Deprecated: Use DeleteTaskHubResponse.ProtoReflect.Descriptor instead. +func (*DeleteTaskHubResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{76} +} + +type SignalEntityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + RequestId string `protobuf:"bytes,4,opt,name=requestId,proto3" json:"requestId,omitempty"` + ScheduledTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,6,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + RequestTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=requestTime,proto3" json:"requestTime,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SignalEntityRequest) Reset() { + *x = SignalEntityRequest{} + mi := &file_orchestrator_service_proto_msgTypes[77] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SignalEntityRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SuspendRequest) ProtoMessage() {} +func (*SignalEntityRequest) ProtoMessage() {} -func (x *SuspendRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[48] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SignalEntityRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[77] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } - return mi.MessageOf(x) + return mi.MessageOf(x) +} + +// Deprecated: Use SignalEntityRequest.ProtoReflect.Descriptor instead. +func (*SignalEntityRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{77} +} + +func (x *SignalEntityRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *SignalEntityRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *SignalEntityRequest) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input + } + return nil } -// Deprecated: Use SuspendRequest.ProtoReflect.Descriptor instead. -func (*SuspendRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{48} +func (x *SignalEntityRequest) GetRequestId() string { + if x != nil { + return x.RequestId + } + return "" } -func (x *SuspendRequest) GetInstanceId() string { +func (x *SignalEntityRequest) GetScheduledTime() *timestamppb.Timestamp { if x != nil { - return x.InstanceId + return x.ScheduledTime } - return "" + return nil } -func (x *SuspendRequest) GetReason() *wrappers.StringValue { +func (x *SignalEntityRequest) GetParentTraceContext() *TraceContext { if x != nil { - return x.Reason + return x.ParentTraceContext } return nil } -type SuspendResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +func (x *SignalEntityRequest) GetRequestTime() *timestamppb.Timestamp { + if x != nil { + return x.RequestTime + } + return nil +} + +type SignalEntityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *SuspendResponse) Reset() { - *x = SuspendResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[49] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *SignalEntityResponse) Reset() { + *x = SignalEntityResponse{} + mi := &file_orchestrator_service_proto_msgTypes[78] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *SuspendResponse) String() string { +func (x *SignalEntityResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SuspendResponse) ProtoMessage() {} +func (*SignalEntityResponse) ProtoMessage() {} -func (x *SuspendResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[49] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SignalEntityResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[78] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3521,38 +5626,35 @@ func (x *SuspendResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SuspendResponse.ProtoReflect.Descriptor instead. -func (*SuspendResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{49} +// Deprecated: Use SignalEntityResponse.ProtoReflect.Descriptor instead. +func (*SignalEntityResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{78} } -type ResumeRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type GetEntityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + IncludeState bool `protobuf:"varint,2,opt,name=includeState,proto3" json:"includeState,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Reason *wrappers.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *ResumeRequest) Reset() { - *x = ResumeRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[50] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *GetEntityRequest) Reset() { + *x = GetEntityRequest{} + mi := &file_orchestrator_service_proto_msgTypes[79] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *ResumeRequest) String() string { +func (x *GetEntityRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ResumeRequest) ProtoMessage() {} +func (*GetEntityRequest) ProtoMessage() {} -func (x *ResumeRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[50] - if protoimpl.UnsafeEnabled && x != nil { +func (x *GetEntityRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[79] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3562,49 +5664,49 @@ func (x *ResumeRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ResumeRequest.ProtoReflect.Descriptor instead. -func (*ResumeRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{50} +// Deprecated: Use GetEntityRequest.ProtoReflect.Descriptor instead. +func (*GetEntityRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{79} } -func (x *ResumeRequest) GetInstanceId() string { +func (x *GetEntityRequest) GetInstanceId() string { if x != nil { return x.InstanceId } return "" } -func (x *ResumeRequest) GetReason() *wrappers.StringValue { +func (x *GetEntityRequest) GetIncludeState() bool { if x != nil { - return x.Reason + return x.IncludeState } - return nil + return false } -type ResumeResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type GetEntityResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` + Entity *EntityMetadata `protobuf:"bytes,2,opt,name=entity,proto3" json:"entity,omitempty"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *ResumeResponse) Reset() { - *x = ResumeResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[51] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *GetEntityResponse) Reset() { + *x = GetEntityResponse{} + mi := &file_orchestrator_service_proto_msgTypes[80] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *ResumeResponse) String() string { +func (x *GetEntityResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*ResumeResponse) ProtoMessage() {} +func (*GetEntityResponse) ProtoMessage() {} -func (x *ResumeResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[51] - if protoimpl.UnsafeEnabled && x != nil { +func (x *GetEntityResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[80] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3614,91 +5716,54 @@ func (x *ResumeResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use ResumeResponse.ProtoReflect.Descriptor instead. -func (*ResumeResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{51} -} - -type QueryInstancesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Query *InstanceQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` -} - -func (x *QueryInstancesRequest) Reset() { - *x = QueryInstancesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[52] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } -} - -func (x *QueryInstancesRequest) String() string { - return protoimpl.X.MessageStringOf(x) +// Deprecated: Use GetEntityResponse.ProtoReflect.Descriptor instead. +func (*GetEntityResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{80} } -func (*QueryInstancesRequest) ProtoMessage() {} - -func (x *QueryInstancesRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[52] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *GetEntityResponse) GetExists() bool { + if x != nil { + return x.Exists } - return mi.MessageOf(x) -} - -// Deprecated: Use QueryInstancesRequest.ProtoReflect.Descriptor instead. -func (*QueryInstancesRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{52} + return false } -func (x *QueryInstancesRequest) GetQuery() *InstanceQuery { +func (x *GetEntityResponse) GetEntity() *EntityMetadata { if x != nil { - return x.Query + return x.Entity } return nil } -type InstanceQuery struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - RuntimeStatus []OrchestrationStatus `protobuf:"varint,1,rep,packed,name=runtimeStatus,proto3,enum=OrchestrationStatus" json:"runtimeStatus,omitempty"` - CreatedTimeFrom *timestamp.Timestamp `protobuf:"bytes,2,opt,name=createdTimeFrom,proto3" json:"createdTimeFrom,omitempty"` - CreatedTimeTo *timestamp.Timestamp `protobuf:"bytes,3,opt,name=createdTimeTo,proto3" json:"createdTimeTo,omitempty"` - TaskHubNames []*wrappers.StringValue `protobuf:"bytes,4,rep,name=taskHubNames,proto3" json:"taskHubNames,omitempty"` - MaxInstanceCount int32 `protobuf:"varint,5,opt,name=maxInstanceCount,proto3" json:"maxInstanceCount,omitempty"` - ContinuationToken *wrappers.StringValue `protobuf:"bytes,6,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` - InstanceIdPrefix *wrappers.StringValue `protobuf:"bytes,7,opt,name=instanceIdPrefix,proto3" json:"instanceIdPrefix,omitempty"` - FetchInputsAndOutputs bool `protobuf:"varint,8,opt,name=fetchInputsAndOutputs,proto3" json:"fetchInputsAndOutputs,omitempty"` +type EntityQuery struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceIdStartsWith *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=instanceIdStartsWith,proto3" json:"instanceIdStartsWith,omitempty"` + LastModifiedFrom *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=lastModifiedFrom,proto3" json:"lastModifiedFrom,omitempty"` + LastModifiedTo *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=lastModifiedTo,proto3" json:"lastModifiedTo,omitempty"` + IncludeState bool `protobuf:"varint,4,opt,name=includeState,proto3" json:"includeState,omitempty"` + IncludeTransient bool `protobuf:"varint,5,opt,name=includeTransient,proto3" json:"includeTransient,omitempty"` + PageSize *wrapperspb.Int32Value `protobuf:"bytes,6,opt,name=pageSize,proto3" json:"pageSize,omitempty"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,7,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *InstanceQuery) Reset() { - *x = InstanceQuery{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[53] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityQuery) Reset() { + *x = EntityQuery{} + mi := &file_orchestrator_service_proto_msgTypes[81] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *InstanceQuery) String() string { +func (x *EntityQuery) String() string { return protoimpl.X.MessageStringOf(x) } -func (*InstanceQuery) ProtoMessage() {} +func (*EntityQuery) ProtoMessage() {} -func (x *InstanceQuery) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[53] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityQuery) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[81] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3708,94 +5773,83 @@ func (x *InstanceQuery) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use InstanceQuery.ProtoReflect.Descriptor instead. -func (*InstanceQuery) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{53} +// Deprecated: Use EntityQuery.ProtoReflect.Descriptor instead. +func (*EntityQuery) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{81} } -func (x *InstanceQuery) GetRuntimeStatus() []OrchestrationStatus { +func (x *EntityQuery) GetInstanceIdStartsWith() *wrapperspb.StringValue { if x != nil { - return x.RuntimeStatus + return x.InstanceIdStartsWith } return nil } -func (x *InstanceQuery) GetCreatedTimeFrom() *timestamp.Timestamp { +func (x *EntityQuery) GetLastModifiedFrom() *timestamppb.Timestamp { if x != nil { - return x.CreatedTimeFrom + return x.LastModifiedFrom } return nil } -func (x *InstanceQuery) GetCreatedTimeTo() *timestamp.Timestamp { +func (x *EntityQuery) GetLastModifiedTo() *timestamppb.Timestamp { if x != nil { - return x.CreatedTimeTo + return x.LastModifiedTo } return nil } -func (x *InstanceQuery) GetTaskHubNames() []*wrappers.StringValue { +func (x *EntityQuery) GetIncludeState() bool { if x != nil { - return x.TaskHubNames + return x.IncludeState } - return nil + return false } -func (x *InstanceQuery) GetMaxInstanceCount() int32 { +func (x *EntityQuery) GetIncludeTransient() bool { if x != nil { - return x.MaxInstanceCount + return x.IncludeTransient } - return 0 + return false } -func (x *InstanceQuery) GetContinuationToken() *wrappers.StringValue { +func (x *EntityQuery) GetPageSize() *wrapperspb.Int32Value { if x != nil { - return x.ContinuationToken + return x.PageSize } return nil } -func (x *InstanceQuery) GetInstanceIdPrefix() *wrappers.StringValue { +func (x *EntityQuery) GetContinuationToken() *wrapperspb.StringValue { if x != nil { - return x.InstanceIdPrefix + return x.ContinuationToken } return nil } -func (x *InstanceQuery) GetFetchInputsAndOutputs() bool { - if x != nil { - return x.FetchInputsAndOutputs - } - return false -} - -type QueryInstancesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type QueryEntitiesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Query *EntityQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` unknownFields protoimpl.UnknownFields - - OrchestrationState []*OrchestrationState `protobuf:"bytes,1,rep,name=orchestrationState,proto3" json:"orchestrationState,omitempty"` - ContinuationToken *wrappers.StringValue `protobuf:"bytes,2,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *QueryInstancesResponse) Reset() { - *x = QueryInstancesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[54] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *QueryEntitiesRequest) Reset() { + *x = QueryEntitiesRequest{} + mi := &file_orchestrator_service_proto_msgTypes[82] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *QueryInstancesResponse) String() string { +func (x *QueryEntitiesRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryInstancesResponse) ProtoMessage() {} +func (*QueryEntitiesRequest) ProtoMessage() {} -func (x *QueryInstancesResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[54] - if protoimpl.UnsafeEnabled && x != nil { +func (x *QueryEntitiesRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[82] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3805,56 +5859,42 @@ func (x *QueryInstancesResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryInstancesResponse.ProtoReflect.Descriptor instead. -func (*QueryInstancesResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{54} -} - -func (x *QueryInstancesResponse) GetOrchestrationState() []*OrchestrationState { - if x != nil { - return x.OrchestrationState - } - return nil +// Deprecated: Use QueryEntitiesRequest.ProtoReflect.Descriptor instead. +func (*QueryEntitiesRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{82} } -func (x *QueryInstancesResponse) GetContinuationToken() *wrappers.StringValue { +func (x *QueryEntitiesRequest) GetQuery() *EntityQuery { if x != nil { - return x.ContinuationToken + return x.Query } return nil } -type PurgeInstancesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - // Types that are assignable to Request: - // - // *PurgeInstancesRequest_InstanceId - // *PurgeInstancesRequest_PurgeInstanceFilter - Request isPurgeInstancesRequest_Request `protobuf_oneof:"request"` - Recursive bool `protobuf:"varint,3,opt,name=recursive,proto3" json:"recursive,omitempty"` +type QueryEntitiesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Entities []*EntityMetadata `protobuf:"bytes,1,rep,name=entities,proto3" json:"entities,omitempty"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *PurgeInstancesRequest) Reset() { - *x = PurgeInstancesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[55] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *QueryEntitiesResponse) Reset() { + *x = QueryEntitiesResponse{} + mi := &file_orchestrator_service_proto_msgTypes[83] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *PurgeInstancesRequest) String() string { +func (x *QueryEntitiesResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PurgeInstancesRequest) ProtoMessage() {} +func (*QueryEntitiesResponse) ProtoMessage() {} -func (x *PurgeInstancesRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[55] - if protoimpl.UnsafeEnabled && x != nil { +func (x *QueryEntitiesResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[83] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3864,83 +5904,52 @@ func (x *PurgeInstancesRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PurgeInstancesRequest.ProtoReflect.Descriptor instead. -func (*PurgeInstancesRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{55} -} - -func (m *PurgeInstancesRequest) GetRequest() isPurgeInstancesRequest_Request { - if m != nil { - return m.Request - } - return nil -} - -func (x *PurgeInstancesRequest) GetInstanceId() string { - if x, ok := x.GetRequest().(*PurgeInstancesRequest_InstanceId); ok { - return x.InstanceId - } - return "" +// Deprecated: Use QueryEntitiesResponse.ProtoReflect.Descriptor instead. +func (*QueryEntitiesResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{83} } -func (x *PurgeInstancesRequest) GetPurgeInstanceFilter() *PurgeInstanceFilter { - if x, ok := x.GetRequest().(*PurgeInstancesRequest_PurgeInstanceFilter); ok { - return x.PurgeInstanceFilter +func (x *QueryEntitiesResponse) GetEntities() []*EntityMetadata { + if x != nil { + return x.Entities } return nil } -func (x *PurgeInstancesRequest) GetRecursive() bool { +func (x *QueryEntitiesResponse) GetContinuationToken() *wrapperspb.StringValue { if x != nil { - return x.Recursive + return x.ContinuationToken } - return false -} - -type isPurgeInstancesRequest_Request interface { - isPurgeInstancesRequest_Request() -} - -type PurgeInstancesRequest_InstanceId struct { - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3,oneof"` -} - -type PurgeInstancesRequest_PurgeInstanceFilter struct { - PurgeInstanceFilter *PurgeInstanceFilter `protobuf:"bytes,2,opt,name=purgeInstanceFilter,proto3,oneof"` + return nil } -func (*PurgeInstancesRequest_InstanceId) isPurgeInstancesRequest_Request() {} - -func (*PurgeInstancesRequest_PurgeInstanceFilter) isPurgeInstancesRequest_Request() {} - -type PurgeInstanceFilter struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - CreatedTimeFrom *timestamp.Timestamp `protobuf:"bytes,1,opt,name=createdTimeFrom,proto3" json:"createdTimeFrom,omitempty"` - CreatedTimeTo *timestamp.Timestamp `protobuf:"bytes,2,opt,name=createdTimeTo,proto3" json:"createdTimeTo,omitempty"` - RuntimeStatus []OrchestrationStatus `protobuf:"varint,3,rep,packed,name=runtimeStatus,proto3,enum=OrchestrationStatus" json:"runtimeStatus,omitempty"` +type EntityMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + LastModifiedTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=lastModifiedTime,proto3" json:"lastModifiedTime,omitempty"` + BacklogQueueSize int32 `protobuf:"varint,3,opt,name=backlogQueueSize,proto3" json:"backlogQueueSize,omitempty"` + LockedBy *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=lockedBy,proto3" json:"lockedBy,omitempty"` + SerializedState *wrapperspb.StringValue `protobuf:"bytes,5,opt,name=serializedState,proto3" json:"serializedState,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *PurgeInstanceFilter) Reset() { - *x = PurgeInstanceFilter{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[56] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityMetadata) Reset() { + *x = EntityMetadata{} + mi := &file_orchestrator_service_proto_msgTypes[84] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *PurgeInstanceFilter) String() string { +func (x *EntityMetadata) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PurgeInstanceFilter) ProtoMessage() {} +func (*EntityMetadata) ProtoMessage() {} -func (x *PurgeInstanceFilter) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[56] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityMetadata) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[84] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -3950,58 +5959,71 @@ func (x *PurgeInstanceFilter) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PurgeInstanceFilter.ProtoReflect.Descriptor instead. -func (*PurgeInstanceFilter) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{56} +// Deprecated: Use EntityMetadata.ProtoReflect.Descriptor instead. +func (*EntityMetadata) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{84} } -func (x *PurgeInstanceFilter) GetCreatedTimeFrom() *timestamp.Timestamp { +func (x *EntityMetadata) GetInstanceId() string { if x != nil { - return x.CreatedTimeFrom + return x.InstanceId } - return nil + return "" } -func (x *PurgeInstanceFilter) GetCreatedTimeTo() *timestamp.Timestamp { +func (x *EntityMetadata) GetLastModifiedTime() *timestamppb.Timestamp { if x != nil { - return x.CreatedTimeTo + return x.LastModifiedTime } return nil } -func (x *PurgeInstanceFilter) GetRuntimeStatus() []OrchestrationStatus { +func (x *EntityMetadata) GetBacklogQueueSize() int32 { if x != nil { - return x.RuntimeStatus + return x.BacklogQueueSize + } + return 0 +} + +func (x *EntityMetadata) GetLockedBy() *wrapperspb.StringValue { + if x != nil { + return x.LockedBy } return nil } -type PurgeInstancesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *EntityMetadata) GetSerializedState() *wrapperspb.StringValue { + if x != nil { + return x.SerializedState + } + return nil +} - DeletedInstanceCount int32 `protobuf:"varint,1,opt,name=deletedInstanceCount,proto3" json:"deletedInstanceCount,omitempty"` +type CleanEntityStorageRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + RemoveEmptyEntities bool `protobuf:"varint,2,opt,name=removeEmptyEntities,proto3" json:"removeEmptyEntities,omitempty"` + ReleaseOrphanedLocks bool `protobuf:"varint,3,opt,name=releaseOrphanedLocks,proto3" json:"releaseOrphanedLocks,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *PurgeInstancesResponse) Reset() { - *x = PurgeInstancesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[57] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *CleanEntityStorageRequest) Reset() { + *x = CleanEntityStorageRequest{} + mi := &file_orchestrator_service_proto_msgTypes[85] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *PurgeInstancesResponse) String() string { +func (x *CleanEntityStorageRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*PurgeInstancesResponse) ProtoMessage() {} +func (*CleanEntityStorageRequest) ProtoMessage() {} -func (x *PurgeInstancesResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[57] - if protoimpl.UnsafeEnabled && x != nil { +func (x *CleanEntityStorageRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[85] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4011,89 +6033,57 @@ func (x *PurgeInstancesResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use PurgeInstancesResponse.ProtoReflect.Descriptor instead. -func (*PurgeInstancesResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{57} +// Deprecated: Use CleanEntityStorageRequest.ProtoReflect.Descriptor instead. +func (*CleanEntityStorageRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{85} } -func (x *PurgeInstancesResponse) GetDeletedInstanceCount() int32 { +func (x *CleanEntityStorageRequest) GetContinuationToken() *wrapperspb.StringValue { if x != nil { - return x.DeletedInstanceCount - } - return 0 -} - -type CreateTaskHubRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - RecreateIfExists bool `protobuf:"varint,1,opt,name=recreateIfExists,proto3" json:"recreateIfExists,omitempty"` -} - -func (x *CreateTaskHubRequest) Reset() { - *x = CreateTaskHubRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[58] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) + return x.ContinuationToken } + return nil } -func (x *CreateTaskHubRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*CreateTaskHubRequest) ProtoMessage() {} - -func (x *CreateTaskHubRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[58] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *CleanEntityStorageRequest) GetRemoveEmptyEntities() bool { + if x != nil { + return x.RemoveEmptyEntities } - return mi.MessageOf(x) -} - -// Deprecated: Use CreateTaskHubRequest.ProtoReflect.Descriptor instead. -func (*CreateTaskHubRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{58} + return false } -func (x *CreateTaskHubRequest) GetRecreateIfExists() bool { +func (x *CleanEntityStorageRequest) GetReleaseOrphanedLocks() bool { if x != nil { - return x.RecreateIfExists + return x.ReleaseOrphanedLocks } return false } -type CreateTaskHubResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +type CleanEntityStorageResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + ContinuationToken *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` + EmptyEntitiesRemoved int32 `protobuf:"varint,2,opt,name=emptyEntitiesRemoved,proto3" json:"emptyEntitiesRemoved,omitempty"` + OrphanedLocksReleased int32 `protobuf:"varint,3,opt,name=orphanedLocksReleased,proto3" json:"orphanedLocksReleased,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CreateTaskHubResponse) Reset() { - *x = CreateTaskHubResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[59] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *CleanEntityStorageResponse) Reset() { + *x = CleanEntityStorageResponse{} + mi := &file_orchestrator_service_proto_msgTypes[86] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CreateTaskHubResponse) String() string { +func (x *CleanEntityStorageResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CreateTaskHubResponse) ProtoMessage() {} +func (*CleanEntityStorageResponse) ProtoMessage() {} -func (x *CreateTaskHubResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[59] - if protoimpl.UnsafeEnabled && x != nil { +func (x *CleanEntityStorageResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[86] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4103,73 +6093,55 @@ func (x *CreateTaskHubResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CreateTaskHubResponse.ProtoReflect.Descriptor instead. -func (*CreateTaskHubResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{59} -} - -type DeleteTaskHubRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +// Deprecated: Use CleanEntityStorageResponse.ProtoReflect.Descriptor instead. +func (*CleanEntityStorageResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{86} } -func (x *DeleteTaskHubRequest) Reset() { - *x = DeleteTaskHubRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[60] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *CleanEntityStorageResponse) GetContinuationToken() *wrapperspb.StringValue { + if x != nil { + return x.ContinuationToken } + return nil } -func (x *DeleteTaskHubRequest) String() string { - return protoimpl.X.MessageStringOf(x) -} - -func (*DeleteTaskHubRequest) ProtoMessage() {} - -func (x *DeleteTaskHubRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[60] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *CleanEntityStorageResponse) GetEmptyEntitiesRemoved() int32 { + if x != nil { + return x.EmptyEntitiesRemoved } - return mi.MessageOf(x) + return 0 } -// Deprecated: Use DeleteTaskHubRequest.ProtoReflect.Descriptor instead. -func (*DeleteTaskHubRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{60} +func (x *CleanEntityStorageResponse) GetOrphanedLocksReleased() int32 { + if x != nil { + return x.OrphanedLocksReleased + } + return 0 } -type DeleteTaskHubResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +type OrchestratorEntityParameters struct { + state protoimpl.MessageState `protogen:"open.v1"` + EntityMessageReorderWindow *durationpb.Duration `protobuf:"bytes,1,opt,name=entityMessageReorderWindow,proto3" json:"entityMessageReorderWindow,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *DeleteTaskHubResponse) Reset() { - *x = DeleteTaskHubResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[61] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OrchestratorEntityParameters) Reset() { + *x = OrchestratorEntityParameters{} + mi := &file_orchestrator_service_proto_msgTypes[87] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *DeleteTaskHubResponse) String() string { +func (x *OrchestratorEntityParameters) String() string { return protoimpl.X.MessageStringOf(x) } -func (*DeleteTaskHubResponse) ProtoMessage() {} +func (*OrchestratorEntityParameters) ProtoMessage() {} -func (x *DeleteTaskHubResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[61] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OrchestratorEntityParameters) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[87] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4179,41 +6151,44 @@ func (x *DeleteTaskHubResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use DeleteTaskHubResponse.ProtoReflect.Descriptor instead. -func (*DeleteTaskHubResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{61} +// Deprecated: Use OrchestratorEntityParameters.ProtoReflect.Descriptor instead. +func (*OrchestratorEntityParameters) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{87} +} + +func (x *OrchestratorEntityParameters) GetEntityMessageReorderWindow() *durationpb.Duration { + if x != nil { + return x.EntityMessageReorderWindow + } + return nil } -type SignalEntityRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type EntityBatchRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + EntityState *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=entityState,proto3" json:"entityState,omitempty"` + Operations []*OperationRequest `protobuf:"bytes,3,rep,name=operations,proto3" json:"operations,omitempty"` + Properties map[string]*structpb.Value `protobuf:"bytes,4,rep,name=properties,proto3" json:"properties,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - RequestId string `protobuf:"bytes,4,opt,name=requestId,proto3" json:"requestId,omitempty"` - ScheduledTime *timestamp.Timestamp `protobuf:"bytes,5,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *SignalEntityRequest) Reset() { - *x = SignalEntityRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[62] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityBatchRequest) Reset() { + *x = EntityBatchRequest{} + mi := &file_orchestrator_service_proto_msgTypes[88] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *SignalEntityRequest) String() string { +func (x *EntityBatchRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SignalEntityRequest) ProtoMessage() {} +func (*EntityBatchRequest) ProtoMessage() {} -func (x *SignalEntityRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[62] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityBatchRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[88] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4223,70 +6198,69 @@ func (x *SignalEntityRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SignalEntityRequest.ProtoReflect.Descriptor instead. -func (*SignalEntityRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{62} +// Deprecated: Use EntityBatchRequest.ProtoReflect.Descriptor instead. +func (*EntityBatchRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{88} } -func (x *SignalEntityRequest) GetInstanceId() string { +func (x *EntityBatchRequest) GetInstanceId() string { if x != nil { return x.InstanceId } return "" } -func (x *SignalEntityRequest) GetName() string { - if x != nil { - return x.Name - } - return "" -} - -func (x *SignalEntityRequest) GetInput() *wrappers.StringValue { +func (x *EntityBatchRequest) GetEntityState() *wrapperspb.StringValue { if x != nil { - return x.Input + return x.EntityState } return nil } -func (x *SignalEntityRequest) GetRequestId() string { +func (x *EntityBatchRequest) GetOperations() []*OperationRequest { if x != nil { - return x.RequestId + return x.Operations } - return "" + return nil } -func (x *SignalEntityRequest) GetScheduledTime() *timestamp.Timestamp { +func (x *EntityBatchRequest) GetProperties() map[string]*structpb.Value { if x != nil { - return x.ScheduledTime + return x.Properties } return nil } -type SignalEntityResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type EntityBatchResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*OperationResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` + Actions []*OperationAction `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty"` + EntityState *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=entityState,proto3" json:"entityState,omitempty"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,4,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + CompletionToken string `protobuf:"bytes,5,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + OperationInfos []*OperationInfo `protobuf:"bytes,6,rep,name=operationInfos,proto3" json:"operationInfos,omitempty"` // used only with DTS + // Whether or not an entity state is required to complete the original EntityBatchRequest and none was provided. + RequiresState bool `protobuf:"varint,7,opt,name=requiresState,proto3" json:"requiresState,omitempty"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *SignalEntityResponse) Reset() { - *x = SignalEntityResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[63] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityBatchResult) Reset() { + *x = EntityBatchResult{} + mi := &file_orchestrator_service_proto_msgTypes[89] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *SignalEntityResponse) String() string { +func (x *EntityBatchResult) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SignalEntityResponse) ProtoMessage() {} +func (*EntityBatchResult) ProtoMessage() {} -func (x *SignalEntityResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[63] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityBatchResult) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[89] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4296,93 +6270,86 @@ func (x *SignalEntityResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SignalEntityResponse.ProtoReflect.Descriptor instead. -func (*SignalEntityResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{63} +// Deprecated: Use EntityBatchResult.ProtoReflect.Descriptor instead. +func (*EntityBatchResult) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{89} } -type GetEntityRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - IncludeState bool `protobuf:"varint,2,opt,name=includeState,proto3" json:"includeState,omitempty"` +func (x *EntityBatchResult) GetResults() []*OperationResult { + if x != nil { + return x.Results + } + return nil } -func (x *GetEntityRequest) Reset() { - *x = GetEntityRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[64] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *EntityBatchResult) GetActions() []*OperationAction { + if x != nil { + return x.Actions } + return nil } -func (x *GetEntityRequest) String() string { - return protoimpl.X.MessageStringOf(x) +func (x *EntityBatchResult) GetEntityState() *wrapperspb.StringValue { + if x != nil { + return x.EntityState + } + return nil } -func (*GetEntityRequest) ProtoMessage() {} - -func (x *GetEntityRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[64] - if protoimpl.UnsafeEnabled && x != nil { - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - if ms.LoadMessageInfo() == nil { - ms.StoreMessageInfo(mi) - } - return ms +func (x *EntityBatchResult) GetFailureDetails() *TaskFailureDetails { + if x != nil { + return x.FailureDetails } - return mi.MessageOf(x) + return nil } -// Deprecated: Use GetEntityRequest.ProtoReflect.Descriptor instead. -func (*GetEntityRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{64} +func (x *EntityBatchResult) GetCompletionToken() string { + if x != nil { + return x.CompletionToken + } + return "" } -func (x *GetEntityRequest) GetInstanceId() string { +func (x *EntityBatchResult) GetOperationInfos() []*OperationInfo { if x != nil { - return x.InstanceId + return x.OperationInfos } - return "" + return nil } -func (x *GetEntityRequest) GetIncludeState() bool { +func (x *EntityBatchResult) GetRequiresState() bool { if x != nil { - return x.IncludeState + return x.RequiresState } return false } -type GetEntityResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Exists bool `protobuf:"varint,1,opt,name=exists,proto3" json:"exists,omitempty"` - Entity *EntityMetadata `protobuf:"bytes,2,opt,name=entity,proto3" json:"entity,omitempty"` +type EntityRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + ExecutionId string `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` + EntityState *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=entityState,proto3" json:"entityState,omitempty"` // null if entity does not exist + OperationRequests []*HistoryEvent `protobuf:"bytes,4,rep,name=operationRequests,proto3" json:"operationRequests,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *GetEntityResponse) Reset() { - *x = GetEntityResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[65] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityRequest) Reset() { + *x = EntityRequest{} + mi := &file_orchestrator_service_proto_msgTypes[90] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *GetEntityResponse) String() string { +func (x *EntityRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetEntityResponse) ProtoMessage() {} +func (*EntityRequest) ProtoMessage() {} -func (x *GetEntityResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[65] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[90] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4392,57 +6359,65 @@ func (x *GetEntityResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetEntityResponse.ProtoReflect.Descriptor instead. -func (*GetEntityResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{65} +// Deprecated: Use EntityRequest.ProtoReflect.Descriptor instead. +func (*EntityRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{90} } -func (x *GetEntityResponse) GetExists() bool { +func (x *EntityRequest) GetInstanceId() string { if x != nil { - return x.Exists + return x.InstanceId } - return false + return "" } -func (x *GetEntityResponse) GetEntity() *EntityMetadata { +func (x *EntityRequest) GetExecutionId() string { if x != nil { - return x.Entity + return x.ExecutionId + } + return "" +} + +func (x *EntityRequest) GetEntityState() *wrapperspb.StringValue { + if x != nil { + return x.EntityState } return nil } -type EntityQuery struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *EntityRequest) GetOperationRequests() []*HistoryEvent { + if x != nil { + return x.OperationRequests + } + return nil +} - InstanceIdStartsWith *wrappers.StringValue `protobuf:"bytes,1,opt,name=instanceIdStartsWith,proto3" json:"instanceIdStartsWith,omitempty"` - LastModifiedFrom *timestamp.Timestamp `protobuf:"bytes,2,opt,name=lastModifiedFrom,proto3" json:"lastModifiedFrom,omitempty"` - LastModifiedTo *timestamp.Timestamp `protobuf:"bytes,3,opt,name=lastModifiedTo,proto3" json:"lastModifiedTo,omitempty"` - IncludeState bool `protobuf:"varint,4,opt,name=includeState,proto3" json:"includeState,omitempty"` - IncludeTransient bool `protobuf:"varint,5,opt,name=includeTransient,proto3" json:"includeTransient,omitempty"` - PageSize *wrappers.Int32Value `protobuf:"bytes,6,opt,name=pageSize,proto3" json:"pageSize,omitempty"` - ContinuationToken *wrappers.StringValue `protobuf:"bytes,7,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` +type OperationRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` + RequestId string `protobuf:"bytes,2,opt,name=requestId,proto3" json:"requestId,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + TraceContext *TraceContext `protobuf:"bytes,4,opt,name=traceContext,proto3" json:"traceContext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *EntityQuery) Reset() { - *x = EntityQuery{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[66] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationRequest) Reset() { + *x = OperationRequest{} + mi := &file_orchestrator_service_proto_msgTypes[91] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *EntityQuery) String() string { +func (x *OperationRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EntityQuery) ProtoMessage() {} +func (*OperationRequest) ProtoMessage() {} -func (x *EntityQuery) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[66] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[91] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4452,86 +6427,66 @@ func (x *EntityQuery) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use EntityQuery.ProtoReflect.Descriptor instead. -func (*EntityQuery) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{66} -} - -func (x *EntityQuery) GetInstanceIdStartsWith() *wrappers.StringValue { - if x != nil { - return x.InstanceIdStartsWith - } - return nil -} - -func (x *EntityQuery) GetLastModifiedFrom() *timestamp.Timestamp { - if x != nil { - return x.LastModifiedFrom - } - return nil -} - -func (x *EntityQuery) GetLastModifiedTo() *timestamp.Timestamp { - if x != nil { - return x.LastModifiedTo - } - return nil +// Deprecated: Use OperationRequest.ProtoReflect.Descriptor instead. +func (*OperationRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{91} } -func (x *EntityQuery) GetIncludeState() bool { +func (x *OperationRequest) GetOperation() string { if x != nil { - return x.IncludeState + return x.Operation } - return false + return "" } -func (x *EntityQuery) GetIncludeTransient() bool { +func (x *OperationRequest) GetRequestId() string { if x != nil { - return x.IncludeTransient + return x.RequestId } - return false + return "" } -func (x *EntityQuery) GetPageSize() *wrappers.Int32Value { +func (x *OperationRequest) GetInput() *wrapperspb.StringValue { if x != nil { - return x.PageSize + return x.Input } return nil } -func (x *EntityQuery) GetContinuationToken() *wrappers.StringValue { +func (x *OperationRequest) GetTraceContext() *TraceContext { if x != nil { - return x.ContinuationToken + return x.TraceContext } return nil } -type QueryEntitiesRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type OperationResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to ResultType: + // + // *OperationResult_Success + // *OperationResult_Failure + ResultType isOperationResult_ResultType `protobuf_oneof:"resultType"` unknownFields protoimpl.UnknownFields - - Query *EntityQuery `protobuf:"bytes,1,opt,name=query,proto3" json:"query,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *QueryEntitiesRequest) Reset() { - *x = QueryEntitiesRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[67] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationResult) Reset() { + *x = OperationResult{} + mi := &file_orchestrator_service_proto_msgTypes[92] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *QueryEntitiesRequest) String() string { +func (x *OperationResult) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryEntitiesRequest) ProtoMessage() {} +func (*OperationResult) ProtoMessage() {} -func (x *QueryEntitiesRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[67] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationResult) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[92] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4541,45 +6496,76 @@ func (x *QueryEntitiesRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryEntitiesRequest.ProtoReflect.Descriptor instead. -func (*QueryEntitiesRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{67} +// Deprecated: Use OperationResult.ProtoReflect.Descriptor instead. +func (*OperationResult) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{92} } -func (x *QueryEntitiesRequest) GetQuery() *EntityQuery { +func (x *OperationResult) GetResultType() isOperationResult_ResultType { if x != nil { - return x.Query + return x.ResultType + } + return nil +} + +func (x *OperationResult) GetSuccess() *OperationResultSuccess { + if x != nil { + if x, ok := x.ResultType.(*OperationResult_Success); ok { + return x.Success + } + } + return nil +} + +func (x *OperationResult) GetFailure() *OperationResultFailure { + if x != nil { + if x, ok := x.ResultType.(*OperationResult_Failure); ok { + return x.Failure + } } return nil } -type QueryEntitiesResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +type isOperationResult_ResultType interface { + isOperationResult_ResultType() +} + +type OperationResult_Success struct { + Success *OperationResultSuccess `protobuf:"bytes,1,opt,name=success,proto3,oneof"` +} + +type OperationResult_Failure struct { + Failure *OperationResultFailure `protobuf:"bytes,2,opt,name=failure,proto3,oneof"` +} + +func (*OperationResult_Success) isOperationResult_ResultType() {} - Entities []*EntityMetadata `protobuf:"bytes,1,rep,name=entities,proto3" json:"entities,omitempty"` - ContinuationToken *wrappers.StringValue `protobuf:"bytes,2,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` +func (*OperationResult_Failure) isOperationResult_ResultType() {} + +type OperationInfo struct { + state protoimpl.MessageState `protogen:"open.v1"` + RequestId string `protobuf:"bytes,1,opt,name=requestId,proto3" json:"requestId,omitempty"` + ResponseDestination *OrchestrationInstance `protobuf:"bytes,2,opt,name=responseDestination,proto3" json:"responseDestination,omitempty"` // null for signals + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *QueryEntitiesResponse) Reset() { - *x = QueryEntitiesResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[68] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationInfo) Reset() { + *x = OperationInfo{} + mi := &file_orchestrator_service_proto_msgTypes[93] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *QueryEntitiesResponse) String() string { +func (x *OperationInfo) String() string { return protoimpl.X.MessageStringOf(x) } -func (*QueryEntitiesResponse) ProtoMessage() {} +func (*OperationInfo) ProtoMessage() {} -func (x *QueryEntitiesResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[68] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationInfo) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[93] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4589,55 +6575,50 @@ func (x *QueryEntitiesResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use QueryEntitiesResponse.ProtoReflect.Descriptor instead. -func (*QueryEntitiesResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{68} +// Deprecated: Use OperationInfo.ProtoReflect.Descriptor instead. +func (*OperationInfo) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{93} } -func (x *QueryEntitiesResponse) GetEntities() []*EntityMetadata { +func (x *OperationInfo) GetRequestId() string { if x != nil { - return x.Entities + return x.RequestId } - return nil + return "" } -func (x *QueryEntitiesResponse) GetContinuationToken() *wrappers.StringValue { +func (x *OperationInfo) GetResponseDestination() *OrchestrationInstance { if x != nil { - return x.ContinuationToken + return x.ResponseDestination } return nil } -type EntityMetadata struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type OperationResultSuccess struct { + state protoimpl.MessageState `protogen:"open.v1"` + Result *wrapperspb.StringValue `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + StartTimeUtc *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=startTimeUtc,proto3" json:"startTimeUtc,omitempty"` + EndTimeUtc *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=endTimeUtc,proto3" json:"endTimeUtc,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - LastModifiedTime *timestamp.Timestamp `protobuf:"bytes,2,opt,name=lastModifiedTime,proto3" json:"lastModifiedTime,omitempty"` - BacklogQueueSize int32 `protobuf:"varint,3,opt,name=backlogQueueSize,proto3" json:"backlogQueueSize,omitempty"` - LockedBy *wrappers.StringValue `protobuf:"bytes,4,opt,name=lockedBy,proto3" json:"lockedBy,omitempty"` - SerializedState *wrappers.StringValue `protobuf:"bytes,5,opt,name=serializedState,proto3" json:"serializedState,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *EntityMetadata) Reset() { - *x = EntityMetadata{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[69] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationResultSuccess) Reset() { + *x = OperationResultSuccess{} + mi := &file_orchestrator_service_proto_msgTypes[94] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *EntityMetadata) String() string { +func (x *OperationResultSuccess) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EntityMetadata) ProtoMessage() {} +func (*OperationResultSuccess) ProtoMessage() {} -func (x *EntityMetadata) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[69] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationResultSuccess) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[94] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4647,74 +6628,57 @@ func (x *EntityMetadata) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use EntityMetadata.ProtoReflect.Descriptor instead. -func (*EntityMetadata) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{69} -} - -func (x *EntityMetadata) GetInstanceId() string { - if x != nil { - return x.InstanceId - } - return "" +// Deprecated: Use OperationResultSuccess.ProtoReflect.Descriptor instead. +func (*OperationResultSuccess) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{94} } -func (x *EntityMetadata) GetLastModifiedTime() *timestamp.Timestamp { +func (x *OperationResultSuccess) GetResult() *wrapperspb.StringValue { if x != nil { - return x.LastModifiedTime + return x.Result } return nil } -func (x *EntityMetadata) GetBacklogQueueSize() int32 { - if x != nil { - return x.BacklogQueueSize - } - return 0 -} - -func (x *EntityMetadata) GetLockedBy() *wrappers.StringValue { +func (x *OperationResultSuccess) GetStartTimeUtc() *timestamppb.Timestamp { if x != nil { - return x.LockedBy + return x.StartTimeUtc } return nil } -func (x *EntityMetadata) GetSerializedState() *wrappers.StringValue { +func (x *OperationResultSuccess) GetEndTimeUtc() *timestamppb.Timestamp { if x != nil { - return x.SerializedState + return x.EndTimeUtc } return nil } -type CleanEntityStorageRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ContinuationToken *wrappers.StringValue `protobuf:"bytes,1,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` - RemoveEmptyEntities bool `protobuf:"varint,2,opt,name=removeEmptyEntities,proto3" json:"removeEmptyEntities,omitempty"` - ReleaseOrphanedLocks bool `protobuf:"varint,3,opt,name=releaseOrphanedLocks,proto3" json:"releaseOrphanedLocks,omitempty"` +type OperationResultFailure struct { + state protoimpl.MessageState `protogen:"open.v1"` + FailureDetails *TaskFailureDetails `protobuf:"bytes,1,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` + StartTimeUtc *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=startTimeUtc,proto3" json:"startTimeUtc,omitempty"` + EndTimeUtc *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=endTimeUtc,proto3" json:"endTimeUtc,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CleanEntityStorageRequest) Reset() { - *x = CleanEntityStorageRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[70] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationResultFailure) Reset() { + *x = OperationResultFailure{} + mi := &file_orchestrator_service_proto_msgTypes[95] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CleanEntityStorageRequest) String() string { +func (x *OperationResultFailure) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CleanEntityStorageRequest) ProtoMessage() {} +func (*OperationResultFailure) ProtoMessage() {} -func (x *CleanEntityStorageRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[70] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationResultFailure) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[95] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4724,60 +6688,60 @@ func (x *CleanEntityStorageRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CleanEntityStorageRequest.ProtoReflect.Descriptor instead. -func (*CleanEntityStorageRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{70} +// Deprecated: Use OperationResultFailure.ProtoReflect.Descriptor instead. +func (*OperationResultFailure) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{95} } -func (x *CleanEntityStorageRequest) GetContinuationToken() *wrappers.StringValue { +func (x *OperationResultFailure) GetFailureDetails() *TaskFailureDetails { if x != nil { - return x.ContinuationToken + return x.FailureDetails } return nil } -func (x *CleanEntityStorageRequest) GetRemoveEmptyEntities() bool { +func (x *OperationResultFailure) GetStartTimeUtc() *timestamppb.Timestamp { if x != nil { - return x.RemoveEmptyEntities + return x.StartTimeUtc } - return false + return nil } -func (x *CleanEntityStorageRequest) GetReleaseOrphanedLocks() bool { +func (x *OperationResultFailure) GetEndTimeUtc() *timestamppb.Timestamp { if x != nil { - return x.ReleaseOrphanedLocks + return x.EndTimeUtc } - return false + return nil } -type CleanEntityStorageResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - ContinuationToken *wrappers.StringValue `protobuf:"bytes,1,opt,name=continuationToken,proto3" json:"continuationToken,omitempty"` - EmptyEntitiesRemoved int32 `protobuf:"varint,2,opt,name=emptyEntitiesRemoved,proto3" json:"emptyEntitiesRemoved,omitempty"` - OrphanedLocksReleased int32 `protobuf:"varint,3,opt,name=orphanedLocksReleased,proto3" json:"orphanedLocksReleased,omitempty"` +type OperationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + // Types that are valid to be assigned to OperationActionType: + // + // *OperationAction_SendSignal + // *OperationAction_StartNewOrchestration + OperationActionType isOperationAction_OperationActionType `protobuf_oneof:"operationActionType"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *CleanEntityStorageResponse) Reset() { - *x = CleanEntityStorageResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[71] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OperationAction) Reset() { + *x = OperationAction{} + mi := &file_orchestrator_service_proto_msgTypes[96] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *CleanEntityStorageResponse) String() string { +func (x *OperationAction) String() string { return protoimpl.X.MessageStringOf(x) } -func (*CleanEntityStorageResponse) ProtoMessage() {} +func (*OperationAction) ProtoMessage() {} -func (x *CleanEntityStorageResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[71] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OperationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[96] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4787,58 +6751,87 @@ func (x *CleanEntityStorageResponse) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use CleanEntityStorageResponse.ProtoReflect.Descriptor instead. -func (*CleanEntityStorageResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{71} +// Deprecated: Use OperationAction.ProtoReflect.Descriptor instead. +func (*OperationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{96} } -func (x *CleanEntityStorageResponse) GetContinuationToken() *wrappers.StringValue { +func (x *OperationAction) GetId() int32 { if x != nil { - return x.ContinuationToken + return x.Id + } + return 0 +} + +func (x *OperationAction) GetOperationActionType() isOperationAction_OperationActionType { + if x != nil { + return x.OperationActionType } return nil } -func (x *CleanEntityStorageResponse) GetEmptyEntitiesRemoved() int32 { +func (x *OperationAction) GetSendSignal() *SendSignalAction { if x != nil { - return x.EmptyEntitiesRemoved + if x, ok := x.OperationActionType.(*OperationAction_SendSignal); ok { + return x.SendSignal + } } - return 0 + return nil } -func (x *CleanEntityStorageResponse) GetOrphanedLocksReleased() int32 { +func (x *OperationAction) GetStartNewOrchestration() *StartNewOrchestrationAction { if x != nil { - return x.OrphanedLocksReleased + if x, ok := x.OperationActionType.(*OperationAction_StartNewOrchestration); ok { + return x.StartNewOrchestration + } } - return 0 + return nil } -type OrchestratorEntityParameters struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +type isOperationAction_OperationActionType interface { + isOperationAction_OperationActionType() +} + +type OperationAction_SendSignal struct { + SendSignal *SendSignalAction `protobuf:"bytes,2,opt,name=sendSignal,proto3,oneof"` +} - EntityMessageReorderWindow *duration.Duration `protobuf:"bytes,1,opt,name=entityMessageReorderWindow,proto3" json:"entityMessageReorderWindow,omitempty"` +type OperationAction_StartNewOrchestration struct { + StartNewOrchestration *StartNewOrchestrationAction `protobuf:"bytes,3,opt,name=startNewOrchestration,proto3,oneof"` } -func (x *OrchestratorEntityParameters) Reset() { - *x = OrchestratorEntityParameters{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[72] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (*OperationAction_SendSignal) isOperationAction_OperationActionType() {} + +func (*OperationAction_StartNewOrchestration) isOperationAction_OperationActionType() {} + +type SendSignalAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` + ScheduledTime *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + RequestTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=requestTime,proto3" json:"requestTime,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,6,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OrchestratorEntityParameters) String() string { +func (x *SendSignalAction) Reset() { + *x = SendSignalAction{} + mi := &file_orchestrator_service_proto_msgTypes[97] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SendSignalAction) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OrchestratorEntityParameters) ProtoMessage() {} +func (*SendSignalAction) ProtoMessage() {} -func (x *OrchestratorEntityParameters) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[72] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SendSignalAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[97] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4848,110 +6841,168 @@ func (x *OrchestratorEntityParameters) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OrchestratorEntityParameters.ProtoReflect.Descriptor instead. -func (*OrchestratorEntityParameters) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{72} +// Deprecated: Use SendSignalAction.ProtoReflect.Descriptor instead. +func (*SendSignalAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{97} +} + +func (x *SendSignalAction) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" } -func (x *OrchestratorEntityParameters) GetEntityMessageReorderWindow() *duration.Duration { +func (x *SendSignalAction) GetName() string { if x != nil { - return x.EntityMessageReorderWindow + return x.Name + } + return "" +} + +func (x *SendSignalAction) GetInput() *wrapperspb.StringValue { + if x != nil { + return x.Input } return nil } -type EntityBatchRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *SendSignalAction) GetScheduledTime() *timestamppb.Timestamp { + if x != nil { + return x.ScheduledTime + } + return nil +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - EntityState *wrappers.StringValue `protobuf:"bytes,2,opt,name=entityState,proto3" json:"entityState,omitempty"` - Operations []*OperationRequest `protobuf:"bytes,3,rep,name=operations,proto3" json:"operations,omitempty"` +func (x *SendSignalAction) GetRequestTime() *timestamppb.Timestamp { + if x != nil { + return x.RequestTime + } + return nil } -func (x *EntityBatchRequest) Reset() { - *x = EntityBatchRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[73] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) +func (x *SendSignalAction) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext } + return nil } -func (x *EntityBatchRequest) String() string { +type StartNewOrchestrationAction struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Version *wrapperspb.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` + Input *wrapperspb.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` + ScheduledTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + RequestTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=requestTime,proto3" json:"requestTime,omitempty"` + ParentTraceContext *TraceContext `protobuf:"bytes,7,opt,name=parentTraceContext,proto3" json:"parentTraceContext,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StartNewOrchestrationAction) Reset() { + *x = StartNewOrchestrationAction{} + mi := &file_orchestrator_service_proto_msgTypes[98] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StartNewOrchestrationAction) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EntityBatchRequest) ProtoMessage() {} +func (*StartNewOrchestrationAction) ProtoMessage() {} -func (x *EntityBatchRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[73] - if protoimpl.UnsafeEnabled && x != nil { +func (x *StartNewOrchestrationAction) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[98] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) } return ms } - return mi.MessageOf(x) + return mi.MessageOf(x) +} + +// Deprecated: Use StartNewOrchestrationAction.ProtoReflect.Descriptor instead. +func (*StartNewOrchestrationAction) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{98} +} + +func (x *StartNewOrchestrationAction) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *StartNewOrchestrationAction) GetName() string { + if x != nil { + return x.Name + } + return "" } -// Deprecated: Use EntityBatchRequest.ProtoReflect.Descriptor instead. -func (*EntityBatchRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{73} +func (x *StartNewOrchestrationAction) GetVersion() *wrapperspb.StringValue { + if x != nil { + return x.Version + } + return nil } -func (x *EntityBatchRequest) GetInstanceId() string { +func (x *StartNewOrchestrationAction) GetInput() *wrapperspb.StringValue { if x != nil { - return x.InstanceId + return x.Input } - return "" + return nil } -func (x *EntityBatchRequest) GetEntityState() *wrappers.StringValue { +func (x *StartNewOrchestrationAction) GetScheduledTime() *timestamppb.Timestamp { if x != nil { - return x.EntityState + return x.ScheduledTime } return nil } -func (x *EntityBatchRequest) GetOperations() []*OperationRequest { +func (x *StartNewOrchestrationAction) GetRequestTime() *timestamppb.Timestamp { if x != nil { - return x.Operations + return x.RequestTime } return nil } -type EntityBatchResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *StartNewOrchestrationAction) GetParentTraceContext() *TraceContext { + if x != nil { + return x.ParentTraceContext + } + return nil +} - Results []*OperationResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` - Actions []*OperationAction `protobuf:"bytes,2,rep,name=actions,proto3" json:"actions,omitempty"` - EntityState *wrappers.StringValue `protobuf:"bytes,3,opt,name=entityState,proto3" json:"entityState,omitempty"` - FailureDetails *TaskFailureDetails `protobuf:"bytes,4,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` +type AbandonActivityTaskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CompletionToken string `protobuf:"bytes,1,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *EntityBatchResult) Reset() { - *x = EntityBatchResult{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[74] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *AbandonActivityTaskRequest) Reset() { + *x = AbandonActivityTaskRequest{} + mi := &file_orchestrator_service_proto_msgTypes[99] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *EntityBatchResult) String() string { +func (x *AbandonActivityTaskRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*EntityBatchResult) ProtoMessage() {} +func (*AbandonActivityTaskRequest) ProtoMessage() {} -func (x *EntityBatchResult) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[74] - if protoimpl.UnsafeEnabled && x != nil { +func (x *AbandonActivityTaskRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[99] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -4961,67 +7012,77 @@ func (x *EntityBatchResult) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use EntityBatchResult.ProtoReflect.Descriptor instead. -func (*EntityBatchResult) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{74} +// Deprecated: Use AbandonActivityTaskRequest.ProtoReflect.Descriptor instead. +func (*AbandonActivityTaskRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{99} } -func (x *EntityBatchResult) GetResults() []*OperationResult { +func (x *AbandonActivityTaskRequest) GetCompletionToken() string { if x != nil { - return x.Results + return x.CompletionToken } - return nil + return "" } -func (x *EntityBatchResult) GetActions() []*OperationAction { - if x != nil { - return x.Actions - } - return nil +type AbandonActivityTaskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *EntityBatchResult) GetEntityState() *wrappers.StringValue { - if x != nil { - return x.EntityState - } - return nil +func (x *AbandonActivityTaskResponse) Reset() { + *x = AbandonActivityTaskResponse{} + mi := &file_orchestrator_service_proto_msgTypes[100] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *EntityBatchResult) GetFailureDetails() *TaskFailureDetails { +func (x *AbandonActivityTaskResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AbandonActivityTaskResponse) ProtoMessage() {} + +func (x *AbandonActivityTaskResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[100] if x != nil { - return x.FailureDetails + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -type OperationRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +// Deprecated: Use AbandonActivityTaskResponse.ProtoReflect.Descriptor instead. +func (*AbandonActivityTaskResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{100} +} - Operation string `protobuf:"bytes,1,opt,name=operation,proto3" json:"operation,omitempty"` - RequestId string `protobuf:"bytes,2,opt,name=requestId,proto3" json:"requestId,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` +type AbandonOrchestrationTaskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CompletionToken string `protobuf:"bytes,1,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OperationRequest) Reset() { - *x = OperationRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[75] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *AbandonOrchestrationTaskRequest) Reset() { + *x = AbandonOrchestrationTaskRequest{} + mi := &file_orchestrator_service_proto_msgTypes[101] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OperationRequest) String() string { +func (x *AbandonOrchestrationTaskRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationRequest) ProtoMessage() {} +func (*AbandonOrchestrationTaskRequest) ProtoMessage() {} -func (x *OperationRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[75] - if protoimpl.UnsafeEnabled && x != nil { +func (x *AbandonOrchestrationTaskRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[101] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5031,62 +7092,77 @@ func (x *OperationRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationRequest.ProtoReflect.Descriptor instead. -func (*OperationRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{75} +// Deprecated: Use AbandonOrchestrationTaskRequest.ProtoReflect.Descriptor instead. +func (*AbandonOrchestrationTaskRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{101} } -func (x *OperationRequest) GetOperation() string { +func (x *AbandonOrchestrationTaskRequest) GetCompletionToken() string { if x != nil { - return x.Operation + return x.CompletionToken } return "" } -func (x *OperationRequest) GetRequestId() string { - if x != nil { - return x.RequestId - } - return "" +type AbandonOrchestrationTaskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AbandonOrchestrationTaskResponse) Reset() { + *x = AbandonOrchestrationTaskResponse{} + mi := &file_orchestrator_service_proto_msgTypes[102] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AbandonOrchestrationTaskResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -func (x *OperationRequest) GetInput() *wrappers.StringValue { +func (*AbandonOrchestrationTaskResponse) ProtoMessage() {} + +func (x *AbandonOrchestrationTaskResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[102] if x != nil { - return x.Input + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -type OperationResult struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +// Deprecated: Use AbandonOrchestrationTaskResponse.ProtoReflect.Descriptor instead. +func (*AbandonOrchestrationTaskResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{102} +} - // Types that are assignable to ResultType: - // - // *OperationResult_Success - // *OperationResult_Failure - ResultType isOperationResult_ResultType `protobuf_oneof:"resultType"` +type AbandonEntityTaskRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + CompletionToken string `protobuf:"bytes,1,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OperationResult) Reset() { - *x = OperationResult{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[76] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *AbandonEntityTaskRequest) Reset() { + *x = AbandonEntityTaskRequest{} + mi := &file_orchestrator_service_proto_msgTypes[103] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OperationResult) String() string { +func (x *AbandonEntityTaskRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationResult) ProtoMessage() {} +func (*AbandonEntityTaskRequest) ProtoMessage() {} -func (x *OperationResult) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[76] - if protoimpl.UnsafeEnabled && x != nil { +func (x *AbandonEntityTaskRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[103] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5096,74 +7172,78 @@ func (x *OperationResult) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationResult.ProtoReflect.Descriptor instead. -func (*OperationResult) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{76} +// Deprecated: Use AbandonEntityTaskRequest.ProtoReflect.Descriptor instead. +func (*AbandonEntityTaskRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{103} } -func (m *OperationResult) GetResultType() isOperationResult_ResultType { - if m != nil { - return m.ResultType +func (x *AbandonEntityTaskRequest) GetCompletionToken() string { + if x != nil { + return x.CompletionToken } - return nil + return "" } -func (x *OperationResult) GetSuccess() *OperationResultSuccess { - if x, ok := x.GetResultType().(*OperationResult_Success); ok { - return x.Success - } - return nil +type AbandonEntityTaskResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OperationResult) GetFailure() *OperationResultFailure { - if x, ok := x.GetResultType().(*OperationResult_Failure); ok { - return x.Failure - } - return nil +func (x *AbandonEntityTaskResponse) Reset() { + *x = AbandonEntityTaskResponse{} + mi := &file_orchestrator_service_proto_msgTypes[104] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -type isOperationResult_ResultType interface { - isOperationResult_ResultType() +func (x *AbandonEntityTaskResponse) String() string { + return protoimpl.X.MessageStringOf(x) } -type OperationResult_Success struct { - Success *OperationResultSuccess `protobuf:"bytes,1,opt,name=success,proto3,oneof"` -} +func (*AbandonEntityTaskResponse) ProtoMessage() {} -type OperationResult_Failure struct { - Failure *OperationResultFailure `protobuf:"bytes,2,opt,name=failure,proto3,oneof"` +func (x *AbandonEntityTaskResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[104] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) } -func (*OperationResult_Success) isOperationResult_ResultType() {} - -func (*OperationResult_Failure) isOperationResult_ResultType() {} +// Deprecated: Use AbandonEntityTaskResponse.ProtoReflect.Descriptor instead. +func (*AbandonEntityTaskResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{104} +} -type OperationResultSuccess struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type SkipGracefulOrchestrationTerminationsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceBatch *InstanceBatch `protobuf:"bytes,1,opt,name=instanceBatch,proto3" json:"instanceBatch,omitempty"` + Reason *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` unknownFields protoimpl.UnknownFields - - Result *wrappers.StringValue `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *OperationResultSuccess) Reset() { - *x = OperationResultSuccess{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[77] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *SkipGracefulOrchestrationTerminationsRequest) Reset() { + *x = SkipGracefulOrchestrationTerminationsRequest{} + mi := &file_orchestrator_service_proto_msgTypes[105] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OperationResultSuccess) String() string { +func (x *SkipGracefulOrchestrationTerminationsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationResultSuccess) ProtoMessage() {} +func (*SkipGracefulOrchestrationTerminationsRequest) ProtoMessage() {} -func (x *OperationResultSuccess) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[77] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SkipGracefulOrchestrationTerminationsRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[105] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5173,44 +7253,50 @@ func (x *OperationResultSuccess) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationResultSuccess.ProtoReflect.Descriptor instead. -func (*OperationResultSuccess) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{77} +// Deprecated: Use SkipGracefulOrchestrationTerminationsRequest.ProtoReflect.Descriptor instead. +func (*SkipGracefulOrchestrationTerminationsRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{105} } -func (x *OperationResultSuccess) GetResult() *wrappers.StringValue { +func (x *SkipGracefulOrchestrationTerminationsRequest) GetInstanceBatch() *InstanceBatch { if x != nil { - return x.Result + return x.InstanceBatch } return nil } -type OperationResultFailure struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *SkipGracefulOrchestrationTerminationsRequest) GetReason() *wrapperspb.StringValue { + if x != nil { + return x.Reason + } + return nil +} - FailureDetails *TaskFailureDetails `protobuf:"bytes,1,opt,name=failureDetails,proto3" json:"failureDetails,omitempty"` +type SkipGracefulOrchestrationTerminationsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Those instances which could not be terminated because they had locked entities at the time of this termination call, + // are already in a terminal state (completed, failed, terminated, etc.), are not orchestrations, or do not exist (i.e. have been purged) + UnterminatedInstanceIds []string `protobuf:"bytes,1,rep,name=unterminatedInstanceIds,proto3" json:"unterminatedInstanceIds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OperationResultFailure) Reset() { - *x = OperationResultFailure{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[78] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *SkipGracefulOrchestrationTerminationsResponse) Reset() { + *x = SkipGracefulOrchestrationTerminationsResponse{} + mi := &file_orchestrator_service_proto_msgTypes[106] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OperationResultFailure) String() string { +func (x *SkipGracefulOrchestrationTerminationsResponse) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationResultFailure) ProtoMessage() {} +func (*SkipGracefulOrchestrationTerminationsResponse) ProtoMessage() {} -func (x *OperationResultFailure) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[78] - if protoimpl.UnsafeEnabled && x != nil { +func (x *SkipGracefulOrchestrationTerminationsResponse) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[106] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5220,49 +7306,45 @@ func (x *OperationResultFailure) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationResultFailure.ProtoReflect.Descriptor instead. -func (*OperationResultFailure) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{78} +// Deprecated: Use SkipGracefulOrchestrationTerminationsResponse.ProtoReflect.Descriptor instead. +func (*SkipGracefulOrchestrationTerminationsResponse) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{106} } -func (x *OperationResultFailure) GetFailureDetails() *TaskFailureDetails { +func (x *SkipGracefulOrchestrationTerminationsResponse) GetUnterminatedInstanceIds() []string { if x != nil { - return x.FailureDetails + return x.UnterminatedInstanceIds } return nil } -type OperationAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields - - Id int32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` - // Types that are assignable to OperationActionType: - // - // *OperationAction_SendSignal - // *OperationAction_StartNewOrchestration - OperationActionType isOperationAction_OperationActionType `protobuf_oneof:"operationActionType"` +type GetWorkItemsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + MaxConcurrentOrchestrationWorkItems int32 `protobuf:"varint,1,opt,name=maxConcurrentOrchestrationWorkItems,proto3" json:"maxConcurrentOrchestrationWorkItems,omitempty"` + MaxConcurrentActivityWorkItems int32 `protobuf:"varint,2,opt,name=maxConcurrentActivityWorkItems,proto3" json:"maxConcurrentActivityWorkItems,omitempty"` + MaxConcurrentEntityWorkItems int32 `protobuf:"varint,3,opt,name=maxConcurrentEntityWorkItems,proto3" json:"maxConcurrentEntityWorkItems,omitempty"` + Capabilities []WorkerCapability `protobuf:"varint,10,rep,packed,name=capabilities,proto3,enum=WorkerCapability" json:"capabilities,omitempty"` + WorkItemFilters *WorkItemFilters `protobuf:"bytes,11,opt,name=workItemFilters,proto3" json:"workItemFilters,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *OperationAction) Reset() { - *x = OperationAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[79] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *GetWorkItemsRequest) Reset() { + *x = GetWorkItemsRequest{} + mi := &file_orchestrator_service_proto_msgTypes[107] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *OperationAction) String() string { +func (x *GetWorkItemsRequest) String() string { return protoimpl.X.MessageStringOf(x) } -func (*OperationAction) ProtoMessage() {} +func (*GetWorkItemsRequest) ProtoMessage() {} -func (x *OperationAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[79] - if protoimpl.UnsafeEnabled && x != nil { +func (x *GetWorkItemsRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[107] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5272,84 +7354,71 @@ func (x *OperationAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use OperationAction.ProtoReflect.Descriptor instead. -func (*OperationAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{79} +// Deprecated: Use GetWorkItemsRequest.ProtoReflect.Descriptor instead. +func (*GetWorkItemsRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{107} } -func (x *OperationAction) GetId() int32 { +func (x *GetWorkItemsRequest) GetMaxConcurrentOrchestrationWorkItems() int32 { if x != nil { - return x.Id + return x.MaxConcurrentOrchestrationWorkItems } return 0 } -func (m *OperationAction) GetOperationActionType() isOperationAction_OperationActionType { - if m != nil { - return m.OperationActionType +func (x *GetWorkItemsRequest) GetMaxConcurrentActivityWorkItems() int32 { + if x != nil { + return x.MaxConcurrentActivityWorkItems } - return nil + return 0 } -func (x *OperationAction) GetSendSignal() *SendSignalAction { - if x, ok := x.GetOperationActionType().(*OperationAction_SendSignal); ok { - return x.SendSignal +func (x *GetWorkItemsRequest) GetMaxConcurrentEntityWorkItems() int32 { + if x != nil { + return x.MaxConcurrentEntityWorkItems } - return nil + return 0 } -func (x *OperationAction) GetStartNewOrchestration() *StartNewOrchestrationAction { - if x, ok := x.GetOperationActionType().(*OperationAction_StartNewOrchestration); ok { - return x.StartNewOrchestration +func (x *GetWorkItemsRequest) GetCapabilities() []WorkerCapability { + if x != nil { + return x.Capabilities } return nil } -type isOperationAction_OperationActionType interface { - isOperationAction_OperationActionType() -} - -type OperationAction_SendSignal struct { - SendSignal *SendSignalAction `protobuf:"bytes,2,opt,name=sendSignal,proto3,oneof"` -} - -type OperationAction_StartNewOrchestration struct { - StartNewOrchestration *StartNewOrchestrationAction `protobuf:"bytes,3,opt,name=startNewOrchestration,proto3,oneof"` -} - -func (*OperationAction_SendSignal) isOperationAction_OperationActionType() {} - -func (*OperationAction_StartNewOrchestration) isOperationAction_OperationActionType() {} - -type SendSignalAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *GetWorkItemsRequest) GetWorkItemFilters() *WorkItemFilters { + if x != nil { + return x.WorkItemFilters + } + return nil +} - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,3,opt,name=input,proto3" json:"input,omitempty"` - ScheduledTime *timestamp.Timestamp `protobuf:"bytes,4,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` +type WorkItemFilters struct { + state protoimpl.MessageState `protogen:"open.v1"` + Orchestrations []*OrchestrationFilter `protobuf:"bytes,1,rep,name=orchestrations,proto3" json:"orchestrations,omitempty"` + Activities []*ActivityFilter `protobuf:"bytes,2,rep,name=activities,proto3" json:"activities,omitempty"` + Entities []*EntityFilter `protobuf:"bytes,3,rep,name=entities,proto3" json:"entities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *SendSignalAction) Reset() { - *x = SendSignalAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[80] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *WorkItemFilters) Reset() { + *x = WorkItemFilters{} + mi := &file_orchestrator_service_proto_msgTypes[108] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *SendSignalAction) String() string { +func (x *WorkItemFilters) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SendSignalAction) ProtoMessage() {} +func (*WorkItemFilters) ProtoMessage() {} -func (x *SendSignalAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[80] - if protoimpl.UnsafeEnabled && x != nil { +func (x *WorkItemFilters) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[108] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5359,69 +7428,56 @@ func (x *SendSignalAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use SendSignalAction.ProtoReflect.Descriptor instead. -func (*SendSignalAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{80} -} - -func (x *SendSignalAction) GetInstanceId() string { - if x != nil { - return x.InstanceId - } - return "" +// Deprecated: Use WorkItemFilters.ProtoReflect.Descriptor instead. +func (*WorkItemFilters) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{108} } -func (x *SendSignalAction) GetName() string { +func (x *WorkItemFilters) GetOrchestrations() []*OrchestrationFilter { if x != nil { - return x.Name + return x.Orchestrations } - return "" + return nil } -func (x *SendSignalAction) GetInput() *wrappers.StringValue { +func (x *WorkItemFilters) GetActivities() []*ActivityFilter { if x != nil { - return x.Input + return x.Activities } return nil } -func (x *SendSignalAction) GetScheduledTime() *timestamp.Timestamp { +func (x *WorkItemFilters) GetEntities() []*EntityFilter { if x != nil { - return x.ScheduledTime + return x.Entities } return nil } -type StartNewOrchestrationAction struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type OrchestrationFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Versions []string `protobuf:"bytes,2,rep,name=versions,proto3" json:"versions,omitempty"` unknownFields protoimpl.UnknownFields - - InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` - Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` - Version *wrappers.StringValue `protobuf:"bytes,3,opt,name=version,proto3" json:"version,omitempty"` - Input *wrappers.StringValue `protobuf:"bytes,4,opt,name=input,proto3" json:"input,omitempty"` - ScheduledTime *timestamp.Timestamp `protobuf:"bytes,5,opt,name=scheduledTime,proto3" json:"scheduledTime,omitempty"` + sizeCache protoimpl.SizeCache } -func (x *StartNewOrchestrationAction) Reset() { - *x = StartNewOrchestrationAction{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[81] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *OrchestrationFilter) Reset() { + *x = OrchestrationFilter{} + mi := &file_orchestrator_service_proto_msgTypes[109] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *StartNewOrchestrationAction) String() string { +func (x *OrchestrationFilter) String() string { return protoimpl.X.MessageStringOf(x) } -func (*StartNewOrchestrationAction) ProtoMessage() {} +func (*OrchestrationFilter) ProtoMessage() {} -func (x *StartNewOrchestrationAction) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[81] - if protoimpl.UnsafeEnabled && x != nil { +func (x *OrchestrationFilter) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[109] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5431,70 +7487,100 @@ func (x *StartNewOrchestrationAction) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use StartNewOrchestrationAction.ProtoReflect.Descriptor instead. -func (*StartNewOrchestrationAction) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{81} +// Deprecated: Use OrchestrationFilter.ProtoReflect.Descriptor instead. +func (*OrchestrationFilter) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{109} } -func (x *StartNewOrchestrationAction) GetInstanceId() string { +func (x *OrchestrationFilter) GetName() string { if x != nil { - return x.InstanceId + return x.Name } return "" } -func (x *StartNewOrchestrationAction) GetName() string { +func (x *OrchestrationFilter) GetVersions() []string { if x != nil { - return x.Name + return x.Versions } - return "" + return nil +} + +type ActivityFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Versions []string `protobuf:"bytes,2,rep,name=versions,proto3" json:"versions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActivityFilter) Reset() { + *x = ActivityFilter{} + mi := &file_orchestrator_service_proto_msgTypes[110] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *StartNewOrchestrationAction) GetVersion() *wrappers.StringValue { +func (x *ActivityFilter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActivityFilter) ProtoMessage() {} + +func (x *ActivityFilter) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[110] if x != nil { - return x.Version + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms } - return nil + return mi.MessageOf(x) } -func (x *StartNewOrchestrationAction) GetInput() *wrappers.StringValue { +// Deprecated: Use ActivityFilter.ProtoReflect.Descriptor instead. +func (*ActivityFilter) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{110} +} + +func (x *ActivityFilter) GetName() string { if x != nil { - return x.Input + return x.Name } - return nil + return "" } -func (x *StartNewOrchestrationAction) GetScheduledTime() *timestamp.Timestamp { +func (x *ActivityFilter) GetVersions() []string { if x != nil { - return x.ScheduledTime + return x.Versions } return nil } -type GetWorkItemsRequest struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache +type EntityFilter struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } -func (x *GetWorkItemsRequest) Reset() { - *x = GetWorkItemsRequest{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[82] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } +func (x *EntityFilter) Reset() { + *x = EntityFilter{} + mi := &file_orchestrator_service_proto_msgTypes[111] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } -func (x *GetWorkItemsRequest) String() string { +func (x *EntityFilter) String() string { return protoimpl.X.MessageStringOf(x) } -func (*GetWorkItemsRequest) ProtoMessage() {} +func (*EntityFilter) ProtoMessage() {} -func (x *GetWorkItemsRequest) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[82] - if protoimpl.UnsafeEnabled && x != nil { +func (x *EntityFilter) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[111] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5504,31 +7590,38 @@ func (x *GetWorkItemsRequest) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -// Deprecated: Use GetWorkItemsRequest.ProtoReflect.Descriptor instead. -func (*GetWorkItemsRequest) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{82} +// Deprecated: Use EntityFilter.ProtoReflect.Descriptor instead. +func (*EntityFilter) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{111} } -type WorkItem struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache - unknownFields protoimpl.UnknownFields +func (x *EntityFilter) GetName() string { + if x != nil { + return x.Name + } + return "" +} - // Types that are assignable to Request: +type WorkItem struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Request: // // *WorkItem_OrchestratorRequest // *WorkItem_ActivityRequest // *WorkItem_EntityRequest - Request isWorkItem_Request `protobuf_oneof:"request"` + // *WorkItem_HealthPing + // *WorkItem_EntityRequestV2 + Request isWorkItem_Request `protobuf_oneof:"request"` + CompletionToken string `protobuf:"bytes,10,opt,name=completionToken,proto3" json:"completionToken,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *WorkItem) Reset() { *x = WorkItem{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[83] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[112] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *WorkItem) String() string { @@ -5538,8 +7631,8 @@ func (x *WorkItem) String() string { func (*WorkItem) ProtoMessage() {} func (x *WorkItem) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[83] - if protoimpl.UnsafeEnabled && x != nil { + mi := &file_orchestrator_service_proto_msgTypes[112] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5551,37 +7644,68 @@ func (x *WorkItem) ProtoReflect() protoreflect.Message { // Deprecated: Use WorkItem.ProtoReflect.Descriptor instead. func (*WorkItem) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{83} + return file_orchestrator_service_proto_rawDescGZIP(), []int{112} } -func (m *WorkItem) GetRequest() isWorkItem_Request { - if m != nil { - return m.Request +func (x *WorkItem) GetRequest() isWorkItem_Request { + if x != nil { + return x.Request } return nil } func (x *WorkItem) GetOrchestratorRequest() *OrchestratorRequest { - if x, ok := x.GetRequest().(*WorkItem_OrchestratorRequest); ok { - return x.OrchestratorRequest + if x != nil { + if x, ok := x.Request.(*WorkItem_OrchestratorRequest); ok { + return x.OrchestratorRequest + } } return nil } func (x *WorkItem) GetActivityRequest() *ActivityRequest { - if x, ok := x.GetRequest().(*WorkItem_ActivityRequest); ok { - return x.ActivityRequest + if x != nil { + if x, ok := x.Request.(*WorkItem_ActivityRequest); ok { + return x.ActivityRequest + } } return nil } func (x *WorkItem) GetEntityRequest() *EntityBatchRequest { - if x, ok := x.GetRequest().(*WorkItem_EntityRequest); ok { - return x.EntityRequest + if x != nil { + if x, ok := x.Request.(*WorkItem_EntityRequest); ok { + return x.EntityRequest + } + } + return nil +} + +func (x *WorkItem) GetHealthPing() *HealthPing { + if x != nil { + if x, ok := x.Request.(*WorkItem_HealthPing); ok { + return x.HealthPing + } + } + return nil +} + +func (x *WorkItem) GetEntityRequestV2() *EntityRequest { + if x != nil { + if x, ok := x.Request.(*WorkItem_EntityRequestV2); ok { + return x.EntityRequestV2 + } } return nil } +func (x *WorkItem) GetCompletionToken() string { + if x != nil { + return x.CompletionToken + } + return "" +} + type isWorkItem_Request interface { isWorkItem_Request() } @@ -5595,7 +7719,15 @@ type WorkItem_ActivityRequest struct { } type WorkItem_EntityRequest struct { - EntityRequest *EntityBatchRequest `protobuf:"bytes,3,opt,name=entityRequest,proto3,oneof"` + EntityRequest *EntityBatchRequest `protobuf:"bytes,3,opt,name=entityRequest,proto3,oneof"` // (older) used by orchestration services implementations +} + +type WorkItem_HealthPing struct { + HealthPing *HealthPing `protobuf:"bytes,4,opt,name=healthPing,proto3,oneof"` +} + +type WorkItem_EntityRequestV2 struct { + EntityRequestV2 *EntityRequest `protobuf:"bytes,5,opt,name=entityRequestV2,proto3,oneof"` // (newer) used by backend service implementations } func (*WorkItem_OrchestratorRequest) isWorkItem_Request() {} @@ -5604,19 +7736,21 @@ func (*WorkItem_ActivityRequest) isWorkItem_Request() {} func (*WorkItem_EntityRequest) isWorkItem_Request() {} +func (*WorkItem_HealthPing) isWorkItem_Request() {} + +func (*WorkItem_EntityRequestV2) isWorkItem_Request() {} + type CompleteTaskResponse struct { - state protoimpl.MessageState - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"open.v1"` unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CompleteTaskResponse) Reset() { *x = CompleteTaskResponse{} - if protoimpl.UnsafeEnabled { - mi := &file_orchestrator_service_proto_msgTypes[84] - ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) - ms.StoreMessageInfo(mi) - } + mi := &file_orchestrator_service_proto_msgTypes[113] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) } func (x *CompleteTaskResponse) String() string { @@ -5626,8 +7760,8 @@ func (x *CompleteTaskResponse) String() string { func (*CompleteTaskResponse) ProtoMessage() {} func (x *CompleteTaskResponse) ProtoReflect() protoreflect.Message { - mi := &file_orchestrator_service_proto_msgTypes[84] - if protoimpl.UnsafeEnabled && x != nil { + mi := &file_orchestrator_service_proto_msgTypes[113] + if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { ms.StoreMessageInfo(mi) @@ -5639,1380 +7773,1371 @@ func (x *CompleteTaskResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use CompleteTaskResponse.ProtoReflect.Descriptor instead. func (*CompleteTaskResponse) Descriptor() ([]byte, []int) { - return file_orchestrator_service_proto_rawDescGZIP(), []int{84} + return file_orchestrator_service_proto_rawDescGZIP(), []int{113} } -var File_orchestrator_service_proto protoreflect.FileDescriptor +type HealthPing struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HealthPing) Reset() { + *x = HealthPing{} + mi := &file_orchestrator_service_proto_msgTypes[114] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HealthPing) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HealthPing) ProtoMessage() {} + +func (x *HealthPing) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[114] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HealthPing.ProtoReflect.Descriptor instead. +func (*HealthPing) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{114} +} + +type StreamInstanceHistoryRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + InstanceId string `protobuf:"bytes,1,opt,name=instanceId,proto3" json:"instanceId,omitempty"` + ExecutionId *wrapperspb.StringValue `protobuf:"bytes,2,opt,name=executionId,proto3" json:"executionId,omitempty"` + // When set to true, the service may return a more optimized response suitable for workers. + ForWorkItemProcessing bool `protobuf:"varint,3,opt,name=forWorkItemProcessing,proto3" json:"forWorkItemProcessing,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *StreamInstanceHistoryRequest) Reset() { + *x = StreamInstanceHistoryRequest{} + mi := &file_orchestrator_service_proto_msgTypes[115] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StreamInstanceHistoryRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StreamInstanceHistoryRequest) ProtoMessage() {} + +func (x *StreamInstanceHistoryRequest) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[115] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StreamInstanceHistoryRequest.ProtoReflect.Descriptor instead. +func (*StreamInstanceHistoryRequest) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{115} +} + +func (x *StreamInstanceHistoryRequest) GetInstanceId() string { + if x != nil { + return x.InstanceId + } + return "" +} + +func (x *StreamInstanceHistoryRequest) GetExecutionId() *wrapperspb.StringValue { + if x != nil { + return x.ExecutionId + } + return nil +} + +func (x *StreamInstanceHistoryRequest) GetForWorkItemProcessing() bool { + if x != nil { + return x.ForWorkItemProcessing + } + return false +} + +type HistoryChunk struct { + state protoimpl.MessageState `protogen:"open.v1"` + Events []*HistoryEvent `protobuf:"bytes,1,rep,name=events,proto3" json:"events,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HistoryChunk) Reset() { + *x = HistoryChunk{} + mi := &file_orchestrator_service_proto_msgTypes[116] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HistoryChunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HistoryChunk) ProtoMessage() {} + +func (x *HistoryChunk) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[116] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HistoryChunk.ProtoReflect.Descriptor instead. +func (*HistoryChunk) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{116} +} + +func (x *HistoryChunk) GetEvents() []*HistoryEvent { + if x != nil { + return x.Events + } + return nil +} + +type InstanceBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + // A maximum of 500 instance IDs can be provided in this list. + InstanceIds []string `protobuf:"bytes,1,rep,name=instanceIds,proto3" json:"instanceIds,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InstanceBatch) Reset() { + *x = InstanceBatch{} + mi := &file_orchestrator_service_proto_msgTypes[117] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InstanceBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InstanceBatch) ProtoMessage() {} + +func (x *InstanceBatch) ProtoReflect() protoreflect.Message { + mi := &file_orchestrator_service_proto_msgTypes[117] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InstanceBatch.ProtoReflect.Descriptor instead. +func (*InstanceBatch) Descriptor() ([]byte, []int) { + return file_orchestrator_service_proto_rawDescGZIP(), []int{117} +} -var file_orchestrator_service_proto_rawDesc = []byte{ - 0x0a, 0x1a, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x64, - 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x77, - 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1b, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x65, - 0x6d, 0x70, 0x74, 0x79, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x77, 0x0a, 0x15, 0x4f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x12, 0x3e, 0x0a, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x22, 0xf7, 0x01, 0x0a, 0x0f, 0x41, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x4c, 0x0a, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x15, - 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x22, 0xbd, 0x01, - 0x0a, 0x10, 0x41, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x49, 0x64, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x06, 0x74, 0x61, 0x73, 0x6b, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x12, 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, - 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0xf5, 0x01, - 0x0a, 0x12, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x54, 0x79, 0x70, - 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x22, 0x0a, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x65, 0x73, 0x73, 0x61, - 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x12, 0x3c, 0x0a, 0x0a, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x12, 0x37, 0x0a, 0x0c, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x46, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, - 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, - 0x0c, 0x69, 0x6e, 0x6e, 0x65, 0x72, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x26, 0x0a, - 0x0e, 0x69, 0x73, 0x4e, 0x6f, 0x6e, 0x52, 0x65, 0x74, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x69, 0x73, 0x4e, 0x6f, 0x6e, 0x52, 0x65, 0x74, 0x72, - 0x69, 0x61, 0x62, 0x6c, 0x65, 0x22, 0xf6, 0x01, 0x0a, 0x12, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x28, 0x0a, 0x0f, - 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x12, 0x30, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, - 0x12, 0x4c, 0x0a, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x16, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x22, 0x8a, - 0x01, 0x0a, 0x0c, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, - 0x20, 0x0a, 0x0b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x74, 0x72, 0x61, 0x63, 0x65, 0x50, 0x61, 0x72, 0x65, 0x6e, - 0x74, 0x12, 0x1a, 0x0a, 0x06, 0x73, 0x70, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x09, 0x42, 0x02, 0x18, 0x01, 0x52, 0x06, 0x73, 0x70, 0x61, 0x6e, 0x49, 0x44, 0x12, 0x3c, 0x0a, - 0x0a, 0x74, 0x72, 0x61, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x0a, 0x74, 0x72, 0x61, 0x63, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x87, 0x04, 0x0a, 0x15, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x4c, 0x0a, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x04, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x15, 0x6f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x12, 0x3b, 0x0a, 0x0e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x50, 0x61, - 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, - 0x52, 0x0e, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x12, 0x54, 0x0a, 0x17, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, - 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x17, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x3d, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x07, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, - 0x74, 0x52, 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, - 0x6e, 0x74, 0x65, 0x78, 0x74, 0x12, 0x4e, 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x70, 0x61, 0x6e, 0x49, 0x44, 0x18, 0x08, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x70, 0x61, 0x6e, 0x49, 0x44, 0x22, 0xd4, 0x01, 0x0a, 0x17, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x12, 0x46, 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, - 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, - 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x68, 0x0a, 0x18, - 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, - 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x18, 0x0a, 0x07, - 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x07, 0x72, - 0x65, 0x63, 0x75, 0x72, 0x73, 0x65, 0x22, 0xd3, 0x01, 0x0a, 0x12, 0x54, 0x61, 0x73, 0x6b, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x3d, 0x0a, - 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, - 0x65, 0x78, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x54, 0x72, 0x61, 0x63, - 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x52, 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, - 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x22, 0x74, 0x0a, 0x12, - 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x64, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x74, 0x61, 0x73, - 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, - 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x22, 0x78, 0x0a, 0x0f, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, - 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x12, - 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x66, 0x61, - 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x85, 0x02, 0x0a, - 0x24, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, 0x72, - 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, - 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, - 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x3d, 0x0a, 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, - 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0d, 0x2e, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, - 0x52, 0x12, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x54, 0x72, 0x61, 0x63, 0x65, 0x43, 0x6f, 0x6e, - 0x74, 0x65, 0x78, 0x74, 0x22, 0x88, 0x01, 0x0a, 0x26, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, - 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x28, 0x0a, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, - 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, - 0x8c, 0x01, 0x0a, 0x23, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x46, 0x61, 0x69, 0x6c, - 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x0f, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x49, - 0x64, 0x12, 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, - 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, - 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x47, - 0x0a, 0x11, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x06, 0x66, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, - 0x06, 0x66, 0x69, 0x72, 0x65, 0x41, 0x74, 0x22, 0x5f, 0x0a, 0x0f, 0x54, 0x69, 0x6d, 0x65, 0x72, - 0x46, 0x69, 0x72, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x06, 0x66, 0x69, - 0x72, 0x65, 0x41, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x06, 0x66, 0x69, 0x72, 0x65, 0x41, 0x74, 0x12, 0x18, - 0x0a, 0x07, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, - 0x07, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x49, 0x64, 0x22, 0x1a, 0x0a, 0x18, 0x4f, 0x72, 0x63, 0x68, - 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x22, 0x1c, 0x0a, 0x1a, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x22, 0x78, 0x0a, 0x0e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x5a, 0x0a, 0x10, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x61, 0x69, 0x73, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x40, 0x0a, 0x0c, 0x47, 0x65, 0x6e, 0x65, - 0x72, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x30, 0x0a, 0x04, 0x64, 0x61, 0x74, 0x61, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x58, 0x0a, 0x11, 0x48, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x43, 0x0a, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x4f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x52, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x22, 0x48, 0x0a, 0x12, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, - 0x41, 0x73, 0x4e, 0x65, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x4d, - 0x0a, 0x17, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x75, 0x73, 0x70, 0x65, - 0x6e, 0x64, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, - 0x75, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x4b, 0x0a, - 0x15, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, - 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x92, 0x0c, 0x0a, 0x0c, 0x48, - 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x65, - 0x76, 0x65, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x07, 0x65, 0x76, - 0x65, 0x6e, 0x74, 0x49, 0x64, 0x12, 0x38, 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, - 0x44, 0x0a, 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x45, 0x78, 0x65, 0x63, - 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, - 0x74, 0x48, 0x00, 0x52, 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x18, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x12, 0x65, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x12, 0x4d, 0x0a, 0x13, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, - 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, - 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, - 0x61, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x13, 0x65, 0x78, 0x65, - 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x64, - 0x12, 0x3b, 0x0a, 0x0d, 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, - 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x53, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0d, - 0x74, 0x61, 0x73, 0x6b, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x12, 0x3b, 0x0a, - 0x0d, 0x74, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, - 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x74, 0x61, 0x73, - 0x6b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x74, 0x61, - 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, - 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x48, 0x00, 0x52, 0x0a, 0x74, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x71, - 0x0a, 0x1f, 0x73, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, - 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, - 0x52, 0x1f, 0x73, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x12, 0x77, 0x0a, 0x21, 0x73, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x6d, - 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x27, 0x2e, 0x53, - 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x21, 0x73, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, - 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x6e, 0x0a, 0x1e, 0x73, 0x75, - 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x18, 0x0b, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x46, 0x61, 0x69, - 0x6c, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x1e, 0x73, 0x75, 0x62, 0x4f, - 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x12, 0x38, 0x0a, 0x0c, 0x74, 0x69, - 0x6d, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x12, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x43, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x12, 0x32, 0x0a, 0x0a, 0x74, 0x69, 0x6d, 0x65, 0x72, 0x46, 0x69, 0x72, - 0x65, 0x64, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x72, - 0x46, 0x69, 0x72, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0a, 0x74, 0x69, - 0x6d, 0x65, 0x72, 0x46, 0x69, 0x72, 0x65, 0x64, 0x12, 0x4d, 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, - 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x18, - 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x48, 0x00, 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64, 0x12, 0x53, 0x0a, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, - 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x15, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x6f, 0x72, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x12, 0x2f, 0x0a, 0x09, - 0x65, 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x0f, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x48, 0x00, 0x52, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x53, 0x65, 0x6e, 0x74, 0x12, 0x35, 0x0a, - 0x0b, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x61, 0x69, 0x73, 0x65, 0x64, 0x18, 0x11, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x61, 0x69, 0x73, 0x65, 0x64, - 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x61, - 0x69, 0x73, 0x65, 0x64, 0x12, 0x33, 0x0a, 0x0c, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x47, 0x65, 0x6e, - 0x65, 0x72, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x67, 0x65, 0x6e, - 0x65, 0x72, 0x69, 0x63, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x38, 0x0a, 0x0c, 0x68, 0x69, 0x73, - 0x74, 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x13, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x12, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x45, 0x76, - 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x0c, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x12, 0x3b, 0x0a, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x41, - 0x73, 0x4e, 0x65, 0x77, 0x18, 0x14, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x43, 0x6f, 0x6e, - 0x74, 0x69, 0x6e, 0x75, 0x65, 0x41, 0x73, 0x4e, 0x65, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, - 0x00, 0x52, 0x0d, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x65, 0x41, 0x73, 0x4e, 0x65, 0x77, - 0x12, 0x4a, 0x0a, 0x12, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x75, 0x73, - 0x70, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x18, 0x15, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x18, 0x2e, 0x45, - 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x75, 0x73, 0x70, 0x65, 0x6e, 0x64, 0x65, - 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, 0x52, 0x12, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x75, 0x73, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x64, 0x12, 0x44, 0x0a, 0x10, - 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x64, - 0x18, 0x16, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x45, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x48, 0x00, - 0x52, 0x10, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6d, - 0x65, 0x64, 0x42, 0x0b, 0x0a, 0x09, 0x65, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, - 0x94, 0x01, 0x0a, 0x12, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, - 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0xbe, 0x01, 0x0a, 0x1c, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x47, 0x0a, 0x11, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x06, - 0x66, 0x69, 0x72, 0x65, 0x41, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x06, 0x66, 0x69, 0x72, 0x65, 0x41, 0x74, - 0x22, 0x8b, 0x01, 0x0a, 0x0f, 0x53, 0x65, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x41, 0x63, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x08, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x08, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x30, 0x0a, 0x04, - 0x64, 0x61, 0x74, 0x61, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x04, 0x64, 0x61, 0x74, 0x61, 0x22, 0x87, - 0x03, 0x0a, 0x1b, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x46, - 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x4f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x36, 0x0a, 0x07, - 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x64, 0x65, 0x74, - 0x61, 0x69, 0x6c, 0x73, 0x12, 0x3c, 0x0a, 0x0a, 0x6e, 0x65, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0a, 0x6e, 0x65, 0x77, 0x56, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x37, 0x0a, 0x0f, 0x63, 0x61, 0x72, 0x72, 0x79, 0x6f, 0x76, 0x65, 0x72, 0x45, - 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x48, 0x69, - 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x0f, 0x63, 0x61, 0x72, 0x72, - 0x79, 0x6f, 0x76, 0x65, 0x72, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x3b, 0x0a, 0x0e, 0x66, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x06, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, - 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, - 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x8e, 0x01, 0x0a, 0x1c, 0x54, 0x65, 0x72, - 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x61, - 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x12, - 0x18, 0x0a, 0x07, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x07, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x65, 0x22, 0xeb, 0x03, 0x0a, 0x12, 0x4f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, - 0x12, 0x39, 0x0a, 0x0c, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x53, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x54, 0x61, 0x73, 0x6b, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x0c, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x57, 0x0a, 0x16, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x16, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x53, 0x75, 0x62, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x36, 0x0a, 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, - 0x6d, 0x65, 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x43, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, - 0x0b, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x69, 0x6d, 0x65, 0x72, 0x12, 0x30, 0x0a, 0x09, - 0x73, 0x65, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x10, 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x48, 0x00, 0x52, 0x09, 0x73, 0x65, 0x6e, 0x64, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, 0x54, - 0x0a, 0x15, 0x63, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, - 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, - 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x15, 0x63, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x57, 0x0a, 0x16, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, - 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x07, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, - 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x16, 0x74, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, - 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x18, 0x0a, - 0x16, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x41, 0x63, 0x74, - 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0x9c, 0x02, 0x0a, 0x13, 0x4f, 0x72, 0x63, 0x68, - 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, - 0x3e, 0x0a, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x0b, 0x65, 0x78, 0x65, 0x63, 0x75, 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x12, - 0x2d, 0x0a, 0x0a, 0x70, 0x61, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, - 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, - 0x6e, 0x74, 0x52, 0x0a, 0x70, 0x61, 0x73, 0x74, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, - 0x0a, 0x09, 0x6e, 0x65, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x0d, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x72, 0x79, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x52, 0x09, 0x6e, 0x65, 0x77, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x49, 0x0a, 0x10, 0x65, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x18, - 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, - 0x74, 0x65, 0x72, 0x73, 0x52, 0x10, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, - 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x14, 0x4f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, - 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, - 0x2d, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x13, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x41, - 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x40, - 0x0a, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, - 0x22, 0xea, 0x02, 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, - 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, - 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, - 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x54, 0x0a, 0x17, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x17, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x12, 0x5b, 0x0a, 0x1a, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x49, 0x64, 0x52, 0x65, 0x75, 0x73, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x49, 0x64, 0x52, 0x65, 0x75, 0x73, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, - 0x79, 0x52, 0x1a, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x49, 0x64, 0x52, 0x65, 0x75, 0x73, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x22, 0x90, 0x01, - 0x0a, 0x1a, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x49, - 0x64, 0x52, 0x65, 0x75, 0x73, 0x65, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x12, 0x3e, 0x0a, 0x0f, - 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, - 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0f, 0x6f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x32, 0x0a, 0x06, - 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x1a, 0x2e, 0x43, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x06, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x22, 0x38, 0x0a, 0x16, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, - 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x22, 0x66, 0x0a, 0x12, 0x47, 0x65, - 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, - 0x12, 0x30, 0x0a, 0x13, 0x67, 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x41, 0x6e, 0x64, - 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x13, 0x67, - 0x65, 0x74, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x41, 0x6e, 0x64, 0x4f, 0x75, 0x74, 0x70, 0x75, - 0x74, 0x73, 0x22, 0x72, 0x0a, 0x13, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x78, 0x69, - 0x73, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, - 0x73, 0x12, 0x43, 0x0a, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, - 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, - 0x74, 0x65, 0x52, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x6d, 0x0a, 0x15, 0x52, 0x65, 0x77, 0x69, 0x6e, 0x64, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, - 0x34, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, - 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x18, 0x0a, 0x16, 0x52, 0x65, 0x77, 0x69, 0x6e, 0x64, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, - 0x9f, 0x05, 0x0a, 0x12, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x76, 0x65, - 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, - 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, - 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, - 0x14, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, - 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x54, 0x0a, 0x17, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, - 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x17, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, - 0x65, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x12, 0x46, 0x0a, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, - 0x74, 0x61, 0x6d, 0x70, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x4e, 0x0a, 0x14, 0x6c, 0x61, 0x73, 0x74, - 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, - 0x6d, 0x70, 0x52, 0x14, 0x6c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, - 0x74, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, - 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x34, 0x0a, 0x06, - 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, - 0x75, 0x74, 0x12, 0x40, 0x0a, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x53, 0x74, 0x61, 0x74, - 0x75, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, - 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0c, 0x63, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x12, 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, - 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x52, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, - 0x73, 0x22, 0x7b, 0x0a, 0x11, 0x52, 0x61, 0x69, 0x73, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x22, 0x14, - 0x0a, 0x12, 0x52, 0x61, 0x69, 0x73, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x86, 0x01, 0x0a, 0x10, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, - 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x6f, 0x75, 0x74, - 0x70, 0x75, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x12, - 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65, 0x22, 0x13, 0x0a, - 0x11, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x22, 0x66, 0x0a, 0x0e, 0x53, 0x75, 0x73, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x64, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x11, 0x0a, 0x0f, 0x53, 0x75, - 0x73, 0x70, 0x65, 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x65, 0x0a, - 0x0d, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, - 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x34, - 0x0a, 0x06, 0x72, 0x65, 0x61, 0x73, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, - 0x61, 0x73, 0x6f, 0x6e, 0x22, 0x10, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x3d, 0x0a, 0x15, 0x51, 0x75, 0x65, 0x72, 0x79, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, - 0x24, 0x0a, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0e, - 0x2e, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x05, - 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x8d, 0x04, 0x0a, 0x0d, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x3a, 0x0a, 0x0d, 0x72, 0x75, 0x6e, 0x74, 0x69, - 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x14, - 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, - 0x61, 0x74, 0x75, 0x73, 0x52, 0x0d, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, - 0x74, 0x75, 0x73, 0x12, 0x44, 0x0a, 0x0f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, - 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, - 0x64, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x40, 0x0a, 0x0d, 0x63, 0x72, 0x65, - 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x63, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x6f, 0x12, 0x40, 0x0a, 0x0c, 0x74, - 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, - 0x0c, 0x74, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x12, 0x2a, 0x0a, - 0x10, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, - 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, - 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x06, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x48, 0x0a, 0x10, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x10, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, - 0x34, 0x0a, 0x15, 0x66, 0x65, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x41, 0x6e, - 0x64, 0x4f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x08, 0x52, 0x15, - 0x66, 0x65, 0x74, 0x63, 0x68, 0x49, 0x6e, 0x70, 0x75, 0x74, 0x73, 0x41, 0x6e, 0x64, 0x4f, 0x75, - 0x74, 0x70, 0x75, 0x74, 0x73, 0x22, 0xa9, 0x01, 0x0a, 0x16, 0x51, 0x75, 0x65, 0x72, 0x79, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x43, 0x0a, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x4f, - 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x52, 0x12, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x11, - 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x22, 0xac, 0x01, 0x0a, 0x15, 0x50, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, - 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x20, 0x0a, 0x0a, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, - 0x00, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x48, 0x0a, - 0x13, 0x70, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x46, 0x69, - 0x6c, 0x74, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x50, 0x75, 0x72, - 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, - 0x48, 0x00, 0x52, 0x13, 0x70, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x63, 0x75, 0x72, - 0x73, 0x69, 0x76, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x72, 0x65, 0x63, 0x75, - 0x72, 0x73, 0x69, 0x76, 0x65, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x22, 0xd9, 0x01, 0x0a, 0x13, 0x50, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x12, 0x44, 0x0a, 0x0f, 0x63, 0x72, 0x65, 0x61, - 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0f, 0x63, - 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x40, - 0x0a, 0x0d, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x6f, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, - 0x70, 0x52, 0x0d, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x54, 0x6f, - 0x12, 0x3a, 0x0a, 0x0d, 0x72, 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0e, 0x32, 0x14, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, - 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x0d, 0x72, - 0x75, 0x6e, 0x74, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x22, 0x4c, 0x0a, 0x16, - 0x50, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x32, 0x0a, 0x14, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x64, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x05, 0x52, 0x14, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0x42, 0x0a, 0x14, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, - 0x73, 0x74, 0x12, 0x2a, 0x0a, 0x10, 0x72, 0x65, 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x66, - 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x72, 0x65, - 0x63, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x66, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x22, 0x17, - 0x0a, 0x15, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x44, 0x65, 0x6c, 0x65, 0x74, - 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x22, - 0x17, 0x0a, 0x15, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0xdd, 0x01, 0x0a, 0x13, 0x53, 0x69, 0x67, - 0x6e, 0x61, 0x6c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, - 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x49, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, 0x40, 0x0a, 0x0d, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, - 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x73, 0x63, 0x68, 0x65, 0x64, - 0x75, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x16, 0x0a, 0x14, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x6c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x22, 0x56, 0x0a, 0x10, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, - 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x49, 0x64, 0x12, 0x22, 0x0a, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0x54, 0x0a, 0x11, 0x47, 0x65, 0x74, 0x45, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x16, 0x0a, - 0x06, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x65, - 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x27, 0x0a, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, - 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x06, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x22, 0xc0, - 0x03, 0x0a, 0x0b, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x51, 0x75, 0x65, 0x72, 0x79, 0x12, 0x50, - 0x0a, 0x14, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x53, 0x74, 0x61, 0x72, - 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x14, 0x69, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x53, 0x74, 0x61, 0x72, 0x74, 0x73, 0x57, 0x69, 0x74, 0x68, - 0x12, 0x46, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, - 0x46, 0x72, 0x6f, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, - 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, - 0x66, 0x69, 0x65, 0x64, 0x46, 0x72, 0x6f, 0x6d, 0x12, 0x42, 0x0a, 0x0e, 0x6c, 0x61, 0x73, 0x74, - 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x6f, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0e, 0x6c, 0x61, - 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x6f, 0x12, 0x22, 0x0a, 0x0c, - 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, 0x04, 0x20, 0x01, - 0x28, 0x08, 0x52, 0x0c, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x2a, 0x0a, 0x10, 0x69, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, - 0x69, 0x65, 0x6e, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x52, 0x10, 0x69, 0x6e, 0x63, 0x6c, - 0x75, 0x64, 0x65, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74, 0x12, 0x37, 0x0a, 0x08, - 0x70, 0x61, 0x67, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x49, 0x6e, 0x74, 0x33, 0x32, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x08, 0x70, 0x61, 0x67, - 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x11, - 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, - 0x6e, 0x22, 0x3a, 0x0a, 0x14, 0x51, 0x75, 0x65, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, - 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x22, 0x0a, 0x05, 0x71, 0x75, 0x65, - 0x72, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x51, 0x75, 0x65, 0x72, 0x79, 0x52, 0x05, 0x71, 0x75, 0x65, 0x72, 0x79, 0x22, 0x90, 0x01, - 0x0a, 0x15, 0x51, 0x75, 0x65, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, - 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2b, 0x0a, 0x08, 0x65, 0x6e, 0x74, 0x69, 0x74, - 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x45, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x52, 0x08, 0x65, 0x6e, 0x74, 0x69, - 0x74, 0x69, 0x65, 0x73, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, - 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x11, 0x63, - 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, - 0x22, 0xa6, 0x02, 0x0a, 0x0e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, 0x74, 0x61, 0x64, - 0x61, 0x74, 0x61, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, - 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x49, 0x64, 0x12, 0x46, 0x0a, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, - 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x10, 0x6c, 0x61, 0x73, 0x74, 0x4d, - 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x62, - 0x61, 0x63, 0x6b, 0x6c, 0x6f, 0x67, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x10, 0x62, 0x61, 0x63, 0x6b, 0x6c, 0x6f, 0x67, 0x51, 0x75, - 0x65, 0x75, 0x65, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x38, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x65, - 0x64, 0x42, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x6b, 0x65, 0x64, 0x42, - 0x79, 0x12, 0x46, 0x0a, 0x0f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x64, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x69, 0x61, 0x6c, - 0x69, 0x7a, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x65, 0x22, 0xcd, 0x01, 0x0a, 0x19, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, - 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, - 0x6b, 0x65, 0x6e, 0x12, 0x30, 0x0a, 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x6d, 0x70, - 0x74, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, - 0x52, 0x13, 0x72, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x4f, 0x72, 0x70, 0x68, 0x61, 0x6e, 0x65, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x08, 0x52, 0x14, 0x72, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x4f, 0x72, 0x70, 0x68, - 0x61, 0x6e, 0x65, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x22, 0xd2, 0x01, 0x0a, 0x1a, 0x43, 0x6c, - 0x65, 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4a, 0x0a, 0x11, 0x63, 0x6f, 0x6e, 0x74, - 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x01, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, - 0x65, 0x52, 0x11, 0x63, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, - 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x32, 0x0a, 0x14, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x05, 0x52, 0x14, 0x65, 0x6d, 0x70, 0x74, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, - 0x73, 0x52, 0x65, 0x6d, 0x6f, 0x76, 0x65, 0x64, 0x12, 0x34, 0x0a, 0x15, 0x6f, 0x72, 0x70, 0x68, - 0x61, 0x6e, 0x65, 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, - 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x15, 0x6f, 0x72, 0x70, 0x68, 0x61, 0x6e, 0x65, - 0x64, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65, 0x64, 0x22, 0x79, - 0x0a, 0x1c, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x50, 0x61, 0x72, 0x61, 0x6d, 0x65, 0x74, 0x65, 0x72, 0x73, 0x12, 0x59, - 0x0a, 0x1a, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x6f, 0x72, 0x64, 0x65, 0x72, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x1a, 0x65, - 0x6e, 0x74, 0x69, 0x74, 0x79, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x52, 0x65, 0x6f, 0x72, - 0x64, 0x65, 0x72, 0x57, 0x69, 0x6e, 0x64, 0x6f, 0x77, 0x22, 0xa7, 0x01, 0x0a, 0x12, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, - 0x12, 0x3e, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x31, 0x0a, 0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, - 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x52, 0x0a, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x22, 0xe8, 0x01, 0x0a, 0x11, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x42, 0x61, - 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x2a, 0x0a, 0x07, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x4f, 0x70, 0x65, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x52, 0x07, 0x72, 0x65, - 0x73, 0x75, 0x6c, 0x74, 0x73, 0x12, 0x2a, 0x0a, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x07, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x3e, 0x0a, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x0b, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x61, 0x74, - 0x65, 0x12, 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, - 0x69, 0x6c, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, - 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, - 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0x82, - 0x01, 0x0a, 0x10, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x12, 0x1c, 0x0a, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x1c, 0x0a, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x49, 0x64, 0x12, - 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, - 0x70, 0x75, 0x74, 0x22, 0x89, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x12, 0x33, 0x0a, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, - 0x73, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, - 0x73, 0x48, 0x00, 0x52, 0x07, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x33, 0x0a, 0x07, - 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x17, 0x2e, - 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x46, - 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x48, 0x00, 0x52, 0x07, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, - 0x65, 0x42, 0x0c, 0x0a, 0x0a, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x54, 0x79, 0x70, 0x65, 0x22, - 0x4e, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x12, 0x34, 0x0a, 0x06, 0x72, 0x65, 0x73, - 0x75, 0x6c, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, - 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x06, 0x72, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x22, - 0x55, 0x0a, 0x16, 0x4f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x73, 0x75, - 0x6c, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x12, 0x3b, 0x0a, 0x0e, 0x66, 0x61, 0x69, - 0x6c, 0x75, 0x72, 0x65, 0x44, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x13, 0x2e, 0x54, 0x61, 0x73, 0x6b, 0x46, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x52, 0x0e, 0x66, 0x61, 0x69, 0x6c, 0x75, 0x72, 0x65, 0x44, - 0x65, 0x74, 0x61, 0x69, 0x6c, 0x73, 0x22, 0xc3, 0x01, 0x0a, 0x0f, 0x4f, 0x70, 0x65, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x02, 0x69, 0x64, 0x12, 0x33, 0x0a, 0x0a, 0x73, 0x65, - 0x6e, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, - 0x2e, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x48, 0x00, 0x52, 0x0a, 0x73, 0x65, 0x6e, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x12, - 0x54, 0x0a, 0x15, 0x73, 0x74, 0x61, 0x72, 0x74, 0x4e, 0x65, 0x77, 0x4f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, - 0x2e, 0x53, 0x74, 0x61, 0x72, 0x74, 0x4e, 0x65, 0x77, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x48, 0x00, 0x52, 0x15, - 0x73, 0x74, 0x61, 0x72, 0x74, 0x4e, 0x65, 0x77, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x42, 0x15, 0x0a, 0x13, 0x6f, 0x70, 0x65, 0x72, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x79, 0x70, 0x65, 0x22, 0xbc, 0x01, 0x0a, - 0x10, 0x53, 0x65, 0x6e, 0x64, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x41, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, - 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, - 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, - 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x40, 0x0a, 0x0d, 0x73, 0x63, 0x68, - 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, 0x73, 0x63, - 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0xff, 0x01, 0x0a, 0x1b, - 0x53, 0x74, 0x61, 0x72, 0x74, 0x4e, 0x65, 0x77, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x1e, 0x0a, 0x0a, 0x69, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0a, 0x69, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x49, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, - 0x36, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x32, 0x0a, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, - 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, - 0x61, 0x6c, 0x75, 0x65, 0x52, 0x05, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x12, 0x40, 0x0a, 0x0d, 0x73, - 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x0d, - 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x64, 0x54, 0x69, 0x6d, 0x65, 0x22, 0x15, 0x0a, - 0x13, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x22, 0xda, 0x01, 0x0a, 0x08, 0x57, 0x6f, 0x72, 0x6b, 0x49, 0x74, 0x65, - 0x6d, 0x12, 0x48, 0x0a, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, - 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, - 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, - 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x13, 0x6f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, - 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3c, 0x0a, 0x0f, 0x61, - 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x41, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0f, 0x61, 0x63, 0x74, 0x69, 0x76, 0x69, - 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x3b, 0x0a, 0x0d, 0x65, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x13, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, - 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x00, 0x52, 0x0d, 0x65, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x42, 0x09, 0x0a, 0x07, 0x72, 0x65, 0x71, 0x75, 0x65, 0x73, - 0x74, 0x22, 0x16, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0xb5, 0x02, 0x0a, 0x13, 0x4f, 0x72, - 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x12, 0x20, 0x0a, 0x1c, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, 0x49, - 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x52, 0x55, 0x4e, 0x4e, 0x49, 0x4e, - 0x47, 0x10, 0x00, 0x12, 0x22, 0x0a, 0x1e, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x4f, 0x4d, 0x50, - 0x4c, 0x45, 0x54, 0x45, 0x44, 0x10, 0x01, 0x12, 0x29, 0x0a, 0x25, 0x4f, 0x52, 0x43, 0x48, 0x45, - 0x53, 0x54, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, - 0x43, 0x4f, 0x4e, 0x54, 0x49, 0x4e, 0x55, 0x45, 0x44, 0x5f, 0x41, 0x53, 0x5f, 0x4e, 0x45, 0x57, - 0x10, 0x02, 0x12, 0x1f, 0x0a, 0x1b, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, - 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, - 0x44, 0x10, 0x03, 0x12, 0x21, 0x0a, 0x1d, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, - 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x43, 0x41, 0x4e, 0x43, - 0x45, 0x4c, 0x45, 0x44, 0x10, 0x04, 0x12, 0x23, 0x0a, 0x1f, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, - 0x54, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x54, - 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, 0x54, 0x45, 0x44, 0x10, 0x05, 0x12, 0x20, 0x0a, 0x1c, 0x4f, - 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x54, 0x41, - 0x54, 0x55, 0x53, 0x5f, 0x50, 0x45, 0x4e, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x06, 0x12, 0x22, 0x0a, - 0x1e, 0x4f, 0x52, 0x43, 0x48, 0x45, 0x53, 0x54, 0x52, 0x41, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, - 0x54, 0x41, 0x54, 0x55, 0x53, 0x5f, 0x53, 0x55, 0x53, 0x50, 0x45, 0x4e, 0x44, 0x45, 0x44, 0x10, - 0x07, 0x2a, 0x41, 0x0a, 0x19, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, - 0x73, 0x74, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x41, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x09, - 0x0a, 0x05, 0x45, 0x52, 0x52, 0x4f, 0x52, 0x10, 0x00, 0x12, 0x0a, 0x0a, 0x06, 0x49, 0x47, 0x4e, - 0x4f, 0x52, 0x45, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x45, 0x52, 0x4d, 0x49, 0x4e, 0x41, - 0x54, 0x45, 0x10, 0x02, 0x32, 0xfc, 0x0a, 0x0a, 0x15, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, - 0x53, 0x69, 0x64, 0x65, 0x63, 0x61, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, 0x37, - 0x0a, 0x05, 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x12, 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, - 0x16, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, - 0x66, 0x2e, 0x45, 0x6d, 0x70, 0x74, 0x79, 0x12, 0x40, 0x0a, 0x0d, 0x53, 0x74, 0x61, 0x72, 0x74, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x16, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, - 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x17, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x38, 0x0a, 0x0b, 0x47, 0x65, 0x74, - 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x13, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, - 0x47, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x0e, 0x52, 0x65, 0x77, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x73, - 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x16, 0x2e, 0x52, 0x65, 0x77, 0x69, 0x6e, 0x64, 0x49, 0x6e, - 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, - 0x52, 0x65, 0x77, 0x69, 0x6e, 0x64, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, - 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, 0x14, 0x57, 0x61, 0x69, 0x74, 0x46, 0x6f, - 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x53, 0x74, 0x61, 0x72, 0x74, 0x12, 0x13, - 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x46, 0x0a, 0x19, 0x57, 0x61, 0x69, - 0x74, 0x46, 0x6f, 0x72, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x43, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x13, 0x2e, 0x47, 0x65, 0x74, 0x49, 0x6e, 0x73, 0x74, - 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x14, 0x2e, 0x47, 0x65, - 0x74, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x12, 0x35, 0x0a, 0x0a, 0x52, 0x61, 0x69, 0x73, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x12, - 0x12, 0x2e, 0x52, 0x61, 0x69, 0x73, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x52, 0x65, 0x71, 0x75, - 0x65, 0x73, 0x74, 0x1a, 0x13, 0x2e, 0x52, 0x61, 0x69, 0x73, 0x65, 0x45, 0x76, 0x65, 0x6e, 0x74, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3a, 0x0a, 0x11, 0x54, 0x65, 0x72, 0x6d, - 0x69, 0x6e, 0x61, 0x74, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x11, 0x2e, - 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x12, 0x2e, 0x54, 0x65, 0x72, 0x6d, 0x69, 0x6e, 0x61, 0x74, 0x65, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x34, 0x0a, 0x0f, 0x53, 0x75, 0x73, 0x70, 0x65, 0x6e, 0x64, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x0f, 0x2e, 0x53, 0x75, 0x73, 0x70, 0x65, 0x6e, - 0x64, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x10, 0x2e, 0x53, 0x75, 0x73, 0x70, 0x65, - 0x6e, 0x64, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x0e, 0x52, 0x65, - 0x73, 0x75, 0x6d, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x12, 0x0e, 0x2e, 0x52, - 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0f, 0x2e, 0x52, - 0x65, 0x73, 0x75, 0x6d, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x41, 0x0a, - 0x0e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x12, - 0x16, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, - 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x49, - 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x41, 0x0a, 0x0e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, - 0x65, 0x73, 0x12, 0x16, 0x2e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, - 0x63, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x17, 0x2e, 0x50, 0x75, 0x72, - 0x67, 0x65, 0x49, 0x6e, 0x73, 0x74, 0x61, 0x6e, 0x63, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x31, 0x0a, 0x0c, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x49, 0x74, - 0x65, 0x6d, 0x73, 0x12, 0x14, 0x2e, 0x47, 0x65, 0x74, 0x57, 0x6f, 0x72, 0x6b, 0x49, 0x74, 0x65, - 0x6d, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x09, 0x2e, 0x57, 0x6f, 0x72, 0x6b, - 0x49, 0x74, 0x65, 0x6d, 0x30, 0x01, 0x12, 0x40, 0x0a, 0x14, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, - 0x74, 0x65, 0x41, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x11, - 0x2e, 0x41, 0x63, 0x74, 0x69, 0x76, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, - 0x65, 0x1a, 0x15, 0x2e, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, - 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x48, 0x0a, 0x18, 0x43, 0x6f, 0x6d, 0x70, - 0x6c, 0x65, 0x74, 0x65, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, 0x74, 0x6f, 0x72, - 0x54, 0x61, 0x73, 0x6b, 0x12, 0x15, 0x2e, 0x4f, 0x72, 0x63, 0x68, 0x65, 0x73, 0x74, 0x72, 0x61, - 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x1a, 0x15, 0x2e, 0x43, 0x6f, - 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, - 0x73, 0x65, 0x12, 0x3f, 0x0a, 0x12, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x54, 0x61, 0x73, 0x6b, 0x12, 0x12, 0x2e, 0x45, 0x6e, 0x74, 0x69, 0x74, - 0x79, 0x42, 0x61, 0x74, 0x63, 0x68, 0x52, 0x65, 0x73, 0x75, 0x6c, 0x74, 0x1a, 0x15, 0x2e, 0x43, - 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x48, 0x75, 0x62, 0x12, 0x15, 0x2e, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x43, 0x72, - 0x65, 0x61, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x48, 0x75, 0x62, 0x12, 0x15, 0x2e, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, - 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x44, 0x65, - 0x6c, 0x65, 0x74, 0x65, 0x54, 0x61, 0x73, 0x6b, 0x48, 0x75, 0x62, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x12, 0x3b, 0x0a, 0x0c, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x12, 0x14, 0x2e, 0x53, 0x69, 0x67, 0x6e, 0x61, 0x6c, 0x45, 0x6e, 0x74, 0x69, - 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x15, 0x2e, 0x53, 0x69, 0x67, 0x6e, - 0x61, 0x6c, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x32, 0x0a, 0x09, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x12, 0x11, 0x2e, - 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, - 0x1a, 0x12, 0x2e, 0x47, 0x65, 0x74, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3e, 0x0a, 0x0d, 0x51, 0x75, 0x65, 0x72, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x12, 0x15, 0x2e, 0x51, 0x75, 0x65, 0x72, 0x79, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x16, 0x2e, 0x51, - 0x75, 0x65, 0x72, 0x79, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x69, 0x65, 0x73, 0x52, 0x65, 0x73, 0x70, - 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x4d, 0x0a, 0x12, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x45, 0x6e, 0x74, - 0x69, 0x74, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x1a, 0x2e, 0x43, 0x6c, 0x65, - 0x61, 0x6e, 0x45, 0x6e, 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, - 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x1b, 0x2e, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x45, 0x6e, - 0x74, 0x69, 0x74, 0x79, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x52, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x42, 0x66, 0x0a, 0x31, 0x63, 0x6f, 0x6d, 0x2e, 0x6d, 0x69, 0x63, 0x72, 0x6f, - 0x73, 0x6f, 0x66, 0x74, 0x2e, 0x64, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x74, 0x61, 0x73, 0x6b, - 0x2e, 0x69, 0x6d, 0x70, 0x6c, 0x65, 0x6d, 0x65, 0x6e, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, - 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x5a, 0x10, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, - 0x6e, 0x61, 0x6c, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0xaa, 0x02, 0x1e, 0x4d, 0x69, 0x63, - 0x72, 0x6f, 0x73, 0x6f, 0x66, 0x74, 0x2e, 0x44, 0x75, 0x72, 0x61, 0x62, 0x6c, 0x65, 0x54, 0x61, - 0x73, 0x6b, 0x2e, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x62, 0x06, 0x70, 0x72, 0x6f, - 0x74, 0x6f, 0x33, +func (x *InstanceBatch) GetInstanceIds() []string { + if x != nil { + return x.InstanceIds + } + return nil } +var File_orchestrator_service_proto protoreflect.FileDescriptor + +const file_orchestrator_service_proto_rawDesc = "" + + "\n" + + "\x1aorchestrator_service.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1cgoogle/protobuf/struct.proto\"w\n" + + "\x15OrchestrationInstance\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12>\n" + + "\vexecutionId\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\vexecutionId\"\x9f\x03\n" + + "\x0fActivityRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12L\n" + + "\x15orchestrationInstance\x18\x04 \x01(\v2\x16.OrchestrationInstanceR\x15orchestrationInstance\x12\x16\n" + + "\x06taskId\x18\x05 \x01(\x05R\x06taskId\x12=\n" + + "\x12parentTraceContext\x18\x06 \x01(\v2\r.TraceContextR\x12parentTraceContext\x12.\n" + + "\x04tags\x18\a \x03(\v2\x1a.ActivityRequest.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xe7\x01\n" + + "\x10ActivityResponse\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x16\n" + + "\x06taskId\x18\x02 \x01(\x05R\x06taskId\x124\n" + + "\x06result\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\x12;\n" + + "\x0efailureDetails\x18\x04 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\x12(\n" + + "\x0fcompletionToken\x18\x05 \x01(\tR\x0fcompletionToken\"\x91\x03\n" + + "\x12TaskFailureDetails\x12\x1c\n" + + "\terrorType\x18\x01 \x01(\tR\terrorType\x12\"\n" + + "\ferrorMessage\x18\x02 \x01(\tR\ferrorMessage\x12<\n" + + "\n" + + "stackTrace\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\n" + + "stackTrace\x127\n" + + "\finnerFailure\x18\x04 \x01(\v2\x13.TaskFailureDetailsR\finnerFailure\x12&\n" + + "\x0eisNonRetriable\x18\x05 \x01(\bR\x0eisNonRetriable\x12C\n" + + "\n" + + "properties\x18\x06 \x03(\v2#.TaskFailureDetails.PropertiesEntryR\n" + + "properties\x1aU\n" + + "\x0fPropertiesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12,\n" + + "\x05value\x18\x02 \x01(\v2\x16.google.protobuf.ValueR\x05value:\x028\x01\"\xf6\x01\n" + + "\x12ParentInstanceInfo\x12(\n" + + "\x0ftaskScheduledId\x18\x01 \x01(\x05R\x0ftaskScheduledId\x120\n" + + "\x04name\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x12L\n" + + "\x15orchestrationInstance\x18\x04 \x01(\v2\x16.OrchestrationInstanceR\x15orchestrationInstance\"\x8a\x01\n" + + "\fTraceContext\x12 \n" + + "\vtraceParent\x18\x01 \x01(\tR\vtraceParent\x12\x1a\n" + + "\x06spanID\x18\x02 \x01(\tB\x02\x18\x01R\x06spanID\x12<\n" + + "\n" + + "traceState\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\n" + + "traceState\"\xf6\x04\n" + + "\x15ExecutionStartedEvent\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12L\n" + + "\x15orchestrationInstance\x18\x04 \x01(\v2\x16.OrchestrationInstanceR\x15orchestrationInstance\x12;\n" + + "\x0eparentInstance\x18\x05 \x01(\v2\x13.ParentInstanceInfoR\x0eparentInstance\x12T\n" + + "\x17scheduledStartTimestamp\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\x17scheduledStartTimestamp\x12=\n" + + "\x12parentTraceContext\x18\a \x01(\v2\r.TraceContextR\x12parentTraceContext\x12N\n" + + "\x13orchestrationSpanID\x18\b \x01(\v2\x1c.google.protobuf.StringValueR\x13orchestrationSpanID\x124\n" + + "\x04tags\x18\t \x03(\v2 .ExecutionStartedEvent.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xd4\x01\n" + + "\x17ExecutionCompletedEvent\x12F\n" + + "\x13orchestrationStatus\x18\x01 \x01(\x0e2\x14.OrchestrationStatusR\x13orchestrationStatus\x124\n" + + "\x06result\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\x12;\n" + + "\x0efailureDetails\x18\x03 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\"h\n" + + "\x18ExecutionTerminatedEvent\x122\n" + + "\x05input\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12\x18\n" + + "\arecurse\x18\x02 \x01(\bR\arecurse\"\xbf\x02\n" + + "\x12TaskScheduledEvent\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12=\n" + + "\x12parentTraceContext\x18\x04 \x01(\v2\r.TraceContextR\x12parentTraceContext\x121\n" + + "\x04tags\x18\x05 \x03(\v2\x1d.TaskScheduledEvent.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"t\n" + + "\x12TaskCompletedEvent\x12(\n" + + "\x0ftaskScheduledId\x18\x01 \x01(\x05R\x0ftaskScheduledId\x124\n" + + "\x06result\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\"x\n" + + "\x0fTaskFailedEvent\x12(\n" + + "\x0ftaskScheduledId\x18\x01 \x01(\x05R\x0ftaskScheduledId\x12;\n" + + "\x0efailureDetails\x18\x02 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\"\x83\x03\n" + + "$SubOrchestrationInstanceCreatedEvent\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12=\n" + + "\x12parentTraceContext\x18\x05 \x01(\v2\r.TraceContextR\x12parentTraceContext\x12C\n" + + "\x04tags\x18\x06 \x03(\v2/.SubOrchestrationInstanceCreatedEvent.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x88\x01\n" + + "&SubOrchestrationInstanceCompletedEvent\x12(\n" + + "\x0ftaskScheduledId\x18\x01 \x01(\x05R\x0ftaskScheduledId\x124\n" + + "\x06result\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\"\x8c\x01\n" + + "#SubOrchestrationInstanceFailedEvent\x12(\n" + + "\x0ftaskScheduledId\x18\x01 \x01(\x05R\x0ftaskScheduledId\x12;\n" + + "\x0efailureDetails\x18\x02 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\"G\n" + + "\x11TimerCreatedEvent\x122\n" + + "\x06fireAt\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x06fireAt\"_\n" + + "\x0fTimerFiredEvent\x122\n" + + "\x06fireAt\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x06fireAt\x12\x18\n" + + "\atimerId\x18\x02 \x01(\x05R\atimerId\"\x1a\n" + + "\x18OrchestratorStartedEvent\"\x1c\n" + + "\x1aOrchestratorCompletedEvent\"x\n" + + "\x0eEventSentEvent\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"Z\n" + + "\x10EventRaisedEvent\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x122\n" + + "\x05input\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"@\n" + + "\fGenericEvent\x120\n" + + "\x04data\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x04data\"X\n" + + "\x11HistoryStateEvent\x12C\n" + + "\x12orchestrationState\x18\x01 \x01(\v2\x13.OrchestrationStateR\x12orchestrationState\"H\n" + + "\x12ContinueAsNewEvent\x122\n" + + "\x05input\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"M\n" + + "\x17ExecutionSuspendedEvent\x122\n" + + "\x05input\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"K\n" + + "\x15ExecutionResumedEvent\x122\n" + + "\x05input\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"\x9a\x02\n" + + "\x1cEntityOperationSignaledEvent\x12\x1c\n" + + "\trequestId\x18\x01 \x01(\tR\trequestId\x12\x1c\n" + + "\toperation\x18\x02 \x01(\tR\toperation\x12@\n" + + "\rscheduledTime\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\rscheduledTime\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12H\n" + + "\x10targetInstanceId\x18\x05 \x01(\v2\x1c.google.protobuf.StringValueR\x10targetInstanceId\"\xae\x03\n" + + "\x1aEntityOperationCalledEvent\x12\x1c\n" + + "\trequestId\x18\x01 \x01(\tR\trequestId\x12\x1c\n" + + "\toperation\x18\x02 \x01(\tR\toperation\x12@\n" + + "\rscheduledTime\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\rscheduledTime\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12H\n" + + "\x10parentInstanceId\x18\x05 \x01(\v2\x1c.google.protobuf.StringValueR\x10parentInstanceId\x12J\n" + + "\x11parentExecutionId\x18\x06 \x01(\v2\x1c.google.protobuf.StringValueR\x11parentExecutionId\x12H\n" + + "\x10targetInstanceId\x18\a \x01(\v2\x1c.google.protobuf.StringValueR\x10targetInstanceId\"\xc8\x01\n" + + "\x18EntityLockRequestedEvent\x12,\n" + + "\x11criticalSectionId\x18\x01 \x01(\tR\x11criticalSectionId\x12\x18\n" + + "\alockSet\x18\x02 \x03(\tR\alockSet\x12\x1a\n" + + "\bposition\x18\x03 \x01(\x05R\bposition\x12H\n" + + "\x10parentInstanceId\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x10parentInstanceId\"s\n" + + "\x1dEntityOperationCompletedEvent\x12\x1c\n" + + "\trequestId\x18\x01 \x01(\tR\trequestId\x124\n" + + "\x06output\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06output\"w\n" + + "\x1aEntityOperationFailedEvent\x12\x1c\n" + + "\trequestId\x18\x01 \x01(\tR\trequestId\x12;\n" + + "\x0efailureDetails\x18\x02 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\"\xd9\x01\n" + + "\x15EntityUnlockSentEvent\x12,\n" + + "\x11criticalSectionId\x18\x01 \x01(\tR\x11criticalSectionId\x12H\n" + + "\x10parentInstanceId\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x10parentInstanceId\x12H\n" + + "\x10targetInstanceId\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x10targetInstanceId\"F\n" + + "\x16EntityLockGrantedEvent\x12,\n" + + "\x11criticalSectionId\x18\x01 \x01(\tR\x11criticalSectionId\"\xe0\x04\n" + + "\x15ExecutionRewoundEvent\x124\n" + + "\x06reason\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\x12J\n" + + "\x11parentExecutionId\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x11parentExecutionId\x12<\n" + + "\n" + + "instanceId\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\n" + + "instanceId\x12=\n" + + "\x12parentTraceContext\x18\x04 \x01(\v2\r.TraceContextR\x12parentTraceContext\x120\n" + + "\x04name\x18\x05 \x01(\v2\x1c.google.protobuf.StringValueR\x04name\x126\n" + + "\aversion\x18\x06 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\a \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12;\n" + + "\x0eparentInstance\x18\b \x01(\v2\x13.ParentInstanceInfoR\x0eparentInstance\x124\n" + + "\x04tags\x18\t \x03(\v2 .ExecutionRewoundEvent.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x99\x11\n" + + "\fHistoryEvent\x12\x18\n" + + "\aeventId\x18\x01 \x01(\x05R\aeventId\x128\n" + + "\ttimestamp\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\ttimestamp\x12D\n" + + "\x10executionStarted\x18\x03 \x01(\v2\x16.ExecutionStartedEventH\x00R\x10executionStarted\x12J\n" + + "\x12executionCompleted\x18\x04 \x01(\v2\x18.ExecutionCompletedEventH\x00R\x12executionCompleted\x12M\n" + + "\x13executionTerminated\x18\x05 \x01(\v2\x19.ExecutionTerminatedEventH\x00R\x13executionTerminated\x12;\n" + + "\rtaskScheduled\x18\x06 \x01(\v2\x13.TaskScheduledEventH\x00R\rtaskScheduled\x12;\n" + + "\rtaskCompleted\x18\a \x01(\v2\x13.TaskCompletedEventH\x00R\rtaskCompleted\x122\n" + + "\n" + + "taskFailed\x18\b \x01(\v2\x10.TaskFailedEventH\x00R\n" + + "taskFailed\x12q\n" + + "\x1fsubOrchestrationInstanceCreated\x18\t \x01(\v2%.SubOrchestrationInstanceCreatedEventH\x00R\x1fsubOrchestrationInstanceCreated\x12w\n" + + "!subOrchestrationInstanceCompleted\x18\n" + + " \x01(\v2'.SubOrchestrationInstanceCompletedEventH\x00R!subOrchestrationInstanceCompleted\x12n\n" + + "\x1esubOrchestrationInstanceFailed\x18\v \x01(\v2$.SubOrchestrationInstanceFailedEventH\x00R\x1esubOrchestrationInstanceFailed\x128\n" + + "\ftimerCreated\x18\f \x01(\v2\x12.TimerCreatedEventH\x00R\ftimerCreated\x122\n" + + "\n" + + "timerFired\x18\r \x01(\v2\x10.TimerFiredEventH\x00R\n" + + "timerFired\x12M\n" + + "\x13orchestratorStarted\x18\x0e \x01(\v2\x19.OrchestratorStartedEventH\x00R\x13orchestratorStarted\x12S\n" + + "\x15orchestratorCompleted\x18\x0f \x01(\v2\x1b.OrchestratorCompletedEventH\x00R\x15orchestratorCompleted\x12/\n" + + "\teventSent\x18\x10 \x01(\v2\x0f.EventSentEventH\x00R\teventSent\x125\n" + + "\veventRaised\x18\x11 \x01(\v2\x11.EventRaisedEventH\x00R\veventRaised\x123\n" + + "\fgenericEvent\x18\x12 \x01(\v2\r.GenericEventH\x00R\fgenericEvent\x128\n" + + "\fhistoryState\x18\x13 \x01(\v2\x12.HistoryStateEventH\x00R\fhistoryState\x12;\n" + + "\rcontinueAsNew\x18\x14 \x01(\v2\x13.ContinueAsNewEventH\x00R\rcontinueAsNew\x12J\n" + + "\x12executionSuspended\x18\x15 \x01(\v2\x18.ExecutionSuspendedEventH\x00R\x12executionSuspended\x12D\n" + + "\x10executionResumed\x18\x16 \x01(\v2\x16.ExecutionResumedEventH\x00R\x10executionResumed\x12Y\n" + + "\x17entityOperationSignaled\x18\x17 \x01(\v2\x1d.EntityOperationSignaledEventH\x00R\x17entityOperationSignaled\x12S\n" + + "\x15entityOperationCalled\x18\x18 \x01(\v2\x1b.EntityOperationCalledEventH\x00R\x15entityOperationCalled\x12\\\n" + + "\x18entityOperationCompleted\x18\x19 \x01(\v2\x1e.EntityOperationCompletedEventH\x00R\x18entityOperationCompleted\x12S\n" + + "\x15entityOperationFailed\x18\x1a \x01(\v2\x1b.EntityOperationFailedEventH\x00R\x15entityOperationFailed\x12M\n" + + "\x13entityLockRequested\x18\x1b \x01(\v2\x19.EntityLockRequestedEventH\x00R\x13entityLockRequested\x12G\n" + + "\x11entityLockGranted\x18\x1c \x01(\v2\x17.EntityLockGrantedEventH\x00R\x11entityLockGranted\x12D\n" + + "\x10entityUnlockSent\x18\x1d \x01(\v2\x16.EntityUnlockSentEventH\x00R\x10entityUnlockSent\x12D\n" + + "\x10executionRewound\x18\x1e \x01(\v2\x16.ExecutionRewoundEventH\x00R\x10executionRewoundB\v\n" + + "\teventType\"\xbf\x02\n" + + "\x12ScheduleTaskAction\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x121\n" + + "\x04tags\x18\x04 \x03(\v2\x1d.ScheduleTaskAction.TagsEntryR\x04tags\x12=\n" + + "\x12parentTraceContext\x18\x05 \x01(\v2\r.TraceContextR\x12parentTraceContext\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xf3\x02\n" + + "\x1cCreateSubOrchestrationAction\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12=\n" + + "\x12parentTraceContext\x18\x05 \x01(\v2\r.TraceContextR\x12parentTraceContext\x12;\n" + + "\x04tags\x18\x06 \x03(\v2'.CreateSubOrchestrationAction.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"G\n" + + "\x11CreateTimerAction\x122\n" + + "\x06fireAt\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x06fireAt\"\x8b\x01\n" + + "\x0fSendEventAction\x122\n" + + "\binstance\x18\x01 \x01(\v2\x16.OrchestrationInstanceR\binstance\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x120\n" + + "\x04data\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x04data\"\xfc\x03\n" + + "\x1bCompleteOrchestrationAction\x12F\n" + + "\x13orchestrationStatus\x18\x01 \x01(\x0e2\x14.OrchestrationStatusR\x13orchestrationStatus\x124\n" + + "\x06result\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\x126\n" + + "\adetails\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\adetails\x12<\n" + + "\n" + + "newVersion\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\n" + + "newVersion\x127\n" + + "\x0fcarryoverEvents\x18\x05 \x03(\v2\r.HistoryEventR\x0fcarryoverEvents\x12;\n" + + "\x0efailureDetails\x18\x06 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\x12:\n" + + "\x04tags\x18\a \x03(\v2&.CompleteOrchestrationAction.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x8e\x01\n" + + "\x1cTerminateOrchestrationAction\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x124\n" + + "\x06reason\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\x12\x18\n" + + "\arecurse\x18\x03 \x01(\bR\arecurse\"\xf3\x02\n" + + "\x17SendEntityMessageAction\x12Y\n" + + "\x17entityOperationSignaled\x18\x01 \x01(\v2\x1d.EntityOperationSignaledEventH\x00R\x17entityOperationSignaled\x12S\n" + + "\x15entityOperationCalled\x18\x02 \x01(\v2\x1b.EntityOperationCalledEventH\x00R\x15entityOperationCalled\x12M\n" + + "\x13entityLockRequested\x18\x03 \x01(\v2\x19.EntityLockRequestedEventH\x00R\x13entityLockRequested\x12D\n" + + "\x10entityUnlockSent\x18\x04 \x01(\v2\x16.EntityUnlockSentEventH\x00R\x10entityUnlockSentB\x13\n" + + "\x11EntityMessageType\"J\n" + + "\x19RewindOrchestrationAction\x12-\n" + + "\n" + + "newHistory\x18\x01 \x03(\v2\r.HistoryEventR\n" + + "newHistory\"\x85\x05\n" + + "\x12OrchestratorAction\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x05R\x02id\x129\n" + + "\fscheduleTask\x18\x02 \x01(\v2\x13.ScheduleTaskActionH\x00R\fscheduleTask\x12W\n" + + "\x16createSubOrchestration\x18\x03 \x01(\v2\x1d.CreateSubOrchestrationActionH\x00R\x16createSubOrchestration\x126\n" + + "\vcreateTimer\x18\x04 \x01(\v2\x12.CreateTimerActionH\x00R\vcreateTimer\x120\n" + + "\tsendEvent\x18\x05 \x01(\v2\x10.SendEventActionH\x00R\tsendEvent\x12T\n" + + "\x15completeOrchestration\x18\x06 \x01(\v2\x1c.CompleteOrchestrationActionH\x00R\x15completeOrchestration\x12W\n" + + "\x16terminateOrchestration\x18\a \x01(\v2\x1d.TerminateOrchestrationActionH\x00R\x16terminateOrchestration\x12H\n" + + "\x11sendEntityMessage\x18\b \x01(\v2\x18.SendEntityMessageActionH\x00R\x11sendEntityMessage\x12N\n" + + "\x13rewindOrchestration\x18\t \x01(\v2\x1a.RewindOrchestrationActionH\x00R\x13rewindOrchestrationB\x18\n" + + "\x16orchestratorActionType\"\x93\x01\n" + + "\x19OrchestrationTraceContext\x124\n" + + "\x06spanID\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x06spanID\x12@\n" + + "\rspanStartTime\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\rspanStartTime\"\xcf\x04\n" + + "\x13OrchestratorRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12>\n" + + "\vexecutionId\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\vexecutionId\x12-\n" + + "\n" + + "pastEvents\x18\x03 \x03(\v2\r.HistoryEventR\n" + + "pastEvents\x12+\n" + + "\tnewEvents\x18\x04 \x03(\v2\r.HistoryEventR\tnewEvents\x12I\n" + + "\x10entityParameters\x18\x05 \x01(\v2\x1d.OrchestratorEntityParametersR\x10entityParameters\x12:\n" + + "\x18requiresHistoryStreaming\x18\x06 \x01(\bR\x18requiresHistoryStreaming\x12D\n" + + "\n" + + "properties\x18\a \x03(\v2$.OrchestratorRequest.PropertiesEntryR\n" + + "properties\x12X\n" + + "\x19orchestrationTraceContext\x18\b \x01(\v2\x1a.OrchestrationTraceContextR\x19orchestrationTraceContext\x1aU\n" + + "\x0fPropertiesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12,\n" + + "\x05value\x18\x02 \x01(\v2\x16.google.protobuf.ValueR\x05value:\x028\x01\"\x85\x04\n" + + "\x14OrchestratorResponse\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12-\n" + + "\aactions\x18\x02 \x03(\v2\x13.OrchestratorActionR\aactions\x12@\n" + + "\fcustomStatus\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\fcustomStatus\x12(\n" + + "\x0fcompletionToken\x18\x04 \x01(\tR\x0fcompletionToken\x12K\n" + + "\x12numEventsProcessed\x18\x05 \x01(\v2\x1b.google.protobuf.Int32ValueR\x12numEventsProcessed\x12X\n" + + "\x19orchestrationTraceContext\x18\x06 \x01(\v2\x1a.OrchestrationTraceContextR\x19orchestrationTraceContext\x12(\n" + + "\x0frequiresHistory\x18\a \x01(\bR\x0frequiresHistory\x12 \n" + + "\tisPartial\x18\b \x01(\bB\x02\x18\x01R\tisPartial\x12?\n" + + "\n" + + "chunkIndex\x18\t \x01(\v2\x1b.google.protobuf.Int32ValueB\x02\x18\x01R\n" + + "chunkIndex\"\x96\x05\n" + + "\x15CreateInstanceRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12T\n" + + "\x17scheduledStartTimestamp\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x17scheduledStartTimestamp\x12[\n" + + "\x1aorchestrationIdReusePolicy\x18\x06 \x01(\v2\x1b.OrchestrationIdReusePolicyR\x1aorchestrationIdReusePolicy\x12>\n" + + "\vexecutionId\x18\a \x01(\v2\x1c.google.protobuf.StringValueR\vexecutionId\x124\n" + + "\x04tags\x18\b \x03(\v2 .CreateInstanceRequest.TagsEntryR\x04tags\x12=\n" + + "\x12parentTraceContext\x18\t \x01(\v2\r.TraceContextR\x12parentTraceContext\x12<\n" + + "\vrequestTime\x18\n" + + " \x01(\v2\x1a.google.protobuf.TimestampR\vrequestTime\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"f\n" + + "\x1aOrchestrationIdReusePolicy\x12B\n" + + "\x11replaceableStatus\x18\x01 \x03(\x0e2\x14.OrchestrationStatusR\x11replaceableStatusJ\x04\b\x02\x10\x03\"8\n" + + "\x16CreateInstanceResponse\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\"f\n" + + "\x12GetInstanceRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x120\n" + + "\x13getInputsAndOutputs\x18\x02 \x01(\bR\x13getInputsAndOutputs\"r\n" + + "\x13GetInstanceResponse\x12\x16\n" + + "\x06exists\x18\x01 \x01(\bR\x06exists\x12C\n" + + "\x12orchestrationState\x18\x02 \x01(\v2\x13.OrchestrationStateR\x12orchestrationState\"m\n" + + "\x15RewindInstanceRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x124\n" + + "\x06reason\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\"\x18\n" + + "\x16RewindInstanceResponse\"\xe1\a\n" + + "\x12OrchestrationState\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x12F\n" + + "\x13orchestrationStatus\x18\x04 \x01(\x0e2\x14.OrchestrationStatusR\x13orchestrationStatus\x12T\n" + + "\x17scheduledStartTimestamp\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\x17scheduledStartTimestamp\x12F\n" + + "\x10createdTimestamp\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\x10createdTimestamp\x12N\n" + + "\x14lastUpdatedTimestamp\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\x14lastUpdatedTimestamp\x122\n" + + "\x05input\x18\b \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x124\n" + + "\x06output\x18\t \x01(\v2\x1c.google.protobuf.StringValueR\x06output\x12@\n" + + "\fcustomStatus\x18\n" + + " \x01(\v2\x1c.google.protobuf.StringValueR\fcustomStatus\x12;\n" + + "\x0efailureDetails\x18\v \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\x12>\n" + + "\vexecutionId\x18\f \x01(\v2\x1c.google.protobuf.StringValueR\vexecutionId\x12J\n" + + "\x12completedTimestamp\x18\r \x01(\v2\x1a.google.protobuf.TimestampR\x12completedTimestamp\x12H\n" + + "\x10parentInstanceId\x18\x0e \x01(\v2\x1c.google.protobuf.StringValueR\x10parentInstanceId\x121\n" + + "\x04tags\x18\x0f \x03(\v2\x1d.OrchestrationState.TagsEntryR\x04tags\x1a7\n" + + "\tTagsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"{\n" + + "\x11RaiseEventRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\"\x14\n" + + "\x12RaiseEventResponse\"\x86\x01\n" + + "\x10TerminateRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x124\n" + + "\x06output\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06output\x12\x1c\n" + + "\trecursive\x18\x03 \x01(\bR\trecursive\"\x13\n" + + "\x11TerminateResponse\"f\n" + + "\x0eSuspendRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x124\n" + + "\x06reason\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\"\x11\n" + + "\x0fSuspendResponse\"e\n" + + "\rResumeRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x124\n" + + "\x06reason\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\"\x10\n" + + "\x0eResumeResponse\"=\n" + + "\x15QueryInstancesRequest\x12$\n" + + "\x05query\x18\x01 \x01(\v2\x0e.InstanceQueryR\x05query\"\x8d\x04\n" + + "\rInstanceQuery\x12:\n" + + "\rruntimeStatus\x18\x01 \x03(\x0e2\x14.OrchestrationStatusR\rruntimeStatus\x12D\n" + + "\x0fcreatedTimeFrom\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x0fcreatedTimeFrom\x12@\n" + + "\rcreatedTimeTo\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\rcreatedTimeTo\x12@\n" + + "\ftaskHubNames\x18\x04 \x03(\v2\x1c.google.protobuf.StringValueR\ftaskHubNames\x12*\n" + + "\x10maxInstanceCount\x18\x05 \x01(\x05R\x10maxInstanceCount\x12J\n" + + "\x11continuationToken\x18\x06 \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\x12H\n" + + "\x10instanceIdPrefix\x18\a \x01(\v2\x1c.google.protobuf.StringValueR\x10instanceIdPrefix\x124\n" + + "\x15fetchInputsAndOutputs\x18\b \x01(\bR\x15fetchInputsAndOutputs\"\xa9\x01\n" + + "\x16QueryInstancesResponse\x12C\n" + + "\x12orchestrationState\x18\x01 \x03(\v2\x13.OrchestrationStateR\x12orchestrationState\x12J\n" + + "\x11continuationToken\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\"\xc8\x02\n" + + "\x16ListInstanceIdsRequest\x12:\n" + + "\rruntimeStatus\x18\x01 \x03(\x0e2\x14.OrchestrationStatusR\rruntimeStatus\x12H\n" + + "\x11completedTimeFrom\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x11completedTimeFrom\x12D\n" + + "\x0fcompletedTimeTo\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x0fcompletedTimeTo\x12\x1a\n" + + "\bpageSize\x18\x04 \x01(\x05R\bpageSize\x12F\n" + + "\x0flastInstanceKey\x18\x05 \x01(\v2\x1c.google.protobuf.StringValueR\x0flastInstanceKey\"\x83\x01\n" + + "\x17ListInstanceIdsResponse\x12 \n" + + "\vinstanceIds\x18\x01 \x03(\tR\vinstanceIds\x12F\n" + + "\x0flastInstanceKey\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x0flastInstanceKey\"\x8e\x02\n" + + "\x15PurgeInstancesRequest\x12 \n" + + "\n" + + "instanceId\x18\x01 \x01(\tH\x00R\n" + + "instanceId\x12H\n" + + "\x13purgeInstanceFilter\x18\x02 \x01(\v2\x14.PurgeInstanceFilterH\x00R\x13purgeInstanceFilter\x126\n" + + "\rinstanceBatch\x18\x04 \x01(\v2\x0e.InstanceBatchH\x00R\rinstanceBatch\x12\x1c\n" + + "\trecursive\x18\x03 \x01(\bR\trecursive\x12(\n" + + "\x0fisOrchestration\x18\x05 \x01(\bR\x0fisOrchestrationB\t\n" + + "\arequest\"\x8e\x02\n" + + "\x13PurgeInstanceFilter\x12D\n" + + "\x0fcreatedTimeFrom\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x0fcreatedTimeFrom\x12@\n" + + "\rcreatedTimeTo\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\rcreatedTimeTo\x12:\n" + + "\rruntimeStatus\x18\x03 \x03(\x0e2\x14.OrchestrationStatusR\rruntimeStatus\x123\n" + + "\atimeout\x18\x04 \x01(\v2\x19.google.protobuf.DurationR\atimeout\"\x88\x01\n" + + "\x16PurgeInstancesResponse\x122\n" + + "\x14deletedInstanceCount\x18\x01 \x01(\x05R\x14deletedInstanceCount\x12:\n" + + "\n" + + "isComplete\x18\x02 \x01(\v2\x1a.google.protobuf.BoolValueR\n" + + "isComplete\"t\n" + + "\x16RestartInstanceRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12:\n" + + "\x18restartWithNewInstanceId\x18\x02 \x01(\bR\x18restartWithNewInstanceId\"9\n" + + "\x17RestartInstanceResponse\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\"B\n" + + "\x14CreateTaskHubRequest\x12*\n" + + "\x10recreateIfExists\x18\x01 \x01(\bR\x10recreateIfExists\"\x17\n" + + "\x15CreateTaskHubResponse\"\x16\n" + + "\x14DeleteTaskHubRequest\"\x17\n" + + "\x15DeleteTaskHubResponse\"\xda\x02\n" + + "\x13SignalEntityRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12\x1c\n" + + "\trequestId\x18\x04 \x01(\tR\trequestId\x12@\n" + + "\rscheduledTime\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\rscheduledTime\x12=\n" + + "\x12parentTraceContext\x18\x06 \x01(\v2\r.TraceContextR\x12parentTraceContext\x12<\n" + + "\vrequestTime\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\vrequestTime\"\x16\n" + + "\x14SignalEntityResponse\"V\n" + + "\x10GetEntityRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\"\n" + + "\fincludeState\x18\x02 \x01(\bR\fincludeState\"T\n" + + "\x11GetEntityResponse\x12\x16\n" + + "\x06exists\x18\x01 \x01(\bR\x06exists\x12'\n" + + "\x06entity\x18\x02 \x01(\v2\x0f.EntityMetadataR\x06entity\"\xc0\x03\n" + + "\vEntityQuery\x12P\n" + + "\x14instanceIdStartsWith\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x14instanceIdStartsWith\x12F\n" + + "\x10lastModifiedFrom\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x10lastModifiedFrom\x12B\n" + + "\x0elastModifiedTo\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\x0elastModifiedTo\x12\"\n" + + "\fincludeState\x18\x04 \x01(\bR\fincludeState\x12*\n" + + "\x10includeTransient\x18\x05 \x01(\bR\x10includeTransient\x127\n" + + "\bpageSize\x18\x06 \x01(\v2\x1b.google.protobuf.Int32ValueR\bpageSize\x12J\n" + + "\x11continuationToken\x18\a \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\":\n" + + "\x14QueryEntitiesRequest\x12\"\n" + + "\x05query\x18\x01 \x01(\v2\f.EntityQueryR\x05query\"\x90\x01\n" + + "\x15QueryEntitiesResponse\x12+\n" + + "\bentities\x18\x01 \x03(\v2\x0f.EntityMetadataR\bentities\x12J\n" + + "\x11continuationToken\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\"\xa6\x02\n" + + "\x0eEntityMetadata\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12F\n" + + "\x10lastModifiedTime\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\x10lastModifiedTime\x12*\n" + + "\x10backlogQueueSize\x18\x03 \x01(\x05R\x10backlogQueueSize\x128\n" + + "\blockedBy\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\blockedBy\x12F\n" + + "\x0fserializedState\x18\x05 \x01(\v2\x1c.google.protobuf.StringValueR\x0fserializedState\"\xcd\x01\n" + + "\x19CleanEntityStorageRequest\x12J\n" + + "\x11continuationToken\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\x120\n" + + "\x13removeEmptyEntities\x18\x02 \x01(\bR\x13removeEmptyEntities\x122\n" + + "\x14releaseOrphanedLocks\x18\x03 \x01(\bR\x14releaseOrphanedLocks\"\xd2\x01\n" + + "\x1aCleanEntityStorageResponse\x12J\n" + + "\x11continuationToken\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x11continuationToken\x122\n" + + "\x14emptyEntitiesRemoved\x18\x02 \x01(\x05R\x14emptyEntitiesRemoved\x124\n" + + "\x15orphanedLocksReleased\x18\x03 \x01(\x05R\x15orphanedLocksReleased\"y\n" + + "\x1cOrchestratorEntityParameters\x12Y\n" + + "\x1aentityMessageReorderWindow\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\x1aentityMessageReorderWindow\"\xc3\x02\n" + + "\x12EntityBatchRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12>\n" + + "\ventityState\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\ventityState\x121\n" + + "\n" + + "operations\x18\x03 \x03(\v2\x11.OperationRequestR\n" + + "operations\x12C\n" + + "\n" + + "properties\x18\x04 \x03(\v2#.EntityBatchRequest.PropertiesEntryR\n" + + "properties\x1aU\n" + + "\x0fPropertiesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12,\n" + + "\x05value\x18\x02 \x01(\v2\x16.google.protobuf.ValueR\x05value:\x028\x01\"\xf0\x02\n" + + "\x11EntityBatchResult\x12*\n" + + "\aresults\x18\x01 \x03(\v2\x10.OperationResultR\aresults\x12*\n" + + "\aactions\x18\x02 \x03(\v2\x10.OperationActionR\aactions\x12>\n" + + "\ventityState\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\ventityState\x12;\n" + + "\x0efailureDetails\x18\x04 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\x12(\n" + + "\x0fcompletionToken\x18\x05 \x01(\tR\x0fcompletionToken\x126\n" + + "\x0eoperationInfos\x18\x06 \x03(\v2\x0e.OperationInfoR\x0eoperationInfos\x12$\n" + + "\rrequiresState\x18\a \x01(\bR\rrequiresState\"\xce\x01\n" + + "\rEntityRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12 \n" + + "\vexecutionId\x18\x02 \x01(\tR\vexecutionId\x12>\n" + + "\ventityState\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\ventityState\x12;\n" + + "\x11operationRequests\x18\x04 \x03(\v2\r.HistoryEventR\x11operationRequests\"\xb5\x01\n" + + "\x10OperationRequest\x12\x1c\n" + + "\toperation\x18\x01 \x01(\tR\toperation\x12\x1c\n" + + "\trequestId\x18\x02 \x01(\tR\trequestId\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x121\n" + + "\ftraceContext\x18\x04 \x01(\v2\r.TraceContextR\ftraceContext\"\x89\x01\n" + + "\x0fOperationResult\x123\n" + + "\asuccess\x18\x01 \x01(\v2\x17.OperationResultSuccessH\x00R\asuccess\x123\n" + + "\afailure\x18\x02 \x01(\v2\x17.OperationResultFailureH\x00R\afailureB\f\n" + + "\n" + + "resultType\"w\n" + + "\rOperationInfo\x12\x1c\n" + + "\trequestId\x18\x01 \x01(\tR\trequestId\x12H\n" + + "\x13responseDestination\x18\x02 \x01(\v2\x16.OrchestrationInstanceR\x13responseDestination\"\xca\x01\n" + + "\x16OperationResultSuccess\x124\n" + + "\x06result\x18\x01 \x01(\v2\x1c.google.protobuf.StringValueR\x06result\x12>\n" + + "\fstartTimeUtc\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\fstartTimeUtc\x12:\n" + + "\n" + + "endTimeUtc\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "endTimeUtc\"\xd1\x01\n" + + "\x16OperationResultFailure\x12;\n" + + "\x0efailureDetails\x18\x01 \x01(\v2\x13.TaskFailureDetailsR\x0efailureDetails\x12>\n" + + "\fstartTimeUtc\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\fstartTimeUtc\x12:\n" + + "\n" + + "endTimeUtc\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "endTimeUtc\"\xc3\x01\n" + + "\x0fOperationAction\x12\x0e\n" + + "\x02id\x18\x01 \x01(\x05R\x02id\x123\n" + + "\n" + + "sendSignal\x18\x02 \x01(\v2\x11.SendSignalActionH\x00R\n" + + "sendSignal\x12T\n" + + "\x15startNewOrchestration\x18\x03 \x01(\v2\x1c.StartNewOrchestrationActionH\x00R\x15startNewOrchestrationB\x15\n" + + "\x13operationActionType\"\xb9\x02\n" + + "\x10SendSignalAction\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x122\n" + + "\x05input\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12@\n" + + "\rscheduledTime\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\rscheduledTime\x12<\n" + + "\vrequestTime\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\vrequestTime\x12=\n" + + "\x12parentTraceContext\x18\x06 \x01(\v2\r.TraceContextR\x12parentTraceContext\"\xfc\x02\n" + + "\x1bStartNewOrchestrationAction\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x126\n" + + "\aversion\x18\x03 \x01(\v2\x1c.google.protobuf.StringValueR\aversion\x122\n" + + "\x05input\x18\x04 \x01(\v2\x1c.google.protobuf.StringValueR\x05input\x12@\n" + + "\rscheduledTime\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\rscheduledTime\x12<\n" + + "\vrequestTime\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\vrequestTime\x12=\n" + + "\x12parentTraceContext\x18\a \x01(\v2\r.TraceContextR\x12parentTraceContext\"F\n" + + "\x1aAbandonActivityTaskRequest\x12(\n" + + "\x0fcompletionToken\x18\x01 \x01(\tR\x0fcompletionToken\"\x1d\n" + + "\x1bAbandonActivityTaskResponse\"K\n" + + "\x1fAbandonOrchestrationTaskRequest\x12(\n" + + "\x0fcompletionToken\x18\x01 \x01(\tR\x0fcompletionToken\"\"\n" + + " AbandonOrchestrationTaskResponse\"D\n" + + "\x18AbandonEntityTaskRequest\x12(\n" + + "\x0fcompletionToken\x18\x01 \x01(\tR\x0fcompletionToken\"\x1b\n" + + "\x19AbandonEntityTaskResponse\"\x9a\x01\n" + + ",SkipGracefulOrchestrationTerminationsRequest\x124\n" + + "\rinstanceBatch\x18\x01 \x01(\v2\x0e.InstanceBatchR\rinstanceBatch\x124\n" + + "\x06reason\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\x06reason\"i\n" + + "-SkipGracefulOrchestrationTerminationsResponse\x128\n" + + "\x17unterminatedInstanceIds\x18\x01 \x03(\tR\x17unterminatedInstanceIds\"\xe6\x02\n" + + "\x13GetWorkItemsRequest\x12P\n" + + "#maxConcurrentOrchestrationWorkItems\x18\x01 \x01(\x05R#maxConcurrentOrchestrationWorkItems\x12F\n" + + "\x1emaxConcurrentActivityWorkItems\x18\x02 \x01(\x05R\x1emaxConcurrentActivityWorkItems\x12B\n" + + "\x1cmaxConcurrentEntityWorkItems\x18\x03 \x01(\x05R\x1cmaxConcurrentEntityWorkItems\x125\n" + + "\fcapabilities\x18\n" + + " \x03(\x0e2\x11.WorkerCapabilityR\fcapabilities\x12:\n" + + "\x0fworkItemFilters\x18\v \x01(\v2\x10.WorkItemFiltersR\x0fworkItemFilters\"\xab\x01\n" + + "\x0fWorkItemFilters\x12<\n" + + "\x0eorchestrations\x18\x01 \x03(\v2\x14.OrchestrationFilterR\x0eorchestrations\x12/\n" + + "\n" + + "activities\x18\x02 \x03(\v2\x0f.ActivityFilterR\n" + + "activities\x12)\n" + + "\bentities\x18\x03 \x03(\v2\r.EntityFilterR\bentities\"E\n" + + "\x13OrchestrationFilter\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + + "\bversions\x18\x02 \x03(\tR\bversions\"@\n" + + "\x0eActivityFilter\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1a\n" + + "\bversions\x18\x02 \x03(\tR\bversions\"\"\n" + + "\fEntityFilter\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"\xef\x02\n" + + "\bWorkItem\x12H\n" + + "\x13orchestratorRequest\x18\x01 \x01(\v2\x14.OrchestratorRequestH\x00R\x13orchestratorRequest\x12<\n" + + "\x0factivityRequest\x18\x02 \x01(\v2\x10.ActivityRequestH\x00R\x0factivityRequest\x12;\n" + + "\rentityRequest\x18\x03 \x01(\v2\x13.EntityBatchRequestH\x00R\rentityRequest\x12-\n" + + "\n" + + "healthPing\x18\x04 \x01(\v2\v.HealthPingH\x00R\n" + + "healthPing\x12:\n" + + "\x0fentityRequestV2\x18\x05 \x01(\v2\x0e.EntityRequestH\x00R\x0fentityRequestV2\x12(\n" + + "\x0fcompletionToken\x18\n" + + " \x01(\tR\x0fcompletionTokenB\t\n" + + "\arequest\"\x16\n" + + "\x14CompleteTaskResponse\"\f\n" + + "\n" + + "HealthPing\"\xb4\x01\n" + + "\x1cStreamInstanceHistoryRequest\x12\x1e\n" + + "\n" + + "instanceId\x18\x01 \x01(\tR\n" + + "instanceId\x12>\n" + + "\vexecutionId\x18\x02 \x01(\v2\x1c.google.protobuf.StringValueR\vexecutionId\x124\n" + + "\x15forWorkItemProcessing\x18\x03 \x01(\bR\x15forWorkItemProcessing\"5\n" + + "\fHistoryChunk\x12%\n" + + "\x06events\x18\x01 \x03(\v2\r.HistoryEventR\x06events\"1\n" + + "\rInstanceBatch\x12 \n" + + "\vinstanceIds\x18\x01 \x03(\tR\vinstanceIds*\xb5\x02\n" + + "\x13OrchestrationStatus\x12 \n" + + "\x1cORCHESTRATION_STATUS_RUNNING\x10\x00\x12\"\n" + + "\x1eORCHESTRATION_STATUS_COMPLETED\x10\x01\x12)\n" + + "%ORCHESTRATION_STATUS_CONTINUED_AS_NEW\x10\x02\x12\x1f\n" + + "\x1bORCHESTRATION_STATUS_FAILED\x10\x03\x12!\n" + + "\x1dORCHESTRATION_STATUS_CANCELED\x10\x04\x12#\n" + + "\x1fORCHESTRATION_STATUS_TERMINATED\x10\x05\x12 \n" + + "\x1cORCHESTRATION_STATUS_PENDING\x10\x06\x12\"\n" + + "\x1eORCHESTRATION_STATUS_SUSPENDED\x10\a*\xab\x01\n" + + "\x10WorkerCapability\x12!\n" + + "\x1dWORKER_CAPABILITY_UNSPECIFIED\x10\x00\x12'\n" + + "#WORKER_CAPABILITY_HISTORY_STREAMING\x10\x01\x12%\n" + + "!WORKER_CAPABILITY_SCHEDULED_TASKS\x10\x02\x12$\n" + + " WORKER_CAPABILITY_LARGE_PAYLOADS\x10\x032\xf0\x0f\n" + + "\x15TaskHubSidecarService\x127\n" + + "\x05Hello\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\x12@\n" + + "\rStartInstance\x12\x16.CreateInstanceRequest\x1a\x17.CreateInstanceResponse\x128\n" + + "\vGetInstance\x12\x13.GetInstanceRequest\x1a\x14.GetInstanceResponse\x12A\n" + + "\x0eRewindInstance\x12\x16.RewindInstanceRequest\x1a\x17.RewindInstanceResponse\x12D\n" + + "\x0fRestartInstance\x12\x17.RestartInstanceRequest\x1a\x18.RestartInstanceResponse\x12A\n" + + "\x14WaitForInstanceStart\x12\x13.GetInstanceRequest\x1a\x14.GetInstanceResponse\x12F\n" + + "\x19WaitForInstanceCompletion\x12\x13.GetInstanceRequest\x1a\x14.GetInstanceResponse\x125\n" + + "\n" + + "RaiseEvent\x12\x12.RaiseEventRequest\x1a\x13.RaiseEventResponse\x12:\n" + + "\x11TerminateInstance\x12\x11.TerminateRequest\x1a\x12.TerminateResponse\x124\n" + + "\x0fSuspendInstance\x12\x0f.SuspendRequest\x1a\x10.SuspendResponse\x121\n" + + "\x0eResumeInstance\x12\x0e.ResumeRequest\x1a\x0f.ResumeResponse\x12A\n" + + "\x0eQueryInstances\x12\x16.QueryInstancesRequest\x1a\x17.QueryInstancesResponse\x12D\n" + + "\x0fListInstanceIds\x12\x17.ListInstanceIdsRequest\x1a\x18.ListInstanceIdsResponse\x12A\n" + + "\x0ePurgeInstances\x12\x16.PurgeInstancesRequest\x1a\x17.PurgeInstancesResponse\x121\n" + + "\fGetWorkItems\x12\x14.GetWorkItemsRequest\x1a\t.WorkItem0\x01\x12@\n" + + "\x14CompleteActivityTask\x12\x11.ActivityResponse\x1a\x15.CompleteTaskResponse\x12H\n" + + "\x18CompleteOrchestratorTask\x12\x15.OrchestratorResponse\x1a\x15.CompleteTaskResponse\x12?\n" + + "\x12CompleteEntityTask\x12\x12.EntityBatchResult\x1a\x15.CompleteTaskResponse\x12G\n" + + "\x15StreamInstanceHistory\x12\x1d.StreamInstanceHistoryRequest\x1a\r.HistoryChunk0\x01\x12>\n" + + "\rCreateTaskHub\x12\x15.CreateTaskHubRequest\x1a\x16.CreateTaskHubResponse\x12>\n" + + "\rDeleteTaskHub\x12\x15.DeleteTaskHubRequest\x1a\x16.DeleteTaskHubResponse\x12;\n" + + "\fSignalEntity\x12\x14.SignalEntityRequest\x1a\x15.SignalEntityResponse\x122\n" + + "\tGetEntity\x12\x11.GetEntityRequest\x1a\x12.GetEntityResponse\x12>\n" + + "\rQueryEntities\x12\x15.QueryEntitiesRequest\x1a\x16.QueryEntitiesResponse\x12M\n" + + "\x12CleanEntityStorage\x12\x1a.CleanEntityStorageRequest\x1a\x1b.CleanEntityStorageResponse\x12X\n" + + "\x1bAbandonTaskActivityWorkItem\x12\x1b.AbandonActivityTaskRequest\x1a\x1c.AbandonActivityTaskResponse\x12f\n" + + "\x1fAbandonTaskOrchestratorWorkItem\x12 .AbandonOrchestrationTaskRequest\x1a!.AbandonOrchestrationTaskResponse\x12R\n" + + "\x19AbandonTaskEntityWorkItem\x12\x19.AbandonEntityTaskRequest\x1a\x1a.AbandonEntityTaskResponse\x12\x86\x01\n" + + "%SkipGracefulOrchestrationTerminations\x12-.SkipGracefulOrchestrationTerminationsRequest\x1a..SkipGracefulOrchestrationTerminationsResponseBf\n" + + "1com.microsoft.durabletask.implementation.protobufZ\x10/internal/protos\xaa\x02\x1eMicrosoft.DurableTask.Protobufb\x06proto3" + var ( file_orchestrator_service_proto_rawDescOnce sync.Once - file_orchestrator_service_proto_rawDescData = file_orchestrator_service_proto_rawDesc + file_orchestrator_service_proto_rawDescData []byte ) func file_orchestrator_service_proto_rawDescGZIP() []byte { file_orchestrator_service_proto_rawDescOnce.Do(func() { - file_orchestrator_service_proto_rawDescData = protoimpl.X.CompressGZIP(file_orchestrator_service_proto_rawDescData) + file_orchestrator_service_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_orchestrator_service_proto_rawDesc), len(file_orchestrator_service_proto_rawDesc))) }) return file_orchestrator_service_proto_rawDescData } var file_orchestrator_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_orchestrator_service_proto_msgTypes = make([]protoimpl.MessageInfo, 85) -var file_orchestrator_service_proto_goTypes = []interface{}{ - (OrchestrationStatus)(0), // 0: OrchestrationStatus - (CreateOrchestrationAction)(0), // 1: CreateOrchestrationAction - (*OrchestrationInstance)(nil), // 2: OrchestrationInstance - (*ActivityRequest)(nil), // 3: ActivityRequest - (*ActivityResponse)(nil), // 4: ActivityResponse - (*TaskFailureDetails)(nil), // 5: TaskFailureDetails - (*ParentInstanceInfo)(nil), // 6: ParentInstanceInfo - (*TraceContext)(nil), // 7: TraceContext - (*ExecutionStartedEvent)(nil), // 8: ExecutionStartedEvent - (*ExecutionCompletedEvent)(nil), // 9: ExecutionCompletedEvent - (*ExecutionTerminatedEvent)(nil), // 10: ExecutionTerminatedEvent - (*TaskScheduledEvent)(nil), // 11: TaskScheduledEvent - (*TaskCompletedEvent)(nil), // 12: TaskCompletedEvent - (*TaskFailedEvent)(nil), // 13: TaskFailedEvent - (*SubOrchestrationInstanceCreatedEvent)(nil), // 14: SubOrchestrationInstanceCreatedEvent - (*SubOrchestrationInstanceCompletedEvent)(nil), // 15: SubOrchestrationInstanceCompletedEvent - (*SubOrchestrationInstanceFailedEvent)(nil), // 16: SubOrchestrationInstanceFailedEvent - (*TimerCreatedEvent)(nil), // 17: TimerCreatedEvent - (*TimerFiredEvent)(nil), // 18: TimerFiredEvent - (*OrchestratorStartedEvent)(nil), // 19: OrchestratorStartedEvent - (*OrchestratorCompletedEvent)(nil), // 20: OrchestratorCompletedEvent - (*EventSentEvent)(nil), // 21: EventSentEvent - (*EventRaisedEvent)(nil), // 22: EventRaisedEvent - (*GenericEvent)(nil), // 23: GenericEvent - (*HistoryStateEvent)(nil), // 24: HistoryStateEvent - (*ContinueAsNewEvent)(nil), // 25: ContinueAsNewEvent - (*ExecutionSuspendedEvent)(nil), // 26: ExecutionSuspendedEvent - (*ExecutionResumedEvent)(nil), // 27: ExecutionResumedEvent - (*HistoryEvent)(nil), // 28: HistoryEvent - (*ScheduleTaskAction)(nil), // 29: ScheduleTaskAction - (*CreateSubOrchestrationAction)(nil), // 30: CreateSubOrchestrationAction - (*CreateTimerAction)(nil), // 31: CreateTimerAction - (*SendEventAction)(nil), // 32: SendEventAction - (*CompleteOrchestrationAction)(nil), // 33: CompleteOrchestrationAction - (*TerminateOrchestrationAction)(nil), // 34: TerminateOrchestrationAction - (*OrchestratorAction)(nil), // 35: OrchestratorAction - (*OrchestratorRequest)(nil), // 36: OrchestratorRequest - (*OrchestratorResponse)(nil), // 37: OrchestratorResponse - (*CreateInstanceRequest)(nil), // 38: CreateInstanceRequest - (*OrchestrationIdReusePolicy)(nil), // 39: OrchestrationIdReusePolicy - (*CreateInstanceResponse)(nil), // 40: CreateInstanceResponse - (*GetInstanceRequest)(nil), // 41: GetInstanceRequest - (*GetInstanceResponse)(nil), // 42: GetInstanceResponse - (*RewindInstanceRequest)(nil), // 43: RewindInstanceRequest - (*RewindInstanceResponse)(nil), // 44: RewindInstanceResponse - (*OrchestrationState)(nil), // 45: OrchestrationState - (*RaiseEventRequest)(nil), // 46: RaiseEventRequest - (*RaiseEventResponse)(nil), // 47: RaiseEventResponse - (*TerminateRequest)(nil), // 48: TerminateRequest - (*TerminateResponse)(nil), // 49: TerminateResponse - (*SuspendRequest)(nil), // 50: SuspendRequest - (*SuspendResponse)(nil), // 51: SuspendResponse - (*ResumeRequest)(nil), // 52: ResumeRequest - (*ResumeResponse)(nil), // 53: ResumeResponse - (*QueryInstancesRequest)(nil), // 54: QueryInstancesRequest - (*InstanceQuery)(nil), // 55: InstanceQuery - (*QueryInstancesResponse)(nil), // 56: QueryInstancesResponse - (*PurgeInstancesRequest)(nil), // 57: PurgeInstancesRequest - (*PurgeInstanceFilter)(nil), // 58: PurgeInstanceFilter - (*PurgeInstancesResponse)(nil), // 59: PurgeInstancesResponse - (*CreateTaskHubRequest)(nil), // 60: CreateTaskHubRequest - (*CreateTaskHubResponse)(nil), // 61: CreateTaskHubResponse - (*DeleteTaskHubRequest)(nil), // 62: DeleteTaskHubRequest - (*DeleteTaskHubResponse)(nil), // 63: DeleteTaskHubResponse - (*SignalEntityRequest)(nil), // 64: SignalEntityRequest - (*SignalEntityResponse)(nil), // 65: SignalEntityResponse - (*GetEntityRequest)(nil), // 66: GetEntityRequest - (*GetEntityResponse)(nil), // 67: GetEntityResponse - (*EntityQuery)(nil), // 68: EntityQuery - (*QueryEntitiesRequest)(nil), // 69: QueryEntitiesRequest - (*QueryEntitiesResponse)(nil), // 70: QueryEntitiesResponse - (*EntityMetadata)(nil), // 71: EntityMetadata - (*CleanEntityStorageRequest)(nil), // 72: CleanEntityStorageRequest - (*CleanEntityStorageResponse)(nil), // 73: CleanEntityStorageResponse - (*OrchestratorEntityParameters)(nil), // 74: OrchestratorEntityParameters - (*EntityBatchRequest)(nil), // 75: EntityBatchRequest - (*EntityBatchResult)(nil), // 76: EntityBatchResult - (*OperationRequest)(nil), // 77: OperationRequest - (*OperationResult)(nil), // 78: OperationResult - (*OperationResultSuccess)(nil), // 79: OperationResultSuccess - (*OperationResultFailure)(nil), // 80: OperationResultFailure - (*OperationAction)(nil), // 81: OperationAction - (*SendSignalAction)(nil), // 82: SendSignalAction - (*StartNewOrchestrationAction)(nil), // 83: StartNewOrchestrationAction - (*GetWorkItemsRequest)(nil), // 84: GetWorkItemsRequest - (*WorkItem)(nil), // 85: WorkItem - (*CompleteTaskResponse)(nil), // 86: CompleteTaskResponse - (*wrappers.StringValue)(nil), // 87: google.protobuf.StringValue - (*timestamp.Timestamp)(nil), // 88: google.protobuf.Timestamp - (*wrappers.Int32Value)(nil), // 89: google.protobuf.Int32Value - (*duration.Duration)(nil), // 90: google.protobuf.Duration - (*empty.Empty)(nil), // 91: google.protobuf.Empty +var file_orchestrator_service_proto_msgTypes = make([]protoimpl.MessageInfo, 131) +var file_orchestrator_service_proto_goTypes = []any{ + (OrchestrationStatus)(0), // 0: OrchestrationStatus + (WorkerCapability)(0), // 1: WorkerCapability + (*OrchestrationInstance)(nil), // 2: OrchestrationInstance + (*ActivityRequest)(nil), // 3: ActivityRequest + (*ActivityResponse)(nil), // 4: ActivityResponse + (*TaskFailureDetails)(nil), // 5: TaskFailureDetails + (*ParentInstanceInfo)(nil), // 6: ParentInstanceInfo + (*TraceContext)(nil), // 7: TraceContext + (*ExecutionStartedEvent)(nil), // 8: ExecutionStartedEvent + (*ExecutionCompletedEvent)(nil), // 9: ExecutionCompletedEvent + (*ExecutionTerminatedEvent)(nil), // 10: ExecutionTerminatedEvent + (*TaskScheduledEvent)(nil), // 11: TaskScheduledEvent + (*TaskCompletedEvent)(nil), // 12: TaskCompletedEvent + (*TaskFailedEvent)(nil), // 13: TaskFailedEvent + (*SubOrchestrationInstanceCreatedEvent)(nil), // 14: SubOrchestrationInstanceCreatedEvent + (*SubOrchestrationInstanceCompletedEvent)(nil), // 15: SubOrchestrationInstanceCompletedEvent + (*SubOrchestrationInstanceFailedEvent)(nil), // 16: SubOrchestrationInstanceFailedEvent + (*TimerCreatedEvent)(nil), // 17: TimerCreatedEvent + (*TimerFiredEvent)(nil), // 18: TimerFiredEvent + (*OrchestratorStartedEvent)(nil), // 19: OrchestratorStartedEvent + (*OrchestratorCompletedEvent)(nil), // 20: OrchestratorCompletedEvent + (*EventSentEvent)(nil), // 21: EventSentEvent + (*EventRaisedEvent)(nil), // 22: EventRaisedEvent + (*GenericEvent)(nil), // 23: GenericEvent + (*HistoryStateEvent)(nil), // 24: HistoryStateEvent + (*ContinueAsNewEvent)(nil), // 25: ContinueAsNewEvent + (*ExecutionSuspendedEvent)(nil), // 26: ExecutionSuspendedEvent + (*ExecutionResumedEvent)(nil), // 27: ExecutionResumedEvent + (*EntityOperationSignaledEvent)(nil), // 28: EntityOperationSignaledEvent + (*EntityOperationCalledEvent)(nil), // 29: EntityOperationCalledEvent + (*EntityLockRequestedEvent)(nil), // 30: EntityLockRequestedEvent + (*EntityOperationCompletedEvent)(nil), // 31: EntityOperationCompletedEvent + (*EntityOperationFailedEvent)(nil), // 32: EntityOperationFailedEvent + (*EntityUnlockSentEvent)(nil), // 33: EntityUnlockSentEvent + (*EntityLockGrantedEvent)(nil), // 34: EntityLockGrantedEvent + (*ExecutionRewoundEvent)(nil), // 35: ExecutionRewoundEvent + (*HistoryEvent)(nil), // 36: HistoryEvent + (*ScheduleTaskAction)(nil), // 37: ScheduleTaskAction + (*CreateSubOrchestrationAction)(nil), // 38: CreateSubOrchestrationAction + (*CreateTimerAction)(nil), // 39: CreateTimerAction + (*SendEventAction)(nil), // 40: SendEventAction + (*CompleteOrchestrationAction)(nil), // 41: CompleteOrchestrationAction + (*TerminateOrchestrationAction)(nil), // 42: TerminateOrchestrationAction + (*SendEntityMessageAction)(nil), // 43: SendEntityMessageAction + (*RewindOrchestrationAction)(nil), // 44: RewindOrchestrationAction + (*OrchestratorAction)(nil), // 45: OrchestratorAction + (*OrchestrationTraceContext)(nil), // 46: OrchestrationTraceContext + (*OrchestratorRequest)(nil), // 47: OrchestratorRequest + (*OrchestratorResponse)(nil), // 48: OrchestratorResponse + (*CreateInstanceRequest)(nil), // 49: CreateInstanceRequest + (*OrchestrationIdReusePolicy)(nil), // 50: OrchestrationIdReusePolicy + (*CreateInstanceResponse)(nil), // 51: CreateInstanceResponse + (*GetInstanceRequest)(nil), // 52: GetInstanceRequest + (*GetInstanceResponse)(nil), // 53: GetInstanceResponse + (*RewindInstanceRequest)(nil), // 54: RewindInstanceRequest + (*RewindInstanceResponse)(nil), // 55: RewindInstanceResponse + (*OrchestrationState)(nil), // 56: OrchestrationState + (*RaiseEventRequest)(nil), // 57: RaiseEventRequest + (*RaiseEventResponse)(nil), // 58: RaiseEventResponse + (*TerminateRequest)(nil), // 59: TerminateRequest + (*TerminateResponse)(nil), // 60: TerminateResponse + (*SuspendRequest)(nil), // 61: SuspendRequest + (*SuspendResponse)(nil), // 62: SuspendResponse + (*ResumeRequest)(nil), // 63: ResumeRequest + (*ResumeResponse)(nil), // 64: ResumeResponse + (*QueryInstancesRequest)(nil), // 65: QueryInstancesRequest + (*InstanceQuery)(nil), // 66: InstanceQuery + (*QueryInstancesResponse)(nil), // 67: QueryInstancesResponse + (*ListInstanceIdsRequest)(nil), // 68: ListInstanceIdsRequest + (*ListInstanceIdsResponse)(nil), // 69: ListInstanceIdsResponse + (*PurgeInstancesRequest)(nil), // 70: PurgeInstancesRequest + (*PurgeInstanceFilter)(nil), // 71: PurgeInstanceFilter + (*PurgeInstancesResponse)(nil), // 72: PurgeInstancesResponse + (*RestartInstanceRequest)(nil), // 73: RestartInstanceRequest + (*RestartInstanceResponse)(nil), // 74: RestartInstanceResponse + (*CreateTaskHubRequest)(nil), // 75: CreateTaskHubRequest + (*CreateTaskHubResponse)(nil), // 76: CreateTaskHubResponse + (*DeleteTaskHubRequest)(nil), // 77: DeleteTaskHubRequest + (*DeleteTaskHubResponse)(nil), // 78: DeleteTaskHubResponse + (*SignalEntityRequest)(nil), // 79: SignalEntityRequest + (*SignalEntityResponse)(nil), // 80: SignalEntityResponse + (*GetEntityRequest)(nil), // 81: GetEntityRequest + (*GetEntityResponse)(nil), // 82: GetEntityResponse + (*EntityQuery)(nil), // 83: EntityQuery + (*QueryEntitiesRequest)(nil), // 84: QueryEntitiesRequest + (*QueryEntitiesResponse)(nil), // 85: QueryEntitiesResponse + (*EntityMetadata)(nil), // 86: EntityMetadata + (*CleanEntityStorageRequest)(nil), // 87: CleanEntityStorageRequest + (*CleanEntityStorageResponse)(nil), // 88: CleanEntityStorageResponse + (*OrchestratorEntityParameters)(nil), // 89: OrchestratorEntityParameters + (*EntityBatchRequest)(nil), // 90: EntityBatchRequest + (*EntityBatchResult)(nil), // 91: EntityBatchResult + (*EntityRequest)(nil), // 92: EntityRequest + (*OperationRequest)(nil), // 93: OperationRequest + (*OperationResult)(nil), // 94: OperationResult + (*OperationInfo)(nil), // 95: OperationInfo + (*OperationResultSuccess)(nil), // 96: OperationResultSuccess + (*OperationResultFailure)(nil), // 97: OperationResultFailure + (*OperationAction)(nil), // 98: OperationAction + (*SendSignalAction)(nil), // 99: SendSignalAction + (*StartNewOrchestrationAction)(nil), // 100: StartNewOrchestrationAction + (*AbandonActivityTaskRequest)(nil), // 101: AbandonActivityTaskRequest + (*AbandonActivityTaskResponse)(nil), // 102: AbandonActivityTaskResponse + (*AbandonOrchestrationTaskRequest)(nil), // 103: AbandonOrchestrationTaskRequest + (*AbandonOrchestrationTaskResponse)(nil), // 104: AbandonOrchestrationTaskResponse + (*AbandonEntityTaskRequest)(nil), // 105: AbandonEntityTaskRequest + (*AbandonEntityTaskResponse)(nil), // 106: AbandonEntityTaskResponse + (*SkipGracefulOrchestrationTerminationsRequest)(nil), // 107: SkipGracefulOrchestrationTerminationsRequest + (*SkipGracefulOrchestrationTerminationsResponse)(nil), // 108: SkipGracefulOrchestrationTerminationsResponse + (*GetWorkItemsRequest)(nil), // 109: GetWorkItemsRequest + (*WorkItemFilters)(nil), // 110: WorkItemFilters + (*OrchestrationFilter)(nil), // 111: OrchestrationFilter + (*ActivityFilter)(nil), // 112: ActivityFilter + (*EntityFilter)(nil), // 113: EntityFilter + (*WorkItem)(nil), // 114: WorkItem + (*CompleteTaskResponse)(nil), // 115: CompleteTaskResponse + (*HealthPing)(nil), // 116: HealthPing + (*StreamInstanceHistoryRequest)(nil), // 117: StreamInstanceHistoryRequest + (*HistoryChunk)(nil), // 118: HistoryChunk + (*InstanceBatch)(nil), // 119: InstanceBatch + nil, // 120: ActivityRequest.TagsEntry + nil, // 121: TaskFailureDetails.PropertiesEntry + nil, // 122: ExecutionStartedEvent.TagsEntry + nil, // 123: TaskScheduledEvent.TagsEntry + nil, // 124: SubOrchestrationInstanceCreatedEvent.TagsEntry + nil, // 125: ExecutionRewoundEvent.TagsEntry + nil, // 126: ScheduleTaskAction.TagsEntry + nil, // 127: CreateSubOrchestrationAction.TagsEntry + nil, // 128: CompleteOrchestrationAction.TagsEntry + nil, // 129: OrchestratorRequest.PropertiesEntry + nil, // 130: CreateInstanceRequest.TagsEntry + nil, // 131: OrchestrationState.TagsEntry + nil, // 132: EntityBatchRequest.PropertiesEntry + (*wrapperspb.StringValue)(nil), // 133: google.protobuf.StringValue + (*timestamppb.Timestamp)(nil), // 134: google.protobuf.Timestamp + (*wrapperspb.Int32Value)(nil), // 135: google.protobuf.Int32Value + (*durationpb.Duration)(nil), // 136: google.protobuf.Duration + (*wrapperspb.BoolValue)(nil), // 137: google.protobuf.BoolValue + (*structpb.Value)(nil), // 138: google.protobuf.Value + (*emptypb.Empty)(nil), // 139: google.protobuf.Empty } var file_orchestrator_service_proto_depIdxs = []int32{ - 87, // 0: OrchestrationInstance.executionId:type_name -> google.protobuf.StringValue - 87, // 1: ActivityRequest.version:type_name -> google.protobuf.StringValue - 87, // 2: ActivityRequest.input:type_name -> google.protobuf.StringValue + 133, // 0: OrchestrationInstance.executionId:type_name -> google.protobuf.StringValue + 133, // 1: ActivityRequest.version:type_name -> google.protobuf.StringValue + 133, // 2: ActivityRequest.input:type_name -> google.protobuf.StringValue 2, // 3: ActivityRequest.orchestrationInstance:type_name -> OrchestrationInstance - 87, // 4: ActivityResponse.result:type_name -> google.protobuf.StringValue - 5, // 5: ActivityResponse.failureDetails:type_name -> TaskFailureDetails - 87, // 6: TaskFailureDetails.stackTrace:type_name -> google.protobuf.StringValue - 5, // 7: TaskFailureDetails.innerFailure:type_name -> TaskFailureDetails - 87, // 8: ParentInstanceInfo.name:type_name -> google.protobuf.StringValue - 87, // 9: ParentInstanceInfo.version:type_name -> google.protobuf.StringValue - 2, // 10: ParentInstanceInfo.orchestrationInstance:type_name -> OrchestrationInstance - 87, // 11: TraceContext.traceState:type_name -> google.protobuf.StringValue - 87, // 12: ExecutionStartedEvent.version:type_name -> google.protobuf.StringValue - 87, // 13: ExecutionStartedEvent.input:type_name -> google.protobuf.StringValue - 2, // 14: ExecutionStartedEvent.orchestrationInstance:type_name -> OrchestrationInstance - 6, // 15: ExecutionStartedEvent.parentInstance:type_name -> ParentInstanceInfo - 88, // 16: ExecutionStartedEvent.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp - 7, // 17: ExecutionStartedEvent.parentTraceContext:type_name -> TraceContext - 87, // 18: ExecutionStartedEvent.orchestrationSpanID:type_name -> google.protobuf.StringValue - 0, // 19: ExecutionCompletedEvent.orchestrationStatus:type_name -> OrchestrationStatus - 87, // 20: ExecutionCompletedEvent.result:type_name -> google.protobuf.StringValue - 5, // 21: ExecutionCompletedEvent.failureDetails:type_name -> TaskFailureDetails - 87, // 22: ExecutionTerminatedEvent.input:type_name -> google.protobuf.StringValue - 87, // 23: TaskScheduledEvent.version:type_name -> google.protobuf.StringValue - 87, // 24: TaskScheduledEvent.input:type_name -> google.protobuf.StringValue - 7, // 25: TaskScheduledEvent.parentTraceContext:type_name -> TraceContext - 87, // 26: TaskCompletedEvent.result:type_name -> google.protobuf.StringValue - 5, // 27: TaskFailedEvent.failureDetails:type_name -> TaskFailureDetails - 87, // 28: SubOrchestrationInstanceCreatedEvent.version:type_name -> google.protobuf.StringValue - 87, // 29: SubOrchestrationInstanceCreatedEvent.input:type_name -> google.protobuf.StringValue - 7, // 30: SubOrchestrationInstanceCreatedEvent.parentTraceContext:type_name -> TraceContext - 87, // 31: SubOrchestrationInstanceCompletedEvent.result:type_name -> google.protobuf.StringValue - 5, // 32: SubOrchestrationInstanceFailedEvent.failureDetails:type_name -> TaskFailureDetails - 88, // 33: TimerCreatedEvent.fireAt:type_name -> google.protobuf.Timestamp - 88, // 34: TimerFiredEvent.fireAt:type_name -> google.protobuf.Timestamp - 87, // 35: EventSentEvent.input:type_name -> google.protobuf.StringValue - 87, // 36: EventRaisedEvent.input:type_name -> google.protobuf.StringValue - 87, // 37: GenericEvent.data:type_name -> google.protobuf.StringValue - 45, // 38: HistoryStateEvent.orchestrationState:type_name -> OrchestrationState - 87, // 39: ContinueAsNewEvent.input:type_name -> google.protobuf.StringValue - 87, // 40: ExecutionSuspendedEvent.input:type_name -> google.protobuf.StringValue - 87, // 41: ExecutionResumedEvent.input:type_name -> google.protobuf.StringValue - 88, // 42: HistoryEvent.timestamp:type_name -> google.protobuf.Timestamp - 8, // 43: HistoryEvent.executionStarted:type_name -> ExecutionStartedEvent - 9, // 44: HistoryEvent.executionCompleted:type_name -> ExecutionCompletedEvent - 10, // 45: HistoryEvent.executionTerminated:type_name -> ExecutionTerminatedEvent - 11, // 46: HistoryEvent.taskScheduled:type_name -> TaskScheduledEvent - 12, // 47: HistoryEvent.taskCompleted:type_name -> TaskCompletedEvent - 13, // 48: HistoryEvent.taskFailed:type_name -> TaskFailedEvent - 14, // 49: HistoryEvent.subOrchestrationInstanceCreated:type_name -> SubOrchestrationInstanceCreatedEvent - 15, // 50: HistoryEvent.subOrchestrationInstanceCompleted:type_name -> SubOrchestrationInstanceCompletedEvent - 16, // 51: HistoryEvent.subOrchestrationInstanceFailed:type_name -> SubOrchestrationInstanceFailedEvent - 17, // 52: HistoryEvent.timerCreated:type_name -> TimerCreatedEvent - 18, // 53: HistoryEvent.timerFired:type_name -> TimerFiredEvent - 19, // 54: HistoryEvent.orchestratorStarted:type_name -> OrchestratorStartedEvent - 20, // 55: HistoryEvent.orchestratorCompleted:type_name -> OrchestratorCompletedEvent - 21, // 56: HistoryEvent.eventSent:type_name -> EventSentEvent - 22, // 57: HistoryEvent.eventRaised:type_name -> EventRaisedEvent - 23, // 58: HistoryEvent.genericEvent:type_name -> GenericEvent - 24, // 59: HistoryEvent.historyState:type_name -> HistoryStateEvent - 25, // 60: HistoryEvent.continueAsNew:type_name -> ContinueAsNewEvent - 26, // 61: HistoryEvent.executionSuspended:type_name -> ExecutionSuspendedEvent - 27, // 62: HistoryEvent.executionResumed:type_name -> ExecutionResumedEvent - 87, // 63: ScheduleTaskAction.version:type_name -> google.protobuf.StringValue - 87, // 64: ScheduleTaskAction.input:type_name -> google.protobuf.StringValue - 87, // 65: CreateSubOrchestrationAction.version:type_name -> google.protobuf.StringValue - 87, // 66: CreateSubOrchestrationAction.input:type_name -> google.protobuf.StringValue - 88, // 67: CreateTimerAction.fireAt:type_name -> google.protobuf.Timestamp - 2, // 68: SendEventAction.instance:type_name -> OrchestrationInstance - 87, // 69: SendEventAction.data:type_name -> google.protobuf.StringValue - 0, // 70: CompleteOrchestrationAction.orchestrationStatus:type_name -> OrchestrationStatus - 87, // 71: CompleteOrchestrationAction.result:type_name -> google.protobuf.StringValue - 87, // 72: CompleteOrchestrationAction.details:type_name -> google.protobuf.StringValue - 87, // 73: CompleteOrchestrationAction.newVersion:type_name -> google.protobuf.StringValue - 28, // 74: CompleteOrchestrationAction.carryoverEvents:type_name -> HistoryEvent - 5, // 75: CompleteOrchestrationAction.failureDetails:type_name -> TaskFailureDetails - 87, // 76: TerminateOrchestrationAction.reason:type_name -> google.protobuf.StringValue - 29, // 77: OrchestratorAction.scheduleTask:type_name -> ScheduleTaskAction - 30, // 78: OrchestratorAction.createSubOrchestration:type_name -> CreateSubOrchestrationAction - 31, // 79: OrchestratorAction.createTimer:type_name -> CreateTimerAction - 32, // 80: OrchestratorAction.sendEvent:type_name -> SendEventAction - 33, // 81: OrchestratorAction.completeOrchestration:type_name -> CompleteOrchestrationAction - 34, // 82: OrchestratorAction.terminateOrchestration:type_name -> TerminateOrchestrationAction - 87, // 83: OrchestratorRequest.executionId:type_name -> google.protobuf.StringValue - 28, // 84: OrchestratorRequest.pastEvents:type_name -> HistoryEvent - 28, // 85: OrchestratorRequest.newEvents:type_name -> HistoryEvent - 74, // 86: OrchestratorRequest.entityParameters:type_name -> OrchestratorEntityParameters - 35, // 87: OrchestratorResponse.actions:type_name -> OrchestratorAction - 87, // 88: OrchestratorResponse.customStatus:type_name -> google.protobuf.StringValue - 87, // 89: CreateInstanceRequest.version:type_name -> google.protobuf.StringValue - 87, // 90: CreateInstanceRequest.input:type_name -> google.protobuf.StringValue - 88, // 91: CreateInstanceRequest.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp - 39, // 92: CreateInstanceRequest.orchestrationIdReusePolicy:type_name -> OrchestrationIdReusePolicy - 0, // 93: OrchestrationIdReusePolicy.operationStatus:type_name -> OrchestrationStatus - 1, // 94: OrchestrationIdReusePolicy.action:type_name -> CreateOrchestrationAction - 45, // 95: GetInstanceResponse.orchestrationState:type_name -> OrchestrationState - 87, // 96: RewindInstanceRequest.reason:type_name -> google.protobuf.StringValue - 87, // 97: OrchestrationState.version:type_name -> google.protobuf.StringValue - 0, // 98: OrchestrationState.orchestrationStatus:type_name -> OrchestrationStatus - 88, // 99: OrchestrationState.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp - 88, // 100: OrchestrationState.createdTimestamp:type_name -> google.protobuf.Timestamp - 88, // 101: OrchestrationState.lastUpdatedTimestamp:type_name -> google.protobuf.Timestamp - 87, // 102: OrchestrationState.input:type_name -> google.protobuf.StringValue - 87, // 103: OrchestrationState.output:type_name -> google.protobuf.StringValue - 87, // 104: OrchestrationState.customStatus:type_name -> google.protobuf.StringValue - 5, // 105: OrchestrationState.failureDetails:type_name -> TaskFailureDetails - 87, // 106: RaiseEventRequest.input:type_name -> google.protobuf.StringValue - 87, // 107: TerminateRequest.output:type_name -> google.protobuf.StringValue - 87, // 108: SuspendRequest.reason:type_name -> google.protobuf.StringValue - 87, // 109: ResumeRequest.reason:type_name -> google.protobuf.StringValue - 55, // 110: QueryInstancesRequest.query:type_name -> InstanceQuery - 0, // 111: InstanceQuery.runtimeStatus:type_name -> OrchestrationStatus - 88, // 112: InstanceQuery.createdTimeFrom:type_name -> google.protobuf.Timestamp - 88, // 113: InstanceQuery.createdTimeTo:type_name -> google.protobuf.Timestamp - 87, // 114: InstanceQuery.taskHubNames:type_name -> google.protobuf.StringValue - 87, // 115: InstanceQuery.continuationToken:type_name -> google.protobuf.StringValue - 87, // 116: InstanceQuery.instanceIdPrefix:type_name -> google.protobuf.StringValue - 45, // 117: QueryInstancesResponse.orchestrationState:type_name -> OrchestrationState - 87, // 118: QueryInstancesResponse.continuationToken:type_name -> google.protobuf.StringValue - 58, // 119: PurgeInstancesRequest.purgeInstanceFilter:type_name -> PurgeInstanceFilter - 88, // 120: PurgeInstanceFilter.createdTimeFrom:type_name -> google.protobuf.Timestamp - 88, // 121: PurgeInstanceFilter.createdTimeTo:type_name -> google.protobuf.Timestamp - 0, // 122: PurgeInstanceFilter.runtimeStatus:type_name -> OrchestrationStatus - 87, // 123: SignalEntityRequest.input:type_name -> google.protobuf.StringValue - 88, // 124: SignalEntityRequest.scheduledTime:type_name -> google.protobuf.Timestamp - 71, // 125: GetEntityResponse.entity:type_name -> EntityMetadata - 87, // 126: EntityQuery.instanceIdStartsWith:type_name -> google.protobuf.StringValue - 88, // 127: EntityQuery.lastModifiedFrom:type_name -> google.protobuf.Timestamp - 88, // 128: EntityQuery.lastModifiedTo:type_name -> google.protobuf.Timestamp - 89, // 129: EntityQuery.pageSize:type_name -> google.protobuf.Int32Value - 87, // 130: EntityQuery.continuationToken:type_name -> google.protobuf.StringValue - 68, // 131: QueryEntitiesRequest.query:type_name -> EntityQuery - 71, // 132: QueryEntitiesResponse.entities:type_name -> EntityMetadata - 87, // 133: QueryEntitiesResponse.continuationToken:type_name -> google.protobuf.StringValue - 88, // 134: EntityMetadata.lastModifiedTime:type_name -> google.protobuf.Timestamp - 87, // 135: EntityMetadata.lockedBy:type_name -> google.protobuf.StringValue - 87, // 136: EntityMetadata.serializedState:type_name -> google.protobuf.StringValue - 87, // 137: CleanEntityStorageRequest.continuationToken:type_name -> google.protobuf.StringValue - 87, // 138: CleanEntityStorageResponse.continuationToken:type_name -> google.protobuf.StringValue - 90, // 139: OrchestratorEntityParameters.entityMessageReorderWindow:type_name -> google.protobuf.Duration - 87, // 140: EntityBatchRequest.entityState:type_name -> google.protobuf.StringValue - 77, // 141: EntityBatchRequest.operations:type_name -> OperationRequest - 78, // 142: EntityBatchResult.results:type_name -> OperationResult - 81, // 143: EntityBatchResult.actions:type_name -> OperationAction - 87, // 144: EntityBatchResult.entityState:type_name -> google.protobuf.StringValue - 5, // 145: EntityBatchResult.failureDetails:type_name -> TaskFailureDetails - 87, // 146: OperationRequest.input:type_name -> google.protobuf.StringValue - 79, // 147: OperationResult.success:type_name -> OperationResultSuccess - 80, // 148: OperationResult.failure:type_name -> OperationResultFailure - 87, // 149: OperationResultSuccess.result:type_name -> google.protobuf.StringValue - 5, // 150: OperationResultFailure.failureDetails:type_name -> TaskFailureDetails - 82, // 151: OperationAction.sendSignal:type_name -> SendSignalAction - 83, // 152: OperationAction.startNewOrchestration:type_name -> StartNewOrchestrationAction - 87, // 153: SendSignalAction.input:type_name -> google.protobuf.StringValue - 88, // 154: SendSignalAction.scheduledTime:type_name -> google.protobuf.Timestamp - 87, // 155: StartNewOrchestrationAction.version:type_name -> google.protobuf.StringValue - 87, // 156: StartNewOrchestrationAction.input:type_name -> google.protobuf.StringValue - 88, // 157: StartNewOrchestrationAction.scheduledTime:type_name -> google.protobuf.Timestamp - 36, // 158: WorkItem.orchestratorRequest:type_name -> OrchestratorRequest - 3, // 159: WorkItem.activityRequest:type_name -> ActivityRequest - 75, // 160: WorkItem.entityRequest:type_name -> EntityBatchRequest - 91, // 161: TaskHubSidecarService.Hello:input_type -> google.protobuf.Empty - 38, // 162: TaskHubSidecarService.StartInstance:input_type -> CreateInstanceRequest - 41, // 163: TaskHubSidecarService.GetInstance:input_type -> GetInstanceRequest - 43, // 164: TaskHubSidecarService.RewindInstance:input_type -> RewindInstanceRequest - 41, // 165: TaskHubSidecarService.WaitForInstanceStart:input_type -> GetInstanceRequest - 41, // 166: TaskHubSidecarService.WaitForInstanceCompletion:input_type -> GetInstanceRequest - 46, // 167: TaskHubSidecarService.RaiseEvent:input_type -> RaiseEventRequest - 48, // 168: TaskHubSidecarService.TerminateInstance:input_type -> TerminateRequest - 50, // 169: TaskHubSidecarService.SuspendInstance:input_type -> SuspendRequest - 52, // 170: TaskHubSidecarService.ResumeInstance:input_type -> ResumeRequest - 54, // 171: TaskHubSidecarService.QueryInstances:input_type -> QueryInstancesRequest - 57, // 172: TaskHubSidecarService.PurgeInstances:input_type -> PurgeInstancesRequest - 84, // 173: TaskHubSidecarService.GetWorkItems:input_type -> GetWorkItemsRequest - 4, // 174: TaskHubSidecarService.CompleteActivityTask:input_type -> ActivityResponse - 37, // 175: TaskHubSidecarService.CompleteOrchestratorTask:input_type -> OrchestratorResponse - 76, // 176: TaskHubSidecarService.CompleteEntityTask:input_type -> EntityBatchResult - 60, // 177: TaskHubSidecarService.CreateTaskHub:input_type -> CreateTaskHubRequest - 62, // 178: TaskHubSidecarService.DeleteTaskHub:input_type -> DeleteTaskHubRequest - 64, // 179: TaskHubSidecarService.SignalEntity:input_type -> SignalEntityRequest - 66, // 180: TaskHubSidecarService.GetEntity:input_type -> GetEntityRequest - 69, // 181: TaskHubSidecarService.QueryEntities:input_type -> QueryEntitiesRequest - 72, // 182: TaskHubSidecarService.CleanEntityStorage:input_type -> CleanEntityStorageRequest - 91, // 183: TaskHubSidecarService.Hello:output_type -> google.protobuf.Empty - 40, // 184: TaskHubSidecarService.StartInstance:output_type -> CreateInstanceResponse - 42, // 185: TaskHubSidecarService.GetInstance:output_type -> GetInstanceResponse - 44, // 186: TaskHubSidecarService.RewindInstance:output_type -> RewindInstanceResponse - 42, // 187: TaskHubSidecarService.WaitForInstanceStart:output_type -> GetInstanceResponse - 42, // 188: TaskHubSidecarService.WaitForInstanceCompletion:output_type -> GetInstanceResponse - 47, // 189: TaskHubSidecarService.RaiseEvent:output_type -> RaiseEventResponse - 49, // 190: TaskHubSidecarService.TerminateInstance:output_type -> TerminateResponse - 51, // 191: TaskHubSidecarService.SuspendInstance:output_type -> SuspendResponse - 53, // 192: TaskHubSidecarService.ResumeInstance:output_type -> ResumeResponse - 56, // 193: TaskHubSidecarService.QueryInstances:output_type -> QueryInstancesResponse - 59, // 194: TaskHubSidecarService.PurgeInstances:output_type -> PurgeInstancesResponse - 85, // 195: TaskHubSidecarService.GetWorkItems:output_type -> WorkItem - 86, // 196: TaskHubSidecarService.CompleteActivityTask:output_type -> CompleteTaskResponse - 86, // 197: TaskHubSidecarService.CompleteOrchestratorTask:output_type -> CompleteTaskResponse - 86, // 198: TaskHubSidecarService.CompleteEntityTask:output_type -> CompleteTaskResponse - 61, // 199: TaskHubSidecarService.CreateTaskHub:output_type -> CreateTaskHubResponse - 63, // 200: TaskHubSidecarService.DeleteTaskHub:output_type -> DeleteTaskHubResponse - 65, // 201: TaskHubSidecarService.SignalEntity:output_type -> SignalEntityResponse - 67, // 202: TaskHubSidecarService.GetEntity:output_type -> GetEntityResponse - 70, // 203: TaskHubSidecarService.QueryEntities:output_type -> QueryEntitiesResponse - 73, // 204: TaskHubSidecarService.CleanEntityStorage:output_type -> CleanEntityStorageResponse - 183, // [183:205] is the sub-list for method output_type - 161, // [161:183] is the sub-list for method input_type - 161, // [161:161] is the sub-list for extension type_name - 161, // [161:161] is the sub-list for extension extendee - 0, // [0:161] is the sub-list for field type_name + 7, // 4: ActivityRequest.parentTraceContext:type_name -> TraceContext + 120, // 5: ActivityRequest.tags:type_name -> ActivityRequest.TagsEntry + 133, // 6: ActivityResponse.result:type_name -> google.protobuf.StringValue + 5, // 7: ActivityResponse.failureDetails:type_name -> TaskFailureDetails + 133, // 8: TaskFailureDetails.stackTrace:type_name -> google.protobuf.StringValue + 5, // 9: TaskFailureDetails.innerFailure:type_name -> TaskFailureDetails + 121, // 10: TaskFailureDetails.properties:type_name -> TaskFailureDetails.PropertiesEntry + 133, // 11: ParentInstanceInfo.name:type_name -> google.protobuf.StringValue + 133, // 12: ParentInstanceInfo.version:type_name -> google.protobuf.StringValue + 2, // 13: ParentInstanceInfo.orchestrationInstance:type_name -> OrchestrationInstance + 133, // 14: TraceContext.traceState:type_name -> google.protobuf.StringValue + 133, // 15: ExecutionStartedEvent.version:type_name -> google.protobuf.StringValue + 133, // 16: ExecutionStartedEvent.input:type_name -> google.protobuf.StringValue + 2, // 17: ExecutionStartedEvent.orchestrationInstance:type_name -> OrchestrationInstance + 6, // 18: ExecutionStartedEvent.parentInstance:type_name -> ParentInstanceInfo + 134, // 19: ExecutionStartedEvent.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp + 7, // 20: ExecutionStartedEvent.parentTraceContext:type_name -> TraceContext + 133, // 21: ExecutionStartedEvent.orchestrationSpanID:type_name -> google.protobuf.StringValue + 122, // 22: ExecutionStartedEvent.tags:type_name -> ExecutionStartedEvent.TagsEntry + 0, // 23: ExecutionCompletedEvent.orchestrationStatus:type_name -> OrchestrationStatus + 133, // 24: ExecutionCompletedEvent.result:type_name -> google.protobuf.StringValue + 5, // 25: ExecutionCompletedEvent.failureDetails:type_name -> TaskFailureDetails + 133, // 26: ExecutionTerminatedEvent.input:type_name -> google.protobuf.StringValue + 133, // 27: TaskScheduledEvent.version:type_name -> google.protobuf.StringValue + 133, // 28: TaskScheduledEvent.input:type_name -> google.protobuf.StringValue + 7, // 29: TaskScheduledEvent.parentTraceContext:type_name -> TraceContext + 123, // 30: TaskScheduledEvent.tags:type_name -> TaskScheduledEvent.TagsEntry + 133, // 31: TaskCompletedEvent.result:type_name -> google.protobuf.StringValue + 5, // 32: TaskFailedEvent.failureDetails:type_name -> TaskFailureDetails + 133, // 33: SubOrchestrationInstanceCreatedEvent.version:type_name -> google.protobuf.StringValue + 133, // 34: SubOrchestrationInstanceCreatedEvent.input:type_name -> google.protobuf.StringValue + 7, // 35: SubOrchestrationInstanceCreatedEvent.parentTraceContext:type_name -> TraceContext + 124, // 36: SubOrchestrationInstanceCreatedEvent.tags:type_name -> SubOrchestrationInstanceCreatedEvent.TagsEntry + 133, // 37: SubOrchestrationInstanceCompletedEvent.result:type_name -> google.protobuf.StringValue + 5, // 38: SubOrchestrationInstanceFailedEvent.failureDetails:type_name -> TaskFailureDetails + 134, // 39: TimerCreatedEvent.fireAt:type_name -> google.protobuf.Timestamp + 134, // 40: TimerFiredEvent.fireAt:type_name -> google.protobuf.Timestamp + 133, // 41: EventSentEvent.input:type_name -> google.protobuf.StringValue + 133, // 42: EventRaisedEvent.input:type_name -> google.protobuf.StringValue + 133, // 43: GenericEvent.data:type_name -> google.protobuf.StringValue + 56, // 44: HistoryStateEvent.orchestrationState:type_name -> OrchestrationState + 133, // 45: ContinueAsNewEvent.input:type_name -> google.protobuf.StringValue + 133, // 46: ExecutionSuspendedEvent.input:type_name -> google.protobuf.StringValue + 133, // 47: ExecutionResumedEvent.input:type_name -> google.protobuf.StringValue + 134, // 48: EntityOperationSignaledEvent.scheduledTime:type_name -> google.protobuf.Timestamp + 133, // 49: EntityOperationSignaledEvent.input:type_name -> google.protobuf.StringValue + 133, // 50: EntityOperationSignaledEvent.targetInstanceId:type_name -> google.protobuf.StringValue + 134, // 51: EntityOperationCalledEvent.scheduledTime:type_name -> google.protobuf.Timestamp + 133, // 52: EntityOperationCalledEvent.input:type_name -> google.protobuf.StringValue + 133, // 53: EntityOperationCalledEvent.parentInstanceId:type_name -> google.protobuf.StringValue + 133, // 54: EntityOperationCalledEvent.parentExecutionId:type_name -> google.protobuf.StringValue + 133, // 55: EntityOperationCalledEvent.targetInstanceId:type_name -> google.protobuf.StringValue + 133, // 56: EntityLockRequestedEvent.parentInstanceId:type_name -> google.protobuf.StringValue + 133, // 57: EntityOperationCompletedEvent.output:type_name -> google.protobuf.StringValue + 5, // 58: EntityOperationFailedEvent.failureDetails:type_name -> TaskFailureDetails + 133, // 59: EntityUnlockSentEvent.parentInstanceId:type_name -> google.protobuf.StringValue + 133, // 60: EntityUnlockSentEvent.targetInstanceId:type_name -> google.protobuf.StringValue + 133, // 61: ExecutionRewoundEvent.reason:type_name -> google.protobuf.StringValue + 133, // 62: ExecutionRewoundEvent.parentExecutionId:type_name -> google.protobuf.StringValue + 133, // 63: ExecutionRewoundEvent.instanceId:type_name -> google.protobuf.StringValue + 7, // 64: ExecutionRewoundEvent.parentTraceContext:type_name -> TraceContext + 133, // 65: ExecutionRewoundEvent.name:type_name -> google.protobuf.StringValue + 133, // 66: ExecutionRewoundEvent.version:type_name -> google.protobuf.StringValue + 133, // 67: ExecutionRewoundEvent.input:type_name -> google.protobuf.StringValue + 6, // 68: ExecutionRewoundEvent.parentInstance:type_name -> ParentInstanceInfo + 125, // 69: ExecutionRewoundEvent.tags:type_name -> ExecutionRewoundEvent.TagsEntry + 134, // 70: HistoryEvent.timestamp:type_name -> google.protobuf.Timestamp + 8, // 71: HistoryEvent.executionStarted:type_name -> ExecutionStartedEvent + 9, // 72: HistoryEvent.executionCompleted:type_name -> ExecutionCompletedEvent + 10, // 73: HistoryEvent.executionTerminated:type_name -> ExecutionTerminatedEvent + 11, // 74: HistoryEvent.taskScheduled:type_name -> TaskScheduledEvent + 12, // 75: HistoryEvent.taskCompleted:type_name -> TaskCompletedEvent + 13, // 76: HistoryEvent.taskFailed:type_name -> TaskFailedEvent + 14, // 77: HistoryEvent.subOrchestrationInstanceCreated:type_name -> SubOrchestrationInstanceCreatedEvent + 15, // 78: HistoryEvent.subOrchestrationInstanceCompleted:type_name -> SubOrchestrationInstanceCompletedEvent + 16, // 79: HistoryEvent.subOrchestrationInstanceFailed:type_name -> SubOrchestrationInstanceFailedEvent + 17, // 80: HistoryEvent.timerCreated:type_name -> TimerCreatedEvent + 18, // 81: HistoryEvent.timerFired:type_name -> TimerFiredEvent + 19, // 82: HistoryEvent.orchestratorStarted:type_name -> OrchestratorStartedEvent + 20, // 83: HistoryEvent.orchestratorCompleted:type_name -> OrchestratorCompletedEvent + 21, // 84: HistoryEvent.eventSent:type_name -> EventSentEvent + 22, // 85: HistoryEvent.eventRaised:type_name -> EventRaisedEvent + 23, // 86: HistoryEvent.genericEvent:type_name -> GenericEvent + 24, // 87: HistoryEvent.historyState:type_name -> HistoryStateEvent + 25, // 88: HistoryEvent.continueAsNew:type_name -> ContinueAsNewEvent + 26, // 89: HistoryEvent.executionSuspended:type_name -> ExecutionSuspendedEvent + 27, // 90: HistoryEvent.executionResumed:type_name -> ExecutionResumedEvent + 28, // 91: HistoryEvent.entityOperationSignaled:type_name -> EntityOperationSignaledEvent + 29, // 92: HistoryEvent.entityOperationCalled:type_name -> EntityOperationCalledEvent + 31, // 93: HistoryEvent.entityOperationCompleted:type_name -> EntityOperationCompletedEvent + 32, // 94: HistoryEvent.entityOperationFailed:type_name -> EntityOperationFailedEvent + 30, // 95: HistoryEvent.entityLockRequested:type_name -> EntityLockRequestedEvent + 34, // 96: HistoryEvent.entityLockGranted:type_name -> EntityLockGrantedEvent + 33, // 97: HistoryEvent.entityUnlockSent:type_name -> EntityUnlockSentEvent + 35, // 98: HistoryEvent.executionRewound:type_name -> ExecutionRewoundEvent + 133, // 99: ScheduleTaskAction.version:type_name -> google.protobuf.StringValue + 133, // 100: ScheduleTaskAction.input:type_name -> google.protobuf.StringValue + 126, // 101: ScheduleTaskAction.tags:type_name -> ScheduleTaskAction.TagsEntry + 7, // 102: ScheduleTaskAction.parentTraceContext:type_name -> TraceContext + 133, // 103: CreateSubOrchestrationAction.version:type_name -> google.protobuf.StringValue + 133, // 104: CreateSubOrchestrationAction.input:type_name -> google.protobuf.StringValue + 7, // 105: CreateSubOrchestrationAction.parentTraceContext:type_name -> TraceContext + 127, // 106: CreateSubOrchestrationAction.tags:type_name -> CreateSubOrchestrationAction.TagsEntry + 134, // 107: CreateTimerAction.fireAt:type_name -> google.protobuf.Timestamp + 2, // 108: SendEventAction.instance:type_name -> OrchestrationInstance + 133, // 109: SendEventAction.data:type_name -> google.protobuf.StringValue + 0, // 110: CompleteOrchestrationAction.orchestrationStatus:type_name -> OrchestrationStatus + 133, // 111: CompleteOrchestrationAction.result:type_name -> google.protobuf.StringValue + 133, // 112: CompleteOrchestrationAction.details:type_name -> google.protobuf.StringValue + 133, // 113: CompleteOrchestrationAction.newVersion:type_name -> google.protobuf.StringValue + 36, // 114: CompleteOrchestrationAction.carryoverEvents:type_name -> HistoryEvent + 5, // 115: CompleteOrchestrationAction.failureDetails:type_name -> TaskFailureDetails + 128, // 116: CompleteOrchestrationAction.tags:type_name -> CompleteOrchestrationAction.TagsEntry + 133, // 117: TerminateOrchestrationAction.reason:type_name -> google.protobuf.StringValue + 28, // 118: SendEntityMessageAction.entityOperationSignaled:type_name -> EntityOperationSignaledEvent + 29, // 119: SendEntityMessageAction.entityOperationCalled:type_name -> EntityOperationCalledEvent + 30, // 120: SendEntityMessageAction.entityLockRequested:type_name -> EntityLockRequestedEvent + 33, // 121: SendEntityMessageAction.entityUnlockSent:type_name -> EntityUnlockSentEvent + 36, // 122: RewindOrchestrationAction.newHistory:type_name -> HistoryEvent + 37, // 123: OrchestratorAction.scheduleTask:type_name -> ScheduleTaskAction + 38, // 124: OrchestratorAction.createSubOrchestration:type_name -> CreateSubOrchestrationAction + 39, // 125: OrchestratorAction.createTimer:type_name -> CreateTimerAction + 40, // 126: OrchestratorAction.sendEvent:type_name -> SendEventAction + 41, // 127: OrchestratorAction.completeOrchestration:type_name -> CompleteOrchestrationAction + 42, // 128: OrchestratorAction.terminateOrchestration:type_name -> TerminateOrchestrationAction + 43, // 129: OrchestratorAction.sendEntityMessage:type_name -> SendEntityMessageAction + 44, // 130: OrchestratorAction.rewindOrchestration:type_name -> RewindOrchestrationAction + 133, // 131: OrchestrationTraceContext.spanID:type_name -> google.protobuf.StringValue + 134, // 132: OrchestrationTraceContext.spanStartTime:type_name -> google.protobuf.Timestamp + 133, // 133: OrchestratorRequest.executionId:type_name -> google.protobuf.StringValue + 36, // 134: OrchestratorRequest.pastEvents:type_name -> HistoryEvent + 36, // 135: OrchestratorRequest.newEvents:type_name -> HistoryEvent + 89, // 136: OrchestratorRequest.entityParameters:type_name -> OrchestratorEntityParameters + 129, // 137: OrchestratorRequest.properties:type_name -> OrchestratorRequest.PropertiesEntry + 46, // 138: OrchestratorRequest.orchestrationTraceContext:type_name -> OrchestrationTraceContext + 45, // 139: OrchestratorResponse.actions:type_name -> OrchestratorAction + 133, // 140: OrchestratorResponse.customStatus:type_name -> google.protobuf.StringValue + 135, // 141: OrchestratorResponse.numEventsProcessed:type_name -> google.protobuf.Int32Value + 46, // 142: OrchestratorResponse.orchestrationTraceContext:type_name -> OrchestrationTraceContext + 135, // 143: OrchestratorResponse.chunkIndex:type_name -> google.protobuf.Int32Value + 133, // 144: CreateInstanceRequest.version:type_name -> google.protobuf.StringValue + 133, // 145: CreateInstanceRequest.input:type_name -> google.protobuf.StringValue + 134, // 146: CreateInstanceRequest.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp + 50, // 147: CreateInstanceRequest.orchestrationIdReusePolicy:type_name -> OrchestrationIdReusePolicy + 133, // 148: CreateInstanceRequest.executionId:type_name -> google.protobuf.StringValue + 130, // 149: CreateInstanceRequest.tags:type_name -> CreateInstanceRequest.TagsEntry + 7, // 150: CreateInstanceRequest.parentTraceContext:type_name -> TraceContext + 134, // 151: CreateInstanceRequest.requestTime:type_name -> google.protobuf.Timestamp + 0, // 152: OrchestrationIdReusePolicy.replaceableStatus:type_name -> OrchestrationStatus + 56, // 153: GetInstanceResponse.orchestrationState:type_name -> OrchestrationState + 133, // 154: RewindInstanceRequest.reason:type_name -> google.protobuf.StringValue + 133, // 155: OrchestrationState.version:type_name -> google.protobuf.StringValue + 0, // 156: OrchestrationState.orchestrationStatus:type_name -> OrchestrationStatus + 134, // 157: OrchestrationState.scheduledStartTimestamp:type_name -> google.protobuf.Timestamp + 134, // 158: OrchestrationState.createdTimestamp:type_name -> google.protobuf.Timestamp + 134, // 159: OrchestrationState.lastUpdatedTimestamp:type_name -> google.protobuf.Timestamp + 133, // 160: OrchestrationState.input:type_name -> google.protobuf.StringValue + 133, // 161: OrchestrationState.output:type_name -> google.protobuf.StringValue + 133, // 162: OrchestrationState.customStatus:type_name -> google.protobuf.StringValue + 5, // 163: OrchestrationState.failureDetails:type_name -> TaskFailureDetails + 133, // 164: OrchestrationState.executionId:type_name -> google.protobuf.StringValue + 134, // 165: OrchestrationState.completedTimestamp:type_name -> google.protobuf.Timestamp + 133, // 166: OrchestrationState.parentInstanceId:type_name -> google.protobuf.StringValue + 131, // 167: OrchestrationState.tags:type_name -> OrchestrationState.TagsEntry + 133, // 168: RaiseEventRequest.input:type_name -> google.protobuf.StringValue + 133, // 169: TerminateRequest.output:type_name -> google.protobuf.StringValue + 133, // 170: SuspendRequest.reason:type_name -> google.protobuf.StringValue + 133, // 171: ResumeRequest.reason:type_name -> google.protobuf.StringValue + 66, // 172: QueryInstancesRequest.query:type_name -> InstanceQuery + 0, // 173: InstanceQuery.runtimeStatus:type_name -> OrchestrationStatus + 134, // 174: InstanceQuery.createdTimeFrom:type_name -> google.protobuf.Timestamp + 134, // 175: InstanceQuery.createdTimeTo:type_name -> google.protobuf.Timestamp + 133, // 176: InstanceQuery.taskHubNames:type_name -> google.protobuf.StringValue + 133, // 177: InstanceQuery.continuationToken:type_name -> google.protobuf.StringValue + 133, // 178: InstanceQuery.instanceIdPrefix:type_name -> google.protobuf.StringValue + 56, // 179: QueryInstancesResponse.orchestrationState:type_name -> OrchestrationState + 133, // 180: QueryInstancesResponse.continuationToken:type_name -> google.protobuf.StringValue + 0, // 181: ListInstanceIdsRequest.runtimeStatus:type_name -> OrchestrationStatus + 134, // 182: ListInstanceIdsRequest.completedTimeFrom:type_name -> google.protobuf.Timestamp + 134, // 183: ListInstanceIdsRequest.completedTimeTo:type_name -> google.protobuf.Timestamp + 133, // 184: ListInstanceIdsRequest.lastInstanceKey:type_name -> google.protobuf.StringValue + 133, // 185: ListInstanceIdsResponse.lastInstanceKey:type_name -> google.protobuf.StringValue + 71, // 186: PurgeInstancesRequest.purgeInstanceFilter:type_name -> PurgeInstanceFilter + 119, // 187: PurgeInstancesRequest.instanceBatch:type_name -> InstanceBatch + 134, // 188: PurgeInstanceFilter.createdTimeFrom:type_name -> google.protobuf.Timestamp + 134, // 189: PurgeInstanceFilter.createdTimeTo:type_name -> google.protobuf.Timestamp + 0, // 190: PurgeInstanceFilter.runtimeStatus:type_name -> OrchestrationStatus + 136, // 191: PurgeInstanceFilter.timeout:type_name -> google.protobuf.Duration + 137, // 192: PurgeInstancesResponse.isComplete:type_name -> google.protobuf.BoolValue + 133, // 193: SignalEntityRequest.input:type_name -> google.protobuf.StringValue + 134, // 194: SignalEntityRequest.scheduledTime:type_name -> google.protobuf.Timestamp + 7, // 195: SignalEntityRequest.parentTraceContext:type_name -> TraceContext + 134, // 196: SignalEntityRequest.requestTime:type_name -> google.protobuf.Timestamp + 86, // 197: GetEntityResponse.entity:type_name -> EntityMetadata + 133, // 198: EntityQuery.instanceIdStartsWith:type_name -> google.protobuf.StringValue + 134, // 199: EntityQuery.lastModifiedFrom:type_name -> google.protobuf.Timestamp + 134, // 200: EntityQuery.lastModifiedTo:type_name -> google.protobuf.Timestamp + 135, // 201: EntityQuery.pageSize:type_name -> google.protobuf.Int32Value + 133, // 202: EntityQuery.continuationToken:type_name -> google.protobuf.StringValue + 83, // 203: QueryEntitiesRequest.query:type_name -> EntityQuery + 86, // 204: QueryEntitiesResponse.entities:type_name -> EntityMetadata + 133, // 205: QueryEntitiesResponse.continuationToken:type_name -> google.protobuf.StringValue + 134, // 206: EntityMetadata.lastModifiedTime:type_name -> google.protobuf.Timestamp + 133, // 207: EntityMetadata.lockedBy:type_name -> google.protobuf.StringValue + 133, // 208: EntityMetadata.serializedState:type_name -> google.protobuf.StringValue + 133, // 209: CleanEntityStorageRequest.continuationToken:type_name -> google.protobuf.StringValue + 133, // 210: CleanEntityStorageResponse.continuationToken:type_name -> google.protobuf.StringValue + 136, // 211: OrchestratorEntityParameters.entityMessageReorderWindow:type_name -> google.protobuf.Duration + 133, // 212: EntityBatchRequest.entityState:type_name -> google.protobuf.StringValue + 93, // 213: EntityBatchRequest.operations:type_name -> OperationRequest + 132, // 214: EntityBatchRequest.properties:type_name -> EntityBatchRequest.PropertiesEntry + 94, // 215: EntityBatchResult.results:type_name -> OperationResult + 98, // 216: EntityBatchResult.actions:type_name -> OperationAction + 133, // 217: EntityBatchResult.entityState:type_name -> google.protobuf.StringValue + 5, // 218: EntityBatchResult.failureDetails:type_name -> TaskFailureDetails + 95, // 219: EntityBatchResult.operationInfos:type_name -> OperationInfo + 133, // 220: EntityRequest.entityState:type_name -> google.protobuf.StringValue + 36, // 221: EntityRequest.operationRequests:type_name -> HistoryEvent + 133, // 222: OperationRequest.input:type_name -> google.protobuf.StringValue + 7, // 223: OperationRequest.traceContext:type_name -> TraceContext + 96, // 224: OperationResult.success:type_name -> OperationResultSuccess + 97, // 225: OperationResult.failure:type_name -> OperationResultFailure + 2, // 226: OperationInfo.responseDestination:type_name -> OrchestrationInstance + 133, // 227: OperationResultSuccess.result:type_name -> google.protobuf.StringValue + 134, // 228: OperationResultSuccess.startTimeUtc:type_name -> google.protobuf.Timestamp + 134, // 229: OperationResultSuccess.endTimeUtc:type_name -> google.protobuf.Timestamp + 5, // 230: OperationResultFailure.failureDetails:type_name -> TaskFailureDetails + 134, // 231: OperationResultFailure.startTimeUtc:type_name -> google.protobuf.Timestamp + 134, // 232: OperationResultFailure.endTimeUtc:type_name -> google.protobuf.Timestamp + 99, // 233: OperationAction.sendSignal:type_name -> SendSignalAction + 100, // 234: OperationAction.startNewOrchestration:type_name -> StartNewOrchestrationAction + 133, // 235: SendSignalAction.input:type_name -> google.protobuf.StringValue + 134, // 236: SendSignalAction.scheduledTime:type_name -> google.protobuf.Timestamp + 134, // 237: SendSignalAction.requestTime:type_name -> google.protobuf.Timestamp + 7, // 238: SendSignalAction.parentTraceContext:type_name -> TraceContext + 133, // 239: StartNewOrchestrationAction.version:type_name -> google.protobuf.StringValue + 133, // 240: StartNewOrchestrationAction.input:type_name -> google.protobuf.StringValue + 134, // 241: StartNewOrchestrationAction.scheduledTime:type_name -> google.protobuf.Timestamp + 134, // 242: StartNewOrchestrationAction.requestTime:type_name -> google.protobuf.Timestamp + 7, // 243: StartNewOrchestrationAction.parentTraceContext:type_name -> TraceContext + 119, // 244: SkipGracefulOrchestrationTerminationsRequest.instanceBatch:type_name -> InstanceBatch + 133, // 245: SkipGracefulOrchestrationTerminationsRequest.reason:type_name -> google.protobuf.StringValue + 1, // 246: GetWorkItemsRequest.capabilities:type_name -> WorkerCapability + 110, // 247: GetWorkItemsRequest.workItemFilters:type_name -> WorkItemFilters + 111, // 248: WorkItemFilters.orchestrations:type_name -> OrchestrationFilter + 112, // 249: WorkItemFilters.activities:type_name -> ActivityFilter + 113, // 250: WorkItemFilters.entities:type_name -> EntityFilter + 47, // 251: WorkItem.orchestratorRequest:type_name -> OrchestratorRequest + 3, // 252: WorkItem.activityRequest:type_name -> ActivityRequest + 90, // 253: WorkItem.entityRequest:type_name -> EntityBatchRequest + 116, // 254: WorkItem.healthPing:type_name -> HealthPing + 92, // 255: WorkItem.entityRequestV2:type_name -> EntityRequest + 133, // 256: StreamInstanceHistoryRequest.executionId:type_name -> google.protobuf.StringValue + 36, // 257: HistoryChunk.events:type_name -> HistoryEvent + 138, // 258: TaskFailureDetails.PropertiesEntry.value:type_name -> google.protobuf.Value + 138, // 259: OrchestratorRequest.PropertiesEntry.value:type_name -> google.protobuf.Value + 138, // 260: EntityBatchRequest.PropertiesEntry.value:type_name -> google.protobuf.Value + 139, // 261: TaskHubSidecarService.Hello:input_type -> google.protobuf.Empty + 49, // 262: TaskHubSidecarService.StartInstance:input_type -> CreateInstanceRequest + 52, // 263: TaskHubSidecarService.GetInstance:input_type -> GetInstanceRequest + 54, // 264: TaskHubSidecarService.RewindInstance:input_type -> RewindInstanceRequest + 73, // 265: TaskHubSidecarService.RestartInstance:input_type -> RestartInstanceRequest + 52, // 266: TaskHubSidecarService.WaitForInstanceStart:input_type -> GetInstanceRequest + 52, // 267: TaskHubSidecarService.WaitForInstanceCompletion:input_type -> GetInstanceRequest + 57, // 268: TaskHubSidecarService.RaiseEvent:input_type -> RaiseEventRequest + 59, // 269: TaskHubSidecarService.TerminateInstance:input_type -> TerminateRequest + 61, // 270: TaskHubSidecarService.SuspendInstance:input_type -> SuspendRequest + 63, // 271: TaskHubSidecarService.ResumeInstance:input_type -> ResumeRequest + 65, // 272: TaskHubSidecarService.QueryInstances:input_type -> QueryInstancesRequest + 68, // 273: TaskHubSidecarService.ListInstanceIds:input_type -> ListInstanceIdsRequest + 70, // 274: TaskHubSidecarService.PurgeInstances:input_type -> PurgeInstancesRequest + 109, // 275: TaskHubSidecarService.GetWorkItems:input_type -> GetWorkItemsRequest + 4, // 276: TaskHubSidecarService.CompleteActivityTask:input_type -> ActivityResponse + 48, // 277: TaskHubSidecarService.CompleteOrchestratorTask:input_type -> OrchestratorResponse + 91, // 278: TaskHubSidecarService.CompleteEntityTask:input_type -> EntityBatchResult + 117, // 279: TaskHubSidecarService.StreamInstanceHistory:input_type -> StreamInstanceHistoryRequest + 75, // 280: TaskHubSidecarService.CreateTaskHub:input_type -> CreateTaskHubRequest + 77, // 281: TaskHubSidecarService.DeleteTaskHub:input_type -> DeleteTaskHubRequest + 79, // 282: TaskHubSidecarService.SignalEntity:input_type -> SignalEntityRequest + 81, // 283: TaskHubSidecarService.GetEntity:input_type -> GetEntityRequest + 84, // 284: TaskHubSidecarService.QueryEntities:input_type -> QueryEntitiesRequest + 87, // 285: TaskHubSidecarService.CleanEntityStorage:input_type -> CleanEntityStorageRequest + 101, // 286: TaskHubSidecarService.AbandonTaskActivityWorkItem:input_type -> AbandonActivityTaskRequest + 103, // 287: TaskHubSidecarService.AbandonTaskOrchestratorWorkItem:input_type -> AbandonOrchestrationTaskRequest + 105, // 288: TaskHubSidecarService.AbandonTaskEntityWorkItem:input_type -> AbandonEntityTaskRequest + 107, // 289: TaskHubSidecarService.SkipGracefulOrchestrationTerminations:input_type -> SkipGracefulOrchestrationTerminationsRequest + 139, // 290: TaskHubSidecarService.Hello:output_type -> google.protobuf.Empty + 51, // 291: TaskHubSidecarService.StartInstance:output_type -> CreateInstanceResponse + 53, // 292: TaskHubSidecarService.GetInstance:output_type -> GetInstanceResponse + 55, // 293: TaskHubSidecarService.RewindInstance:output_type -> RewindInstanceResponse + 74, // 294: TaskHubSidecarService.RestartInstance:output_type -> RestartInstanceResponse + 53, // 295: TaskHubSidecarService.WaitForInstanceStart:output_type -> GetInstanceResponse + 53, // 296: TaskHubSidecarService.WaitForInstanceCompletion:output_type -> GetInstanceResponse + 58, // 297: TaskHubSidecarService.RaiseEvent:output_type -> RaiseEventResponse + 60, // 298: TaskHubSidecarService.TerminateInstance:output_type -> TerminateResponse + 62, // 299: TaskHubSidecarService.SuspendInstance:output_type -> SuspendResponse + 64, // 300: TaskHubSidecarService.ResumeInstance:output_type -> ResumeResponse + 67, // 301: TaskHubSidecarService.QueryInstances:output_type -> QueryInstancesResponse + 69, // 302: TaskHubSidecarService.ListInstanceIds:output_type -> ListInstanceIdsResponse + 72, // 303: TaskHubSidecarService.PurgeInstances:output_type -> PurgeInstancesResponse + 114, // 304: TaskHubSidecarService.GetWorkItems:output_type -> WorkItem + 115, // 305: TaskHubSidecarService.CompleteActivityTask:output_type -> CompleteTaskResponse + 115, // 306: TaskHubSidecarService.CompleteOrchestratorTask:output_type -> CompleteTaskResponse + 115, // 307: TaskHubSidecarService.CompleteEntityTask:output_type -> CompleteTaskResponse + 118, // 308: TaskHubSidecarService.StreamInstanceHistory:output_type -> HistoryChunk + 76, // 309: TaskHubSidecarService.CreateTaskHub:output_type -> CreateTaskHubResponse + 78, // 310: TaskHubSidecarService.DeleteTaskHub:output_type -> DeleteTaskHubResponse + 80, // 311: TaskHubSidecarService.SignalEntity:output_type -> SignalEntityResponse + 82, // 312: TaskHubSidecarService.GetEntity:output_type -> GetEntityResponse + 85, // 313: TaskHubSidecarService.QueryEntities:output_type -> QueryEntitiesResponse + 88, // 314: TaskHubSidecarService.CleanEntityStorage:output_type -> CleanEntityStorageResponse + 102, // 315: TaskHubSidecarService.AbandonTaskActivityWorkItem:output_type -> AbandonActivityTaskResponse + 104, // 316: TaskHubSidecarService.AbandonTaskOrchestratorWorkItem:output_type -> AbandonOrchestrationTaskResponse + 106, // 317: TaskHubSidecarService.AbandonTaskEntityWorkItem:output_type -> AbandonEntityTaskResponse + 108, // 318: TaskHubSidecarService.SkipGracefulOrchestrationTerminations:output_type -> SkipGracefulOrchestrationTerminationsResponse + 290, // [290:319] is the sub-list for method output_type + 261, // [261:290] is the sub-list for method input_type + 261, // [261:261] is the sub-list for extension type_name + 261, // [261:261] is the sub-list for extension extendee + 0, // [0:261] is the sub-list for field type_name } func init() { file_orchestrator_service_proto_init() } @@ -7020,1029 +9145,7 @@ func file_orchestrator_service_proto_init() { if File_orchestrator_service_proto != nil { return } - if !protoimpl.UnsafeEnabled { - file_orchestrator_service_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestrationInstance); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ActivityRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ActivityResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TaskFailureDetails); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ParentInstanceInfo); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TraceContext); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutionStartedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutionCompletedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutionTerminatedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TaskScheduledEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TaskCompletedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TaskFailedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubOrchestrationInstanceCreatedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubOrchestrationInstanceCompletedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubOrchestrationInstanceFailedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimerCreatedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TimerFiredEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorStartedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[18].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorCompletedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[19].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EventSentEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[20].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EventRaisedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[21].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GenericEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[22].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HistoryStateEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[23].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ContinueAsNewEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[24].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutionSuspendedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[25].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ExecutionResumedEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*HistoryEvent); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ScheduleTaskAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateSubOrchestrationAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[29].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTimerAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[30].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SendEventAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[31].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CompleteOrchestrationAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[32].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TerminateOrchestrationAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[33].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[34].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[35].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[36].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateInstanceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[37].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestrationIdReusePolicy); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[38].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateInstanceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[39].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetInstanceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[40].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetInstanceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[41].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RewindInstanceRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[42].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RewindInstanceResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[43].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestrationState); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[44].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RaiseEventRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[45].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*RaiseEventResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[46].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TerminateRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[47].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TerminateResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SuspendRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SuspendResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[50].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResumeRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[51].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*ResumeResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[52].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryInstancesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[53].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*InstanceQuery); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[54].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryInstancesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[55].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PurgeInstancesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[56].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PurgeInstanceFilter); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[57].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*PurgeInstancesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[58].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTaskHubRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[59].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CreateTaskHubResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[60].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteTaskHubRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[61].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*DeleteTaskHubResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[62].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SignalEntityRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[63].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SignalEntityResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[64].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEntityRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[65].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetEntityResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[66].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EntityQuery); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[67].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryEntitiesRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[68].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*QueryEntitiesResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[69].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EntityMetadata); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[70].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanEntityStorageRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[71].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CleanEntityStorageResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[72].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OrchestratorEntityParameters); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[73].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EntityBatchRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[74].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*EntityBatchResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[75].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[76].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationResult); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[77].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationResultSuccess); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[78].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationResultFailure); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[79].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*OperationAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[80].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SendSignalAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[81].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*StartNewOrchestrationAction); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[82].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*GetWorkItemsRequest); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[83].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*WorkItem); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - file_orchestrator_service_proto_msgTypes[84].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*CompleteTaskResponse); i { - case 0: - return &v.state - case 1: - return &v.sizeCache - case 2: - return &v.unknownFields - default: - return nil - } - } - } - file_orchestrator_service_proto_msgTypes[26].OneofWrappers = []interface{}{ + file_orchestrator_service_proto_msgTypes[34].OneofWrappers = []any{ (*HistoryEvent_ExecutionStarted)(nil), (*HistoryEvent_ExecutionCompleted)(nil), (*HistoryEvent_ExecutionTerminated)(nil), @@ -8063,39 +9166,58 @@ func file_orchestrator_service_proto_init() { (*HistoryEvent_ContinueAsNew)(nil), (*HistoryEvent_ExecutionSuspended)(nil), (*HistoryEvent_ExecutionResumed)(nil), - } - file_orchestrator_service_proto_msgTypes[33].OneofWrappers = []interface{}{ + (*HistoryEvent_EntityOperationSignaled)(nil), + (*HistoryEvent_EntityOperationCalled)(nil), + (*HistoryEvent_EntityOperationCompleted)(nil), + (*HistoryEvent_EntityOperationFailed)(nil), + (*HistoryEvent_EntityLockRequested)(nil), + (*HistoryEvent_EntityLockGranted)(nil), + (*HistoryEvent_EntityUnlockSent)(nil), + (*HistoryEvent_ExecutionRewound)(nil), + } + file_orchestrator_service_proto_msgTypes[41].OneofWrappers = []any{ + (*SendEntityMessageAction_EntityOperationSignaled)(nil), + (*SendEntityMessageAction_EntityOperationCalled)(nil), + (*SendEntityMessageAction_EntityLockRequested)(nil), + (*SendEntityMessageAction_EntityUnlockSent)(nil), + } + file_orchestrator_service_proto_msgTypes[43].OneofWrappers = []any{ (*OrchestratorAction_ScheduleTask)(nil), (*OrchestratorAction_CreateSubOrchestration)(nil), (*OrchestratorAction_CreateTimer)(nil), (*OrchestratorAction_SendEvent)(nil), (*OrchestratorAction_CompleteOrchestration)(nil), (*OrchestratorAction_TerminateOrchestration)(nil), + (*OrchestratorAction_SendEntityMessage)(nil), + (*OrchestratorAction_RewindOrchestration)(nil), } - file_orchestrator_service_proto_msgTypes[55].OneofWrappers = []interface{}{ + file_orchestrator_service_proto_msgTypes[68].OneofWrappers = []any{ (*PurgeInstancesRequest_InstanceId)(nil), (*PurgeInstancesRequest_PurgeInstanceFilter)(nil), + (*PurgeInstancesRequest_InstanceBatch)(nil), } - file_orchestrator_service_proto_msgTypes[76].OneofWrappers = []interface{}{ + file_orchestrator_service_proto_msgTypes[92].OneofWrappers = []any{ (*OperationResult_Success)(nil), (*OperationResult_Failure)(nil), } - file_orchestrator_service_proto_msgTypes[79].OneofWrappers = []interface{}{ + file_orchestrator_service_proto_msgTypes[96].OneofWrappers = []any{ (*OperationAction_SendSignal)(nil), (*OperationAction_StartNewOrchestration)(nil), } - file_orchestrator_service_proto_msgTypes[83].OneofWrappers = []interface{}{ + file_orchestrator_service_proto_msgTypes[112].OneofWrappers = []any{ (*WorkItem_OrchestratorRequest)(nil), (*WorkItem_ActivityRequest)(nil), (*WorkItem_EntityRequest)(nil), + (*WorkItem_HealthPing)(nil), + (*WorkItem_EntityRequestV2)(nil), } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_orchestrator_service_proto_rawDesc, + RawDescriptor: unsafe.Slice(unsafe.StringData(file_orchestrator_service_proto_rawDesc), len(file_orchestrator_service_proto_rawDesc)), NumEnums: 2, - NumMessages: 85, + NumMessages: 131, NumExtensions: 0, NumServices: 1, }, @@ -8105,7 +9227,6 @@ func file_orchestrator_service_proto_init() { MessageInfos: file_orchestrator_service_proto_msgTypes, }.Build() File_orchestrator_service_proto = out.File - file_orchestrator_service_proto_rawDesc = nil file_orchestrator_service_proto_goTypes = nil file_orchestrator_service_proto_depIdxs = nil } diff --git a/internal/protos/orchestrator_service_grpc.pb.go b/internal/protos/orchestrator_service_grpc.pb.go index 0bc36435..d14acd41 100644 --- a/internal/protos/orchestrator_service_grpc.pb.go +++ b/internal/protos/orchestrator_service_grpc.pb.go @@ -3,48 +3,55 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.3.0 -// - protoc v3.12.4 +// - protoc-gen-go-grpc v1.6.2 +// - protoc v7.36.1 // source: orchestrator_service.proto package protos import ( context "context" - empty "google.golang.org/protobuf/types/known/emptypb" grpc "google.golang.org/grpc" codes "google.golang.org/grpc/codes" status "google.golang.org/grpc/status" + emptypb "google.golang.org/protobuf/types/known/emptypb" ) // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -// Requires gRPC-Go v1.32.0 or later. -const _ = grpc.SupportPackageIsVersion7 +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 const ( - TaskHubSidecarService_Hello_FullMethodName = "/TaskHubSidecarService/Hello" - TaskHubSidecarService_StartInstance_FullMethodName = "/TaskHubSidecarService/StartInstance" - TaskHubSidecarService_GetInstance_FullMethodName = "/TaskHubSidecarService/GetInstance" - TaskHubSidecarService_RewindInstance_FullMethodName = "/TaskHubSidecarService/RewindInstance" - TaskHubSidecarService_WaitForInstanceStart_FullMethodName = "/TaskHubSidecarService/WaitForInstanceStart" - TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName = "/TaskHubSidecarService/WaitForInstanceCompletion" - TaskHubSidecarService_RaiseEvent_FullMethodName = "/TaskHubSidecarService/RaiseEvent" - TaskHubSidecarService_TerminateInstance_FullMethodName = "/TaskHubSidecarService/TerminateInstance" - TaskHubSidecarService_SuspendInstance_FullMethodName = "/TaskHubSidecarService/SuspendInstance" - TaskHubSidecarService_ResumeInstance_FullMethodName = "/TaskHubSidecarService/ResumeInstance" - TaskHubSidecarService_QueryInstances_FullMethodName = "/TaskHubSidecarService/QueryInstances" - TaskHubSidecarService_PurgeInstances_FullMethodName = "/TaskHubSidecarService/PurgeInstances" - TaskHubSidecarService_GetWorkItems_FullMethodName = "/TaskHubSidecarService/GetWorkItems" - TaskHubSidecarService_CompleteActivityTask_FullMethodName = "/TaskHubSidecarService/CompleteActivityTask" - TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName = "/TaskHubSidecarService/CompleteOrchestratorTask" - TaskHubSidecarService_CompleteEntityTask_FullMethodName = "/TaskHubSidecarService/CompleteEntityTask" - TaskHubSidecarService_CreateTaskHub_FullMethodName = "/TaskHubSidecarService/CreateTaskHub" - TaskHubSidecarService_DeleteTaskHub_FullMethodName = "/TaskHubSidecarService/DeleteTaskHub" - TaskHubSidecarService_SignalEntity_FullMethodName = "/TaskHubSidecarService/SignalEntity" - TaskHubSidecarService_GetEntity_FullMethodName = "/TaskHubSidecarService/GetEntity" - TaskHubSidecarService_QueryEntities_FullMethodName = "/TaskHubSidecarService/QueryEntities" - TaskHubSidecarService_CleanEntityStorage_FullMethodName = "/TaskHubSidecarService/CleanEntityStorage" + TaskHubSidecarService_Hello_FullMethodName = "/TaskHubSidecarService/Hello" + TaskHubSidecarService_StartInstance_FullMethodName = "/TaskHubSidecarService/StartInstance" + TaskHubSidecarService_GetInstance_FullMethodName = "/TaskHubSidecarService/GetInstance" + TaskHubSidecarService_RewindInstance_FullMethodName = "/TaskHubSidecarService/RewindInstance" + TaskHubSidecarService_RestartInstance_FullMethodName = "/TaskHubSidecarService/RestartInstance" + TaskHubSidecarService_WaitForInstanceStart_FullMethodName = "/TaskHubSidecarService/WaitForInstanceStart" + TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName = "/TaskHubSidecarService/WaitForInstanceCompletion" + TaskHubSidecarService_RaiseEvent_FullMethodName = "/TaskHubSidecarService/RaiseEvent" + TaskHubSidecarService_TerminateInstance_FullMethodName = "/TaskHubSidecarService/TerminateInstance" + TaskHubSidecarService_SuspendInstance_FullMethodName = "/TaskHubSidecarService/SuspendInstance" + TaskHubSidecarService_ResumeInstance_FullMethodName = "/TaskHubSidecarService/ResumeInstance" + TaskHubSidecarService_QueryInstances_FullMethodName = "/TaskHubSidecarService/QueryInstances" + TaskHubSidecarService_ListInstanceIds_FullMethodName = "/TaskHubSidecarService/ListInstanceIds" + TaskHubSidecarService_PurgeInstances_FullMethodName = "/TaskHubSidecarService/PurgeInstances" + TaskHubSidecarService_GetWorkItems_FullMethodName = "/TaskHubSidecarService/GetWorkItems" + TaskHubSidecarService_CompleteActivityTask_FullMethodName = "/TaskHubSidecarService/CompleteActivityTask" + TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName = "/TaskHubSidecarService/CompleteOrchestratorTask" + TaskHubSidecarService_CompleteEntityTask_FullMethodName = "/TaskHubSidecarService/CompleteEntityTask" + TaskHubSidecarService_StreamInstanceHistory_FullMethodName = "/TaskHubSidecarService/StreamInstanceHistory" + TaskHubSidecarService_CreateTaskHub_FullMethodName = "/TaskHubSidecarService/CreateTaskHub" + TaskHubSidecarService_DeleteTaskHub_FullMethodName = "/TaskHubSidecarService/DeleteTaskHub" + TaskHubSidecarService_SignalEntity_FullMethodName = "/TaskHubSidecarService/SignalEntity" + TaskHubSidecarService_GetEntity_FullMethodName = "/TaskHubSidecarService/GetEntity" + TaskHubSidecarService_QueryEntities_FullMethodName = "/TaskHubSidecarService/QueryEntities" + TaskHubSidecarService_CleanEntityStorage_FullMethodName = "/TaskHubSidecarService/CleanEntityStorage" + TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName = "/TaskHubSidecarService/AbandonTaskActivityWorkItem" + TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName = "/TaskHubSidecarService/AbandonTaskOrchestratorWorkItem" + TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName = "/TaskHubSidecarService/AbandonTaskEntityWorkItem" + TaskHubSidecarService_SkipGracefulOrchestrationTerminations_FullMethodName = "/TaskHubSidecarService/SkipGracefulOrchestrationTerminations" ) // TaskHubSidecarServiceClient is the client API for TaskHubSidecarService service. @@ -52,13 +59,15 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. type TaskHubSidecarServiceClient interface { // Sends a hello request to the sidecar service. - Hello(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) + Hello(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) // Starts a new orchestration instance. StartInstance(ctx context.Context, in *CreateInstanceRequest, opts ...grpc.CallOption) (*CreateInstanceResponse, error) // Gets the status of an existing orchestration instance. GetInstance(ctx context.Context, in *GetInstanceRequest, opts ...grpc.CallOption) (*GetInstanceResponse, error) // Rewinds an orchestration instance to last known good state and replays from there. RewindInstance(ctx context.Context, in *RewindInstanceRequest, opts ...grpc.CallOption) (*RewindInstanceResponse, error) + // Restarts an orchestration instance. + RestartInstance(ctx context.Context, in *RestartInstanceRequest, opts ...grpc.CallOption) (*RestartInstanceResponse, error) // Waits for an orchestration instance to reach a running or completion state. WaitForInstanceStart(ctx context.Context, in *GetInstanceRequest, opts ...grpc.CallOption) (*GetInstanceResponse, error) // Waits for an orchestration instance to reach a completion state (completed, failed, terminated, etc.). @@ -72,11 +81,14 @@ type TaskHubSidecarServiceClient interface { // Resumes a suspended orchestration instance. ResumeInstance(ctx context.Context, in *ResumeRequest, opts ...grpc.CallOption) (*ResumeResponse, error) QueryInstances(ctx context.Context, in *QueryInstancesRequest, opts ...grpc.CallOption) (*QueryInstancesResponse, error) + ListInstanceIds(ctx context.Context, in *ListInstanceIdsRequest, opts ...grpc.CallOption) (*ListInstanceIdsResponse, error) PurgeInstances(ctx context.Context, in *PurgeInstancesRequest, opts ...grpc.CallOption) (*PurgeInstancesResponse, error) - GetWorkItems(ctx context.Context, in *GetWorkItemsRequest, opts ...grpc.CallOption) (TaskHubSidecarService_GetWorkItemsClient, error) + GetWorkItems(ctx context.Context, in *GetWorkItemsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WorkItem], error) CompleteActivityTask(ctx context.Context, in *ActivityResponse, opts ...grpc.CallOption) (*CompleteTaskResponse, error) CompleteOrchestratorTask(ctx context.Context, in *OrchestratorResponse, opts ...grpc.CallOption) (*CompleteTaskResponse, error) CompleteEntityTask(ctx context.Context, in *EntityBatchResult, opts ...grpc.CallOption) (*CompleteTaskResponse, error) + // Gets the history of an orchestration instance as a stream of events. + StreamInstanceHistory(ctx context.Context, in *StreamInstanceHistoryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[HistoryChunk], error) // Deletes and Creates the necessary resources for the orchestration service and the instance store CreateTaskHub(ctx context.Context, in *CreateTaskHubRequest, opts ...grpc.CallOption) (*CreateTaskHubResponse, error) // Deletes the resources for the orchestration service and optionally the instance store @@ -89,6 +101,15 @@ type TaskHubSidecarServiceClient interface { QueryEntities(ctx context.Context, in *QueryEntitiesRequest, opts ...grpc.CallOption) (*QueryEntitiesResponse, error) // clean entity storage CleanEntityStorage(ctx context.Context, in *CleanEntityStorageRequest, opts ...grpc.CallOption) (*CleanEntityStorageResponse, error) + // Abandons a single work item + AbandonTaskActivityWorkItem(ctx context.Context, in *AbandonActivityTaskRequest, opts ...grpc.CallOption) (*AbandonActivityTaskResponse, error) + // Abandon an orchestration work item + AbandonTaskOrchestratorWorkItem(ctx context.Context, in *AbandonOrchestrationTaskRequest, opts ...grpc.CallOption) (*AbandonOrchestrationTaskResponse, error) + // Abandon an entity work item + AbandonTaskEntityWorkItem(ctx context.Context, in *AbandonEntityTaskRequest, opts ...grpc.CallOption) (*AbandonEntityTaskResponse, error) + // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". + // Note that a maximum of 500 orchestrations can be terminated at a time using this method. + SkipGracefulOrchestrationTerminations(ctx context.Context, in *SkipGracefulOrchestrationTerminationsRequest, opts ...grpc.CallOption) (*SkipGracefulOrchestrationTerminationsResponse, error) } type taskHubSidecarServiceClient struct { @@ -99,9 +120,10 @@ func NewTaskHubSidecarServiceClient(cc grpc.ClientConnInterface) TaskHubSidecarS return &taskHubSidecarServiceClient{cc} } -func (c *taskHubSidecarServiceClient) Hello(ctx context.Context, in *empty.Empty, opts ...grpc.CallOption) (*empty.Empty, error) { - out := new(empty.Empty) - err := c.cc.Invoke(ctx, TaskHubSidecarService_Hello_FullMethodName, in, out, opts...) +func (c *taskHubSidecarServiceClient) Hello(ctx context.Context, in *emptypb.Empty, opts ...grpc.CallOption) (*emptypb.Empty, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(emptypb.Empty) + err := c.cc.Invoke(ctx, TaskHubSidecarService_Hello_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -109,8 +131,9 @@ func (c *taskHubSidecarServiceClient) Hello(ctx context.Context, in *empty.Empty } func (c *taskHubSidecarServiceClient) StartInstance(ctx context.Context, in *CreateInstanceRequest, opts ...grpc.CallOption) (*CreateInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CreateInstanceResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_StartInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_StartInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -118,8 +141,9 @@ func (c *taskHubSidecarServiceClient) StartInstance(ctx context.Context, in *Cre } func (c *taskHubSidecarServiceClient) GetInstance(ctx context.Context, in *GetInstanceRequest, opts ...grpc.CallOption) (*GetInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetInstanceResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_GetInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_GetInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -127,8 +151,19 @@ func (c *taskHubSidecarServiceClient) GetInstance(ctx context.Context, in *GetIn } func (c *taskHubSidecarServiceClient) RewindInstance(ctx context.Context, in *RewindInstanceRequest, opts ...grpc.CallOption) (*RewindInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RewindInstanceResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_RewindInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_RewindInstance_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) RestartInstance(ctx context.Context, in *RestartInstanceRequest, opts ...grpc.CallOption) (*RestartInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RestartInstanceResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_RestartInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -136,8 +171,9 @@ func (c *taskHubSidecarServiceClient) RewindInstance(ctx context.Context, in *Re } func (c *taskHubSidecarServiceClient) WaitForInstanceStart(ctx context.Context, in *GetInstanceRequest, opts ...grpc.CallOption) (*GetInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetInstanceResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_WaitForInstanceStart_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_WaitForInstanceStart_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -145,8 +181,9 @@ func (c *taskHubSidecarServiceClient) WaitForInstanceStart(ctx context.Context, } func (c *taskHubSidecarServiceClient) WaitForInstanceCompletion(ctx context.Context, in *GetInstanceRequest, opts ...grpc.CallOption) (*GetInstanceResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetInstanceResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -154,8 +191,9 @@ func (c *taskHubSidecarServiceClient) WaitForInstanceCompletion(ctx context.Cont } func (c *taskHubSidecarServiceClient) RaiseEvent(ctx context.Context, in *RaiseEventRequest, opts ...grpc.CallOption) (*RaiseEventResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(RaiseEventResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_RaiseEvent_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_RaiseEvent_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -163,8 +201,9 @@ func (c *taskHubSidecarServiceClient) RaiseEvent(ctx context.Context, in *RaiseE } func (c *taskHubSidecarServiceClient) TerminateInstance(ctx context.Context, in *TerminateRequest, opts ...grpc.CallOption) (*TerminateResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(TerminateResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_TerminateInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_TerminateInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -172,8 +211,9 @@ func (c *taskHubSidecarServiceClient) TerminateInstance(ctx context.Context, in } func (c *taskHubSidecarServiceClient) SuspendInstance(ctx context.Context, in *SuspendRequest, opts ...grpc.CallOption) (*SuspendResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(SuspendResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_SuspendInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_SuspendInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -181,8 +221,9 @@ func (c *taskHubSidecarServiceClient) SuspendInstance(ctx context.Context, in *S } func (c *taskHubSidecarServiceClient) ResumeInstance(ctx context.Context, in *ResumeRequest, opts ...grpc.CallOption) (*ResumeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(ResumeResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_ResumeInstance_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_ResumeInstance_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -190,8 +231,19 @@ func (c *taskHubSidecarServiceClient) ResumeInstance(ctx context.Context, in *Re } func (c *taskHubSidecarServiceClient) QueryInstances(ctx context.Context, in *QueryInstancesRequest, opts ...grpc.CallOption) (*QueryInstancesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(QueryInstancesResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_QueryInstances_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_QueryInstances_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) ListInstanceIds(ctx context.Context, in *ListInstanceIdsRequest, opts ...grpc.CallOption) (*ListInstanceIdsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListInstanceIdsResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_ListInstanceIds_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -199,20 +251,22 @@ func (c *taskHubSidecarServiceClient) QueryInstances(ctx context.Context, in *Qu } func (c *taskHubSidecarServiceClient) PurgeInstances(ctx context.Context, in *PurgeInstancesRequest, opts ...grpc.CallOption) (*PurgeInstancesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(PurgeInstancesResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_PurgeInstances_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_PurgeInstances_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } -func (c *taskHubSidecarServiceClient) GetWorkItems(ctx context.Context, in *GetWorkItemsRequest, opts ...grpc.CallOption) (TaskHubSidecarService_GetWorkItemsClient, error) { - stream, err := c.cc.NewStream(ctx, &TaskHubSidecarService_ServiceDesc.Streams[0], TaskHubSidecarService_GetWorkItems_FullMethodName, opts...) +func (c *taskHubSidecarServiceClient) GetWorkItems(ctx context.Context, in *GetWorkItemsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[WorkItem], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TaskHubSidecarService_ServiceDesc.Streams[0], TaskHubSidecarService_GetWorkItems_FullMethodName, cOpts...) if err != nil { return nil, err } - x := &taskHubSidecarServiceGetWorkItemsClient{stream} + x := &grpc.GenericClientStream[GetWorkItemsRequest, WorkItem]{ClientStream: stream} if err := x.ClientStream.SendMsg(in); err != nil { return nil, err } @@ -222,26 +276,13 @@ func (c *taskHubSidecarServiceClient) GetWorkItems(ctx context.Context, in *GetW return x, nil } -type TaskHubSidecarService_GetWorkItemsClient interface { - Recv() (*WorkItem, error) - grpc.ClientStream -} - -type taskHubSidecarServiceGetWorkItemsClient struct { - grpc.ClientStream -} - -func (x *taskHubSidecarServiceGetWorkItemsClient) Recv() (*WorkItem, error) { - m := new(WorkItem) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TaskHubSidecarService_GetWorkItemsClient = grpc.ServerStreamingClient[WorkItem] func (c *taskHubSidecarServiceClient) CompleteActivityTask(ctx context.Context, in *ActivityResponse, opts ...grpc.CallOption) (*CompleteTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CompleteTaskResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteActivityTask_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteActivityTask_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -249,8 +290,9 @@ func (c *taskHubSidecarServiceClient) CompleteActivityTask(ctx context.Context, } func (c *taskHubSidecarServiceClient) CompleteOrchestratorTask(ctx context.Context, in *OrchestratorResponse, opts ...grpc.CallOption) (*CompleteTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CompleteTaskResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -258,17 +300,38 @@ func (c *taskHubSidecarServiceClient) CompleteOrchestratorTask(ctx context.Conte } func (c *taskHubSidecarServiceClient) CompleteEntityTask(ctx context.Context, in *EntityBatchResult, opts ...grpc.CallOption) (*CompleteTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CompleteTaskResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteEntityTask_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_CompleteEntityTask_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } return out, nil } +func (c *taskHubSidecarServiceClient) StreamInstanceHistory(ctx context.Context, in *StreamInstanceHistoryRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[HistoryChunk], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TaskHubSidecarService_ServiceDesc.Streams[1], TaskHubSidecarService_StreamInstanceHistory_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[StreamInstanceHistoryRequest, HistoryChunk]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TaskHubSidecarService_StreamInstanceHistoryClient = grpc.ServerStreamingClient[HistoryChunk] + func (c *taskHubSidecarServiceClient) CreateTaskHub(ctx context.Context, in *CreateTaskHubRequest, opts ...grpc.CallOption) (*CreateTaskHubResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CreateTaskHubResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_CreateTaskHub_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_CreateTaskHub_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -276,8 +339,9 @@ func (c *taskHubSidecarServiceClient) CreateTaskHub(ctx context.Context, in *Cre } func (c *taskHubSidecarServiceClient) DeleteTaskHub(ctx context.Context, in *DeleteTaskHubRequest, opts ...grpc.CallOption) (*DeleteTaskHubResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(DeleteTaskHubResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_DeleteTaskHub_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_DeleteTaskHub_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -285,8 +349,9 @@ func (c *taskHubSidecarServiceClient) DeleteTaskHub(ctx context.Context, in *Del } func (c *taskHubSidecarServiceClient) SignalEntity(ctx context.Context, in *SignalEntityRequest, opts ...grpc.CallOption) (*SignalEntityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(SignalEntityResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_SignalEntity_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_SignalEntity_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -294,8 +359,9 @@ func (c *taskHubSidecarServiceClient) SignalEntity(ctx context.Context, in *Sign } func (c *taskHubSidecarServiceClient) GetEntity(ctx context.Context, in *GetEntityRequest, opts ...grpc.CallOption) (*GetEntityResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(GetEntityResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_GetEntity_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_GetEntity_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -303,8 +369,9 @@ func (c *taskHubSidecarServiceClient) GetEntity(ctx context.Context, in *GetEnti } func (c *taskHubSidecarServiceClient) QueryEntities(ctx context.Context, in *QueryEntitiesRequest, opts ...grpc.CallOption) (*QueryEntitiesResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(QueryEntitiesResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_QueryEntities_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_QueryEntities_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -312,8 +379,49 @@ func (c *taskHubSidecarServiceClient) QueryEntities(ctx context.Context, in *Que } func (c *taskHubSidecarServiceClient) CleanEntityStorage(ctx context.Context, in *CleanEntityStorageRequest, opts ...grpc.CallOption) (*CleanEntityStorageResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) out := new(CleanEntityStorageResponse) - err := c.cc.Invoke(ctx, TaskHubSidecarService_CleanEntityStorage_FullMethodName, in, out, opts...) + err := c.cc.Invoke(ctx, TaskHubSidecarService_CleanEntityStorage_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) AbandonTaskActivityWorkItem(ctx context.Context, in *AbandonActivityTaskRequest, opts ...grpc.CallOption) (*AbandonActivityTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AbandonActivityTaskResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) AbandonTaskOrchestratorWorkItem(ctx context.Context, in *AbandonOrchestrationTaskRequest, opts ...grpc.CallOption) (*AbandonOrchestrationTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AbandonOrchestrationTaskResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) AbandonTaskEntityWorkItem(ctx context.Context, in *AbandonEntityTaskRequest, opts ...grpc.CallOption) (*AbandonEntityTaskResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(AbandonEntityTaskResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *taskHubSidecarServiceClient) SkipGracefulOrchestrationTerminations(ctx context.Context, in *SkipGracefulOrchestrationTerminationsRequest, opts ...grpc.CallOption) (*SkipGracefulOrchestrationTerminationsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SkipGracefulOrchestrationTerminationsResponse) + err := c.cc.Invoke(ctx, TaskHubSidecarService_SkipGracefulOrchestrationTerminations_FullMethodName, in, out, cOpts...) if err != nil { return nil, err } @@ -322,16 +430,18 @@ func (c *taskHubSidecarServiceClient) CleanEntityStorage(ctx context.Context, in // TaskHubSidecarServiceServer is the server API for TaskHubSidecarService service. // All implementations must embed UnimplementedTaskHubSidecarServiceServer -// for forward compatibility +// for forward compatibility. type TaskHubSidecarServiceServer interface { // Sends a hello request to the sidecar service. - Hello(context.Context, *empty.Empty) (*empty.Empty, error) + Hello(context.Context, *emptypb.Empty) (*emptypb.Empty, error) // Starts a new orchestration instance. StartInstance(context.Context, *CreateInstanceRequest) (*CreateInstanceResponse, error) // Gets the status of an existing orchestration instance. GetInstance(context.Context, *GetInstanceRequest) (*GetInstanceResponse, error) // Rewinds an orchestration instance to last known good state and replays from there. RewindInstance(context.Context, *RewindInstanceRequest) (*RewindInstanceResponse, error) + // Restarts an orchestration instance. + RestartInstance(context.Context, *RestartInstanceRequest) (*RestartInstanceResponse, error) // Waits for an orchestration instance to reach a running or completion state. WaitForInstanceStart(context.Context, *GetInstanceRequest) (*GetInstanceResponse, error) // Waits for an orchestration instance to reach a completion state (completed, failed, terminated, etc.). @@ -345,11 +455,14 @@ type TaskHubSidecarServiceServer interface { // Resumes a suspended orchestration instance. ResumeInstance(context.Context, *ResumeRequest) (*ResumeResponse, error) QueryInstances(context.Context, *QueryInstancesRequest) (*QueryInstancesResponse, error) + ListInstanceIds(context.Context, *ListInstanceIdsRequest) (*ListInstanceIdsResponse, error) PurgeInstances(context.Context, *PurgeInstancesRequest) (*PurgeInstancesResponse, error) - GetWorkItems(*GetWorkItemsRequest, TaskHubSidecarService_GetWorkItemsServer) error + GetWorkItems(*GetWorkItemsRequest, grpc.ServerStreamingServer[WorkItem]) error CompleteActivityTask(context.Context, *ActivityResponse) (*CompleteTaskResponse, error) CompleteOrchestratorTask(context.Context, *OrchestratorResponse) (*CompleteTaskResponse, error) CompleteEntityTask(context.Context, *EntityBatchResult) (*CompleteTaskResponse, error) + // Gets the history of an orchestration instance as a stream of events. + StreamInstanceHistory(*StreamInstanceHistoryRequest, grpc.ServerStreamingServer[HistoryChunk]) error // Deletes and Creates the necessary resources for the orchestration service and the instance store CreateTaskHub(context.Context, *CreateTaskHubRequest) (*CreateTaskHubResponse, error) // Deletes the resources for the orchestration service and optionally the instance store @@ -362,80 +475,114 @@ type TaskHubSidecarServiceServer interface { QueryEntities(context.Context, *QueryEntitiesRequest) (*QueryEntitiesResponse, error) // clean entity storage CleanEntityStorage(context.Context, *CleanEntityStorageRequest) (*CleanEntityStorageResponse, error) + // Abandons a single work item + AbandonTaskActivityWorkItem(context.Context, *AbandonActivityTaskRequest) (*AbandonActivityTaskResponse, error) + // Abandon an orchestration work item + AbandonTaskOrchestratorWorkItem(context.Context, *AbandonOrchestrationTaskRequest) (*AbandonOrchestrationTaskResponse, error) + // Abandon an entity work item + AbandonTaskEntityWorkItem(context.Context, *AbandonEntityTaskRequest) (*AbandonEntityTaskResponse, error) + // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". + // Note that a maximum of 500 orchestrations can be terminated at a time using this method. + SkipGracefulOrchestrationTerminations(context.Context, *SkipGracefulOrchestrationTerminationsRequest) (*SkipGracefulOrchestrationTerminationsResponse, error) mustEmbedUnimplementedTaskHubSidecarServiceServer() } -// UnimplementedTaskHubSidecarServiceServer must be embedded to have forward compatible implementations. -type UnimplementedTaskHubSidecarServiceServer struct { -} +// UnimplementedTaskHubSidecarServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedTaskHubSidecarServiceServer struct{} -func (UnimplementedTaskHubSidecarServiceServer) Hello(context.Context, *empty.Empty) (*empty.Empty, error) { - return nil, status.Errorf(codes.Unimplemented, "method Hello not implemented") +func (UnimplementedTaskHubSidecarServiceServer) Hello(context.Context, *emptypb.Empty) (*emptypb.Empty, error) { + return nil, status.Error(codes.Unimplemented, "method Hello not implemented") } func (UnimplementedTaskHubSidecarServiceServer) StartInstance(context.Context, *CreateInstanceRequest) (*CreateInstanceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method StartInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method StartInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) GetInstance(context.Context, *GetInstanceRequest) (*GetInstanceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method GetInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) RewindInstance(context.Context, *RewindInstanceRequest) (*RewindInstanceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RewindInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method RewindInstance not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) RestartInstance(context.Context, *RestartInstanceRequest) (*RestartInstanceResponse, error) { + return nil, status.Error(codes.Unimplemented, "method RestartInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) WaitForInstanceStart(context.Context, *GetInstanceRequest) (*GetInstanceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method WaitForInstanceStart not implemented") + return nil, status.Error(codes.Unimplemented, "method WaitForInstanceStart not implemented") } func (UnimplementedTaskHubSidecarServiceServer) WaitForInstanceCompletion(context.Context, *GetInstanceRequest) (*GetInstanceResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method WaitForInstanceCompletion not implemented") + return nil, status.Error(codes.Unimplemented, "method WaitForInstanceCompletion not implemented") } func (UnimplementedTaskHubSidecarServiceServer) RaiseEvent(context.Context, *RaiseEventRequest) (*RaiseEventResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method RaiseEvent not implemented") + return nil, status.Error(codes.Unimplemented, "method RaiseEvent not implemented") } func (UnimplementedTaskHubSidecarServiceServer) TerminateInstance(context.Context, *TerminateRequest) (*TerminateResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method TerminateInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method TerminateInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) SuspendInstance(context.Context, *SuspendRequest) (*SuspendResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method SuspendInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method SuspendInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) ResumeInstance(context.Context, *ResumeRequest) (*ResumeResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method ResumeInstance not implemented") + return nil, status.Error(codes.Unimplemented, "method ResumeInstance not implemented") } func (UnimplementedTaskHubSidecarServiceServer) QueryInstances(context.Context, *QueryInstancesRequest) (*QueryInstancesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method QueryInstances not implemented") + return nil, status.Error(codes.Unimplemented, "method QueryInstances not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) ListInstanceIds(context.Context, *ListInstanceIdsRequest) (*ListInstanceIdsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListInstanceIds not implemented") } func (UnimplementedTaskHubSidecarServiceServer) PurgeInstances(context.Context, *PurgeInstancesRequest) (*PurgeInstancesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method PurgeInstances not implemented") + return nil, status.Error(codes.Unimplemented, "method PurgeInstances not implemented") } -func (UnimplementedTaskHubSidecarServiceServer) GetWorkItems(*GetWorkItemsRequest, TaskHubSidecarService_GetWorkItemsServer) error { - return status.Errorf(codes.Unimplemented, "method GetWorkItems not implemented") +func (UnimplementedTaskHubSidecarServiceServer) GetWorkItems(*GetWorkItemsRequest, grpc.ServerStreamingServer[WorkItem]) error { + return status.Error(codes.Unimplemented, "method GetWorkItems not implemented") } func (UnimplementedTaskHubSidecarServiceServer) CompleteActivityTask(context.Context, *ActivityResponse) (*CompleteTaskResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CompleteActivityTask not implemented") + return nil, status.Error(codes.Unimplemented, "method CompleteActivityTask not implemented") } func (UnimplementedTaskHubSidecarServiceServer) CompleteOrchestratorTask(context.Context, *OrchestratorResponse) (*CompleteTaskResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CompleteOrchestratorTask not implemented") + return nil, status.Error(codes.Unimplemented, "method CompleteOrchestratorTask not implemented") } func (UnimplementedTaskHubSidecarServiceServer) CompleteEntityTask(context.Context, *EntityBatchResult) (*CompleteTaskResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CompleteEntityTask not implemented") + return nil, status.Error(codes.Unimplemented, "method CompleteEntityTask not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) StreamInstanceHistory(*StreamInstanceHistoryRequest, grpc.ServerStreamingServer[HistoryChunk]) error { + return status.Error(codes.Unimplemented, "method StreamInstanceHistory not implemented") } func (UnimplementedTaskHubSidecarServiceServer) CreateTaskHub(context.Context, *CreateTaskHubRequest) (*CreateTaskHubResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CreateTaskHub not implemented") + return nil, status.Error(codes.Unimplemented, "method CreateTaskHub not implemented") } func (UnimplementedTaskHubSidecarServiceServer) DeleteTaskHub(context.Context, *DeleteTaskHubRequest) (*DeleteTaskHubResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteTaskHub not implemented") + return nil, status.Error(codes.Unimplemented, "method DeleteTaskHub not implemented") } func (UnimplementedTaskHubSidecarServiceServer) SignalEntity(context.Context, *SignalEntityRequest) (*SignalEntityResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method SignalEntity not implemented") + return nil, status.Error(codes.Unimplemented, "method SignalEntity not implemented") } func (UnimplementedTaskHubSidecarServiceServer) GetEntity(context.Context, *GetEntityRequest) (*GetEntityResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetEntity not implemented") + return nil, status.Error(codes.Unimplemented, "method GetEntity not implemented") } func (UnimplementedTaskHubSidecarServiceServer) QueryEntities(context.Context, *QueryEntitiesRequest) (*QueryEntitiesResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method QueryEntities not implemented") + return nil, status.Error(codes.Unimplemented, "method QueryEntities not implemented") } func (UnimplementedTaskHubSidecarServiceServer) CleanEntityStorage(context.Context, *CleanEntityStorageRequest) (*CleanEntityStorageResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method CleanEntityStorage not implemented") + return nil, status.Error(codes.Unimplemented, "method CleanEntityStorage not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) AbandonTaskActivityWorkItem(context.Context, *AbandonActivityTaskRequest) (*AbandonActivityTaskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AbandonTaskActivityWorkItem not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) AbandonTaskOrchestratorWorkItem(context.Context, *AbandonOrchestrationTaskRequest) (*AbandonOrchestrationTaskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AbandonTaskOrchestratorWorkItem not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) AbandonTaskEntityWorkItem(context.Context, *AbandonEntityTaskRequest) (*AbandonEntityTaskResponse, error) { + return nil, status.Error(codes.Unimplemented, "method AbandonTaskEntityWorkItem not implemented") +} +func (UnimplementedTaskHubSidecarServiceServer) SkipGracefulOrchestrationTerminations(context.Context, *SkipGracefulOrchestrationTerminationsRequest) (*SkipGracefulOrchestrationTerminationsResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SkipGracefulOrchestrationTerminations not implemented") } func (UnimplementedTaskHubSidecarServiceServer) mustEmbedUnimplementedTaskHubSidecarServiceServer() {} +func (UnimplementedTaskHubSidecarServiceServer) testEmbeddedByValue() {} // UnsafeTaskHubSidecarServiceServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to TaskHubSidecarServiceServer will @@ -445,11 +592,18 @@ type UnsafeTaskHubSidecarServiceServer interface { } func RegisterTaskHubSidecarServiceServer(s grpc.ServiceRegistrar, srv TaskHubSidecarServiceServer) { + // If the following call panics, it indicates UnimplementedTaskHubSidecarServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } s.RegisterService(&TaskHubSidecarService_ServiceDesc, srv) } func _TaskHubSidecarService_Hello_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(empty.Empty) + in := new(emptypb.Empty) if err := dec(in); err != nil { return nil, err } @@ -461,7 +615,7 @@ func _TaskHubSidecarService_Hello_Handler(srv interface{}, ctx context.Context, FullMethod: TaskHubSidecarService_Hello_FullMethodName, } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(TaskHubSidecarServiceServer).Hello(ctx, req.(*empty.Empty)) + return srv.(TaskHubSidecarServiceServer).Hello(ctx, req.(*emptypb.Empty)) } return interceptor(ctx, in, info, handler) } @@ -520,6 +674,24 @@ func _TaskHubSidecarService_RewindInstance_Handler(srv interface{}, ctx context. return interceptor(ctx, in, info, handler) } +func _TaskHubSidecarService_RestartInstance_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RestartInstanceRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).RestartInstance(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_RestartInstance_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).RestartInstance(ctx, req.(*RestartInstanceRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _TaskHubSidecarService_WaitForInstanceStart_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(GetInstanceRequest) if err := dec(in); err != nil { @@ -646,6 +818,24 @@ func _TaskHubSidecarService_QueryInstances_Handler(srv interface{}, ctx context. return interceptor(ctx, in, info, handler) } +func _TaskHubSidecarService_ListInstanceIds_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListInstanceIdsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).ListInstanceIds(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_ListInstanceIds_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).ListInstanceIds(ctx, req.(*ListInstanceIdsRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _TaskHubSidecarService_PurgeInstances_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(PurgeInstancesRequest) if err := dec(in); err != nil { @@ -669,21 +859,11 @@ func _TaskHubSidecarService_GetWorkItems_Handler(srv interface{}, stream grpc.Se if err := stream.RecvMsg(m); err != nil { return err } - return srv.(TaskHubSidecarServiceServer).GetWorkItems(m, &taskHubSidecarServiceGetWorkItemsServer{stream}) -} - -type TaskHubSidecarService_GetWorkItemsServer interface { - Send(*WorkItem) error - grpc.ServerStream -} - -type taskHubSidecarServiceGetWorkItemsServer struct { - grpc.ServerStream + return srv.(TaskHubSidecarServiceServer).GetWorkItems(m, &grpc.GenericServerStream[GetWorkItemsRequest, WorkItem]{ServerStream: stream}) } -func (x *taskHubSidecarServiceGetWorkItemsServer) Send(m *WorkItem) error { - return x.ServerStream.SendMsg(m) -} +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TaskHubSidecarService_GetWorkItemsServer = grpc.ServerStreamingServer[WorkItem] func _TaskHubSidecarService_CompleteActivityTask_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ActivityResponse) @@ -739,6 +919,17 @@ func _TaskHubSidecarService_CompleteEntityTask_Handler(srv interface{}, ctx cont return interceptor(ctx, in, info, handler) } +func _TaskHubSidecarService_StreamInstanceHistory_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(StreamInstanceHistoryRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(TaskHubSidecarServiceServer).StreamInstanceHistory(m, &grpc.GenericServerStream[StreamInstanceHistoryRequest, HistoryChunk]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TaskHubSidecarService_StreamInstanceHistoryServer = grpc.ServerStreamingServer[HistoryChunk] + func _TaskHubSidecarService_CreateTaskHub_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateTaskHubRequest) if err := dec(in); err != nil { @@ -847,6 +1038,78 @@ func _TaskHubSidecarService_CleanEntityStorage_Handler(srv interface{}, ctx cont return interceptor(ctx, in, info, handler) } +func _TaskHubSidecarService_AbandonTaskActivityWorkItem_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AbandonActivityTaskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).AbandonTaskActivityWorkItem(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_AbandonTaskActivityWorkItem_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).AbandonTaskActivityWorkItem(ctx, req.(*AbandonActivityTaskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AbandonOrchestrationTaskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).AbandonTaskOrchestratorWorkItem(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).AbandonTaskOrchestratorWorkItem(ctx, req.(*AbandonOrchestrationTaskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TaskHubSidecarService_AbandonTaskEntityWorkItem_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AbandonEntityTaskRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).AbandonTaskEntityWorkItem(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_AbandonTaskEntityWorkItem_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).AbandonTaskEntityWorkItem(ctx, req.(*AbandonEntityTaskRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _TaskHubSidecarService_SkipGracefulOrchestrationTerminations_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SkipGracefulOrchestrationTerminationsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(TaskHubSidecarServiceServer).SkipGracefulOrchestrationTerminations(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: TaskHubSidecarService_SkipGracefulOrchestrationTerminations_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(TaskHubSidecarServiceServer).SkipGracefulOrchestrationTerminations(ctx, req.(*SkipGracefulOrchestrationTerminationsRequest)) + } + return interceptor(ctx, in, info, handler) +} + // TaskHubSidecarService_ServiceDesc is the grpc.ServiceDesc for TaskHubSidecarService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -870,6 +1133,10 @@ var TaskHubSidecarService_ServiceDesc = grpc.ServiceDesc{ MethodName: "RewindInstance", Handler: _TaskHubSidecarService_RewindInstance_Handler, }, + { + MethodName: "RestartInstance", + Handler: _TaskHubSidecarService_RestartInstance_Handler, + }, { MethodName: "WaitForInstanceStart", Handler: _TaskHubSidecarService_WaitForInstanceStart_Handler, @@ -898,6 +1165,10 @@ var TaskHubSidecarService_ServiceDesc = grpc.ServiceDesc{ MethodName: "QueryInstances", Handler: _TaskHubSidecarService_QueryInstances_Handler, }, + { + MethodName: "ListInstanceIds", + Handler: _TaskHubSidecarService_ListInstanceIds_Handler, + }, { MethodName: "PurgeInstances", Handler: _TaskHubSidecarService_PurgeInstances_Handler, @@ -938,6 +1209,22 @@ var TaskHubSidecarService_ServiceDesc = grpc.ServiceDesc{ MethodName: "CleanEntityStorage", Handler: _TaskHubSidecarService_CleanEntityStorage_Handler, }, + { + MethodName: "AbandonTaskActivityWorkItem", + Handler: _TaskHubSidecarService_AbandonTaskActivityWorkItem_Handler, + }, + { + MethodName: "AbandonTaskOrchestratorWorkItem", + Handler: _TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_Handler, + }, + { + MethodName: "AbandonTaskEntityWorkItem", + Handler: _TaskHubSidecarService_AbandonTaskEntityWorkItem_Handler, + }, + { + MethodName: "SkipGracefulOrchestrationTerminations", + Handler: _TaskHubSidecarService_SkipGracefulOrchestrationTerminations_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -945,6 +1232,11 @@ var TaskHubSidecarService_ServiceDesc = grpc.ServiceDesc{ Handler: _TaskHubSidecarService_GetWorkItems_Handler, ServerStreams: true, }, + { + StreamName: "StreamInstanceHistory", + Handler: _TaskHubSidecarService_StreamInstanceHistory_Handler, + ServerStreams: true, + }, }, Metadata: "orchestrator_service.proto", } diff --git a/internal/tagcodec/tags.go b/internal/tagcodec/tags.go new file mode 100644 index 00000000..01d53c12 --- /dev/null +++ b/internal/tagcodec/tags.go @@ -0,0 +1,95 @@ +package tagcodec + +import ( + "maps" + "strings" +) + +const ( + ContextPrefix = "__durabletask.context." + ContextFieldPrefix = ContextPrefix + "field." + ContextEncodingTag = ContextPrefix + "encoding" + UserTagPrefix = "__durabletask.tags." +) + +func EncodeContextFields(fields map[string]string) map[string]string { + if len(fields) == 0 { + return nil + } + encoded := make(map[string]string, len(fields)) + encoded[ContextEncodingTag] = "1" + for key, value := range fields { + if strings.HasPrefix(key, ContextPrefix) || strings.HasPrefix(key, UserTagPrefix) { + continue + } + encoded[ContextFieldPrefix+key] = value + } + return encoded +} + +func DecodeContextFields(tags map[string]string) map[string]string { + var fields map[string]string + encoded := tags[ContextEncodingTag] == "1" + for key, value := range tags { + switch { + case strings.HasPrefix(key, ContextFieldPrefix): + if fields == nil { + fields = make(map[string]string) + } + fields[strings.TrimPrefix(key, ContextFieldPrefix)] = value + case !encoded && !strings.HasPrefix(key, ContextPrefix) && !strings.HasPrefix(key, UserTagPrefix): + // Backward compatibility for histories written before fields had a namespace. + if fields == nil { + fields = make(map[string]string) + } + fields[key] = value + } + } + return fields +} + +func EncodeUserTags(tags map[string]string) map[string]string { + if len(tags) == 0 { + return nil + } + encoded := make(map[string]string, len(tags)+1) + encoded[ContextEncodingTag] = "1" + for key, value := range tags { + encoded[key] = value + } + return encoded +} + +func DecodeUserTags(tags map[string]string) map[string]string { + var decoded map[string]string + for key, value := range tags { + switch { + case strings.HasPrefix(key, UserTagPrefix): + key = strings.TrimPrefix(key, UserTagPrefix) + case strings.HasPrefix(key, ContextPrefix): + continue + } + if decoded == nil { + decoded = make(map[string]string) + } + decoded[key] = value + } + return decoded +} + +// DecodeUserTagsOrPlain supports services that return user tags without the Go +// SDK namespace while filtering Durable Task context metadata. +func DecodeUserTagsOrPlain(tags map[string]string) map[string]string { + return DecodeUserTags(tags) +} + +func Merge(destination map[string]string, encoded map[string]string) map[string]string { + if len(encoded) == 0 { + return destination + } + if destination == nil { + destination = make(map[string]string, len(encoded)) + } + maps.Copy(destination, encoded) + return destination +} diff --git a/main.go b/main.go deleted file mode 100644 index f816ec16..00000000 --- a/main.go +++ /dev/null @@ -1,53 +0,0 @@ -package main - -import ( - "context" - "flag" - "fmt" - "log" - "net" - - "google.golang.org/grpc" - - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" -) - -var ( - port = flag.Int("port", 4001, "The server port") - host = flag.String("host", "localhost", "The host to bind to") - dbFilePath = flag.String("db", "", "The path to the sqlite file to use (or create if not exists)") - ctx = context.Background() -) - -func main() { - // Parse command-line arguments - flag.Parse() - - grpcServer := grpc.NewServer() - worker := createTaskHubWorker(grpcServer, *dbFilePath, backend.DefaultLogger()) - if err := worker.Start(ctx); err != nil { - log.Fatalf("failed to start worker: %v", err) - } - - lis, err := net.Listen("tcp", fmt.Sprintf("%s:%d", *host, *port)) - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - - fmt.Printf("server listening at %v\n", lis.Addr()) - if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) - } -} - -func createTaskHubWorker(server *grpc.Server, sqliteFilePath string, logger backend.Logger) backend.TaskHubWorker { - sqliteOptions := sqlite.NewSqliteOptions(sqliteFilePath) - be := sqlite.NewSqliteBackend(sqliteOptions, logger) - executor, registerFn := backend.NewGrpcExecutor(be, logger) - registerFn(server) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - return taskHubWorker -} diff --git a/payload/azure_blob.go b/payload/azure_blob.go new file mode 100644 index 00000000..494de7c3 --- /dev/null +++ b/payload/azure_blob.go @@ -0,0 +1,655 @@ +package payload + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/md5" //nolint:gosec // Azure Blob uses MD5 as a transport checksum. + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/url" + "strconv" + "strings" + "sync" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/microsoft/durabletask-go/api" +) + +const ( + azureBlobTokenV1 = api.AzureBlobPayloadReferencePrefixV1 + azureBlobTokenV2 = api.AzureBlobPayloadReferencePrefixV2 +) + +// AzureBlobStoreOptions configures AzureBlobStore. Exactly one authentication +// mode must be supplied: ConnectionString, or AccountURL with Credential. +type AzureBlobStoreOptions struct { + ConnectionString string + AccountURL string + Credential azcore.TokenCredential + Container string + // CompressionEnabled defaults to true when nil. + CompressionEnabled *bool + // AllowedHosts explicitly permits cross-account blob:v2 reads with Credential. + // .NET-authored blobs on these hosts may not include Go integrity metadata; + // they remain bounded by MaxPayloadBytes. + AllowedHosts []string + AllowInsecureHTTP bool + MaxPayloadBytes int +} + +// AzureBlobStore stores payloads in Azure Blob Storage using .NET-compatible +// blob:v2 tokens. It also resolves .NET's legacy blob:v1 tokens. +type AzureBlobStore struct { + client *azblob.Client + credential azcore.TokenCredential + clientOptions *azblob.ClientOptions + container string + containerURL *url.URL + allowHosts map[string]struct{} + allowInsecureHTTP bool + compressionEnabled bool + maxPayloadBytes int + + containerMu sync.Mutex + // containerGeneration is the initialization currently believed valid, or + // zero when the container still needs to be created. Generations are never + // reused, so a stale upload cannot invalidate a newer initialization. + containerGeneration uint64 + nextContainerGeneration uint64 + containerInitDone chan struct{} + + // Narrow hooks stand in for the individual *azblob.Client calls and the + // backoff wait rather than for the client as a whole: azblob.Client is a + // concrete struct, so the only alternatives are an HTTP-level fake, which + // would test the SDK instead of this file, or a wrapper interface far wider + // than the three operations whose ordering and retry behavior matter here. + // Every hook is nil in production, where the real client is used. + createContainerHook func(ctx context.Context) error + uploadBlobHook func(ctx context.Context, name string, body []byte, options *azblob.UploadBufferOptions) error + waitHook func(ctx context.Context, d time.Duration) error +} + +// Azure keeps a deleted container's name reserved until the delete finishes, +// rejecting recreation with ContainerBeingDeleted for anywhere up to about +// half a minute. Recreation therefore retries with a capped exponential +// backoff instead of failing the payload write. +const ( + containerBeingDeletedAttempts = 9 + containerBeingDeletedInitialBackoff = 250 * time.Millisecond + containerBeingDeletedMaxBackoff = 8 * time.Second +) + +var ( + _ api.LargePayloadStore = (*AzureBlobStore)(nil) + _ api.LargePayloadResolver = (*AzureBlobStore)(nil) + _ api.LargePayloadTokenStore = (*AzureBlobStore)(nil) + _ api.LargePayloadTokenValidator = (*AzureBlobStore)(nil) + _ api.LargePayloadDefaults = (*AzureBlobStore)(nil) + _ api.InclusiveLargePayloadThreshold = (*AzureBlobStore)(nil) + _ api.LargePayloadOptionsValidator = (*AzureBlobStore)(nil) +) + +// NewAzureBlobStore constructs a production Azure Blob Storage payload store. +func NewAzureBlobStore(options AzureBlobStoreOptions) (*AzureBlobStore, error) { + container := options.Container + if container == "" { + container = "durabletask-payloads" + } + if !validContainer(container) { + return nil, fmt.Errorf("%w: invalid Azure Blob container name", api.ErrLargePayloadReference) + } + if (options.ConnectionString == "") == (options.AccountURL == "") { + return nil, errors.New("exactly one of connection string or account URL is required") + } + if options.ConnectionString == "" && options.Credential == nil { + return nil, errors.New("azure blob account URL requires a token credential") + } + if options.ConnectionString != "" && options.Credential != nil { + return nil, errors.New("connection string and token credential cannot be combined") + } + compressionEnabled := true + if options.CompressionEnabled != nil { + compressionEnabled = *options.CompressionEnabled + } + maxPayloadBytes := api.DefaultAzureBlobPayloadMaxBytes + if options.MaxPayloadBytes != 0 { + maxPayloadBytes = options.MaxPayloadBytes + } + if maxPayloadBytes <= 0 { + return nil, errors.New("azure blob maximum payload size must be greater than zero") + } + clientOptions := &azblob.ClientOptions{ClientOptions: azcore.ClientOptions{Retry: policy.RetryOptions{ + MaxRetries: 8, + RetryDelay: 250 * time.Millisecond, + MaxRetryDelay: 10 * time.Second, + TryTimeout: 2 * time.Minute, + }}} + var ( + client *azblob.Client + err error + ) + if options.ConnectionString != "" { + client, err = azblob.NewClientFromConnectionString(options.ConnectionString, clientOptions) + } else { + accountURL, parseErr := url.Parse(options.AccountURL) + if parseErr != nil || !isSafeBlobURL(accountURL, options.AllowInsecureHTTP) { + return nil, fmt.Errorf("%w: invalid Azure Blob account URL", api.ErrLargePayloadReference) + } + client, err = azblob.NewClient(options.AccountURL, options.Credential, clientOptions) + } + if err != nil { + return nil, fmt.Errorf("create Azure Blob client: %w", err) + } + serviceURL, err := url.Parse(client.URL()) + if err != nil { + return nil, fmt.Errorf("%w: Azure Blob service URL is not allowed", api.ErrLargePayloadReference) + } + publicURL := *serviceURL + publicURL.RawQuery, publicURL.Fragment, publicURL.User = "", "", nil + publicURL.ForceQuery = false + if !isSafeBlobURL(&publicURL, options.AllowInsecureHTTP) { + return nil, fmt.Errorf("%w: Azure Blob service URL is not allowed", api.ErrLargePayloadReference) + } + containerURL := publicURL + containerURL.Path = strings.TrimRight(publicURL.Path, "/") + "/" + container + containerURL.RawPath = "" + allowedHosts := make(map[string]struct{}, len(options.AllowedHosts)) + for _, host := range options.AllowedHosts { + host = normalizedHost(host) + if host == "" { + return nil, errors.New("azure blob allowed host cannot be empty") + } + allowedHosts[host] = struct{}{} + } + return &AzureBlobStore{ + client: client, + credential: options.Credential, + clientOptions: clientOptions, + container: container, + containerURL: &containerURL, + allowHosts: allowedHosts, + allowInsecureHTTP: options.AllowInsecureHTTP, + compressionEnabled: compressionEnabled, + maxPayloadBytes: maxPayloadBytes, + }, nil +} + +// LargePayloadDefaults supplies Azure Blob's .NET-compatible defaults. +func (s *AzureBlobStore) LargePayloadDefaults() (thresholdBytes, maxPayloadBytes int) { + return api.DefaultAzureBlobPayloadThresholdBytes, s.maxPayloadBytes +} + +// UsesInclusiveLargePayloadThreshold matches the .NET Azure Blob extension. +func (*AzureBlobStore) UsesInclusiveLargePayloadThreshold() bool { return true } + +// ValidateLargePayloadOptions enforces the .NET Azure Blob extension's +// threshold limit and the store's configured payload cap. +func (s *AzureBlobStore) ValidateLargePayloadOptions(thresholdBytes, maxPayloadBytes int) error { + const maxThreshold = 1024 * 1024 + if thresholdBytes > maxThreshold { + return fmt.Errorf("azure blob payload threshold cannot exceed %d bytes", maxThreshold) + } + if maxPayloadBytes > s.maxPayloadBytes { + return fmt.Errorf( + "large payload maximum %d exceeds Azure Blob store maximum %d", + maxPayloadBytes, + s.maxPayloadBytes, + ) + } + return nil +} + +func (s *AzureBlobStore) Store(ctx context.Context, payload []byte) (string, error) { + if len(payload) > s.maxPayloadBytes { + return "", fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), s.maxPayloadBytes) + } + name, err := randomBlobName() + if err != nil { + return "", err + } + body := payload + headers := &blob.HTTPHeaders{} + if s.compressionEnabled { + body, err = gzipPayload(payload) + if err != nil { + return "", err + } + gzipEncoding := "gzip" + headers.BlobContentEncoding = &gzipEncoding + } + bodyDigest := md5.Sum(body) + headers.BlobContentMD5 = bodyDigest[:] + digest := sha256.Sum256(payload) + size := strconv.Itoa(len(payload)) + hash := hex.EncodeToString(digest[:]) + metadata := map[string]*string{"durabletask_size": &size, "durabletask_sha256": &hash} + // A concurrently deleted container is recreated once before giving up. + for attempt := 0; ; attempt++ { + generation, err := s.ensureContainer(ctx) + if err != nil { + return "", err + } + err = s.uploadBlob(ctx, name, body, &azblob.UploadBufferOptions{ + HTTPHeaders: headers, + Metadata: metadata, + }) + if err == nil { + return s.blobURL(name).String(), nil + } + if attempt > 0 || !bloberror.HasCode(err, bloberror.ContainerNotFound) { + return "", fmt.Errorf("upload Azure Blob payload: %w", err) + } + s.invalidateContainer(generation) + } +} + +func (s *AzureBlobStore) Resolve(ctx context.Context, location string) ([]byte, error) { + ref, err := s.parseToken(azureBlobTokenV2 + location) + if err != nil || !ref.same || ref.container != s.container { + return nil, fmt.Errorf("%w: invalid Azure Blob location", api.ErrLargePayloadReference) + } + return s.download(ctx, ref) +} + +// StoreToken stores a payload and returns the unwrapped .NET blob:v2 token. +func (s *AzureBlobStore) StoreToken(ctx context.Context, payload []byte) (string, error) { + location, err := s.Store(ctx, payload) + if err != nil { + return "", err + } + return azureBlobTokenV2 + location, nil +} + +// IsLargePayloadToken recognizes both .NET Azure Blob token versions. +func (*AzureBlobStore) IsLargePayloadToken(value string) bool { + return strings.HasPrefix(value, azureBlobTokenV1) || strings.HasPrefix(value, azureBlobTokenV2) +} + +// ValidateLargePayloadToken validates a recognized token without accessing the network. +func (s *AzureBlobStore) ValidateLargePayloadToken(token string) error { + _, err := s.parseToken(token) + return err +} + +// ResolveToken resolves a .NET blob:v1 or blob:v2 token. +func (s *AzureBlobStore) ResolveToken(ctx context.Context, token string) ([]byte, error) { + parsed, err := s.parseToken(token) + if err != nil { + return nil, err + } + return s.download(ctx, parsed) +} + +type azureBlobReference struct { + container string + name string + blobURL *url.URL + same bool +} + +func (s *AzureBlobStore) parseToken(token string) (azureBlobReference, error) { + if strings.HasPrefix(token, azureBlobTokenV1) { + parts := strings.SplitN(strings.TrimPrefix(token, azureBlobTokenV1), ":", 2) + if len(parts) != 2 || parts[0] != s.container || parts[1] == "" { + return azureBlobReference{}, fmt.Errorf("%w: invalid blob:v1 token", api.ErrLargePayloadReference) + } + return azureBlobReference{container: s.container, name: parts[1], same: true}, nil + } + if !strings.HasPrefix(token, azureBlobTokenV2) { + return azureBlobReference{}, fmt.Errorf("%w: invalid Azure Blob token prefix", api.ErrLargePayloadReference) + } + blobURL, err := url.Parse(strings.TrimPrefix(token, azureBlobTokenV2)) + if err != nil || !isSafeBlobURL(blobURL, s.allowInsecureHTTP) { + return azureBlobReference{}, fmt.Errorf("%w: invalid blob:v2 URL", api.ErrLargePayloadReference) + } + container, name, err := parseAzureBlobPath(blobURL) + if err != nil || !validContainer(container) || name == "" { + return azureBlobReference{}, fmt.Errorf("%w: invalid blob:v2 path", api.ErrLargePayloadReference) + } + same := sameURLContainer(blobURL, s.containerURL) + if !same { + if s.credential == nil { + return azureBlobReference{}, fmt.Errorf("%w: cross-account Azure Blob tokens require a token credential", api.ErrLargePayloadReference) + } + if _, allowed := s.allowHosts[normalizedHost(blobURL.Host)]; !allowed { + return azureBlobReference{}, fmt.Errorf("%w: Azure Blob token host is not allowed", api.ErrLargePayloadReference) + } + } + return azureBlobReference{container: container, name: name, blobURL: blobURL, same: same}, nil +} + +func (s *AzureBlobStore) download(ctx context.Context, ref azureBlobReference) ([]byte, error) { + var ( + response azblob.DownloadStreamResponse + err error + ) + if ref.same { + response, err = s.client.DownloadStream(ctx, s.container, ref.name, nil) + } else { + client, clientErr := azblob.NewClient(blobAccountURL(ref.blobURL).String(), s.credential, s.clientOptions) + if clientErr != nil { + return nil, fmt.Errorf("create cross-account Azure Blob client: %w", clientErr) + } + response, err = client.DownloadStream(ctx, ref.container, ref.name, nil) + } + if err != nil { + if bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + return nil, fmt.Errorf( + "%w: blob %q was not found in container %q", + api.ErrLargePayloadReference, + ref.name, + ref.container, + ) + } + return nil, fmt.Errorf("download Azure Blob payload: %w", err) + } + defer response.Body.Close() //nolint:errcheck // read-only cleanup + if response.ContentLength != nil && *response.ContentLength > int64(s.maxPayloadBytes)*2 { + return nil, fmt.Errorf("%w: compressed payload exceeds configured bound", api.ErrLargePayloadTooLarge) + } + reader := io.Reader(response.Body) + if response.ContentEncoding != nil && strings.EqualFold(*response.ContentEncoding, "gzip") { + gzipReader, gzipErr := gzip.NewReader(reader) + if gzipErr != nil { + return nil, fmt.Errorf("%w: invalid gzip payload", api.ErrLargePayloadIntegrity) + } + defer gzipReader.Close() //nolint:errcheck // read-only cleanup + reader = gzipReader + } + payload, err := readBounded(reader, s.maxPayloadBytes) + if err != nil { + return nil, err + } + if err := verifyMetadata(payload, response.Metadata); err != nil { + return nil, err + } + return payload, nil +} + +// ensureContainer creates the payload container once and caches the result. +// The returned generation identifies the initialization the caller relied on so +// a later ContainerNotFound failure can invalidate exactly that generation. +func (s *AzureBlobStore) ensureContainer(ctx context.Context) (uint64, error) { + for { + s.containerMu.Lock() + if s.containerGeneration != 0 { + generation := s.containerGeneration + s.containerMu.Unlock() + return generation, nil + } + if done := s.containerInitDone; done != nil { + s.containerMu.Unlock() + // Checking ctx before the select is load bearing: when done is + // already closed, select would otherwise pick pseudo-randomly + // between the two ready cases and an already cancelled caller + // could still be reported as successful. + if err := ctx.Err(); err != nil { + return 0, err + } + // Waiting is cancellable, but a waiter's cancellation must never + // cancel the in-flight initializer or the work it is creating for. + select { + case <-ctx.Done(): + return 0, ctx.Err() + case <-done: + } + continue + } + done := make(chan struct{}) + s.containerInitDone = done + s.containerMu.Unlock() + + err := s.createContainerWithRetry(ctx) + + s.containerMu.Lock() + s.containerInitDone = nil + if err != nil && !bloberror.HasCode(err, bloberror.ContainerAlreadyExists) { + // A failed or cancelled initializer publishes no generation, so the + // next caller retries initialization with its own context. + s.containerMu.Unlock() + close(done) + return 0, fmt.Errorf("create Azure Blob payload container: %w", err) + } + s.nextContainerGeneration++ + generation := s.nextContainerGeneration + s.containerGeneration = generation + s.containerMu.Unlock() + close(done) + return generation, nil + } +} + +func (s *AzureBlobStore) invalidateContainer(generation uint64) { + s.containerMu.Lock() + defer s.containerMu.Unlock() + if s.containerGeneration == generation { + s.containerGeneration = 0 + } +} + +// createContainerWithRetry creates the container, waiting out an in-progress +// delete of a container with the same name. Any other outcome, including +// cancellation and ContainerAlreadyExists, is returned to the caller +// unchanged on the first attempt. +func (s *AzureBlobStore) createContainerWithRetry(ctx context.Context) error { + backoff := containerBeingDeletedInitialBackoff + for attempt := 1; ; attempt++ { + err := s.createContainer(ctx) + if attempt == containerBeingDeletedAttempts || + !bloberror.HasCode(err, bloberror.ContainerBeingDeleted) { + return err + } + if waitErr := s.waitForRetry(ctx, backoff); waitErr != nil { + return waitErr + } + backoff = min(backoff*2, containerBeingDeletedMaxBackoff) + } +} + +func (s *AzureBlobStore) createContainer(ctx context.Context) error { + if s.createContainerHook != nil { + return s.createContainerHook(ctx) + } + _, err := s.client.CreateContainer(ctx, s.container, nil) + return err +} + +func (s *AzureBlobStore) uploadBlob( + ctx context.Context, + name string, + body []byte, + options *azblob.UploadBufferOptions, +) error { + if s.uploadBlobHook != nil { + return s.uploadBlobHook(ctx, name, body, options) + } + _, err := s.client.UploadBuffer(ctx, s.container, name, body, options) + return err +} + +// waitForRetry waits for d, reporting cancellation instead of blocking through +// it. +func (s *AzureBlobStore) waitForRetry(ctx context.Context, d time.Duration) error { + if s.waitHook != nil { + return s.waitHook(ctx, d) + } + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + +func (s *AzureBlobStore) blobURL(name string) *url.URL { + u := *s.containerURL + u.Path = strings.TrimRight(u.Path, "/") + "/" + name + return &u +} + +func parseAzureBlobPath(u *url.URL) (string, string, error) { + segments := strings.Split(strings.Trim(u.EscapedPath(), "/"), "/") + pathStyle := net.ParseIP(u.Hostname()) != nil || strings.EqualFold(u.Hostname(), "localhost") + if pathStyle { + if len(segments) < 3 || segments[0] == "" { + return "", "", errors.New("invalid path-style Azure Blob URL") + } + segments = segments[1:] + } + if len(segments) < 2 { + return "", "", errors.New("invalid Azure Blob URL path") + } + container, err := url.PathUnescape(segments[0]) + if err != nil { + return "", "", err + } + name, err := url.PathUnescape(strings.Join(segments[1:], "/")) + if err != nil || name == "" { + return "", "", errors.New("invalid Azure Blob name") + } + return container, name, nil +} + +func blobAccountURL(blobURL *url.URL) *url.URL { + u := *blobURL + segments := strings.Split(strings.Trim(blobURL.EscapedPath(), "/"), "/") + if net.ParseIP(blobURL.Hostname()) != nil || strings.EqualFold(blobURL.Hostname(), "localhost") { + u.Path = "/" + segments[0] + "/" + } else { + u.Path = "/" + } + u.RawPath, u.RawQuery, u.Fragment, u.User = "", "", "", nil + return &u +} + +func sameURLContainer(blobURL, containerURL *url.URL) bool { + if blobURL == nil || containerURL == nil { + return false + } + tokenContainer := *blobURL + container, _, err := parseAzureBlobPath(&tokenContainer) + if err != nil { + return false + } + expected := *containerURL + expected.Path = strings.TrimRight(expected.Path, "/") + tokenContainer.Path = strings.TrimRight(blobAccountURL(&tokenContainer).Path, "/") + "/" + container + tokenContainer.RawPath, tokenContainer.RawQuery, tokenContainer.Fragment, tokenContainer.User = "", "", "", nil + return strings.EqualFold(tokenContainer.Scheme, expected.Scheme) && + strings.EqualFold(tokenContainer.Hostname(), expected.Hostname()) && + effectivePort(&tokenContainer) == effectivePort(&expected) && + tokenContainer.Path == expected.Path +} + +func effectivePort(u *url.URL) string { + if port := u.Port(); port != "" { + return port + } + if strings.EqualFold(u.Scheme, "https") { + return "443" + } + if strings.EqualFold(u.Scheme, "http") { + return "80" + } + return "" +} + +func isSafeBlobURL(u *url.URL, allowHTTP bool) bool { + return u != nil && u.IsAbs() && u.Host != "" && u.User == nil && u.RawQuery == "" && u.Fragment == "" && + (u.Scheme == "https" || (allowHTTP && u.Scheme == "http")) +} + +func validContainer(container string) bool { + if len(container) < 3 || len(container) > 63 || container[0] == '-' || container[len(container)-1] == '-' { + return false + } + for _, r := range container { + if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' { + return false + } + } + return !strings.Contains(container, "--") +} + +func normalizedHost(host string) string { + host = strings.TrimSpace(strings.ToLower(host)) + if parsedHost, _, err := net.SplitHostPort(host); err == nil { + return parsedHost + } + return strings.Trim(host, "[]") +} + +func randomBlobName() (string, error) { + var bytes [16]byte + if _, err := rand.Read(bytes[:]); err != nil { + return "", fmt.Errorf("generate Azure Blob payload name: %w", err) + } + return hex.EncodeToString(bytes[:]), nil +} + +func gzipPayload(payload []byte) ([]byte, error) { + var buffer bytes.Buffer + writer := gzip.NewWriter(&buffer) + if _, err := writer.Write(payload); err != nil { + return nil, err + } + if err := writer.Close(); err != nil { + return nil, err + } + return buffer.Bytes(), nil +} + +func readBounded(reader io.Reader, max int) ([]byte, error) { + payload, err := io.ReadAll(io.LimitReader(reader, int64(max)+1)) + if err != nil { + return nil, fmt.Errorf("read Azure Blob payload: %w", err) + } + if len(payload) > max { + return nil, fmt.Errorf("%w: payload exceeds %d bytes", api.ErrLargePayloadTooLarge, max) + } + return payload, nil +} + +func verifyMetadata(payload []byte, metadata map[string]*string) error { + size, hasSize := metadataValue(metadata, "durabletask_size") + hash, hasHash := metadataValue(metadata, "durabletask_sha256") + if !hasSize && !hasHash { + return nil + } + if !hasSize || !hasHash || size == nil || hash == nil { + return fmt.Errorf("%w: incomplete Azure Blob payload metadata", api.ErrLargePayloadIntegrity) + } + expectedSize, err := strconv.Atoi(*size) + if err != nil || expectedSize != len(payload) { + return fmt.Errorf("%w: Azure Blob payload size mismatch", api.ErrLargePayloadIntegrity) + } + expectedHash, err := hex.DecodeString(*hash) + digest := sha256.Sum256(payload) + if err != nil || len(expectedHash) != len(digest) || subtle.ConstantTimeCompare(digest[:], expectedHash) != 1 { + return fmt.Errorf("%w: Azure Blob payload hash mismatch", api.ErrLargePayloadIntegrity) + } + return nil +} + +func metadataValue(metadata map[string]*string, name string) (*string, bool) { + for key, value := range metadata { + if strings.EqualFold(key, name) { + return value, true + } + } + return nil, false +} diff --git a/payload/azure_blob_azurite_test.go b/payload/azure_blob_azurite_test.go new file mode 100644 index 00000000..e590462a --- /dev/null +++ b/payload/azure_blob_azurite_test.go @@ -0,0 +1,141 @@ +package payload + +import ( + "context" + "crypto/rand" + "encoding/hex" + "fmt" + "os" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// newAzuriteStore builds a store against a throwaway container so Azurite runs +// exercise the real container-initialization and upload paths. +func newAzuriteStore(t *testing.T, compression *bool) *AzureBlobStore { + t.Helper() + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run against Azurite") + } + var suffix [8]byte + _, err := rand.Read(suffix[:]) + require.NoError(t, err) + container := "dtgo" + hex.EncodeToString(suffix[:]) + store, err := NewAzureBlobStore(AzureBlobStoreOptions{ + ConnectionString: connectionString, + Container: container, + AllowInsecureHTTP: true, + CompressionEnabled: compression, + }) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = store.client.DeleteContainer(context.Background(), container, nil) + }) + return store +} + +func TestAzureBlobStoreAzuriteCompressionModes(t *testing.T) { + enabled, disabled := true, false + payload := []byte(strings.Repeat("Azure Blob payload ", 512)) + tests := []struct { + name string + compression *bool + }{ + {name: "gzip enabled", compression: &enabled}, + {name: "gzip disabled", compression: &disabled}, + {name: "default"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := newAzuriteStore(t, test.compression) + token, err := store.StoreToken(context.Background(), payload) + require.NoError(t, err) + require.True(t, strings.HasPrefix(token, azureBlobTokenV2)) + + resolved, err := store.ResolveToken(context.Background(), token) + require.NoError(t, err) + require.Equal(t, payload, resolved) + }) + } +} + +// TestAzureBlobStoreAzuriteResolvesLegacyV1Token pins .NET blob:v1 read +// interoperability against a blob written with the current blob:v2 writer. +func TestAzureBlobStoreAzuriteResolvesLegacyV1Token(t *testing.T) { + store := newAzuriteStore(t, nil) + token, err := store.StoreToken(context.Background(), []byte("legacy token payload")) + require.NoError(t, err) + + name := token[strings.LastIndex(token, "/")+1:] + legacy := azureBlobTokenV1 + store.container + ":" + name + resolved, err := store.ResolveToken(context.Background(), legacy) + require.NoError(t, err) + require.Equal(t, []byte("legacy token payload"), resolved) +} + +// TestAzureBlobStoreAzuriteConcurrentUploadsShareInitialization exercises the +// single-flight container gate against real storage. +func TestAzureBlobStoreAzuriteConcurrentUploadsShareInitialization(t *testing.T) { + store := newAzuriteStore(t, nil) + const uploaders = 16 + // Assertions run on the test goroutine: require calls FailNow, which is + // undefined behavior from a goroutine other than the one running the test, + // so mismatches are reported as errors on the channel instead. + results := make(chan error, uploaders) + for i := 0; i < uploaders; i++ { + go func(i int) { + want := strings.Repeat("p", i+1) + token, err := store.StoreToken(context.Background(), []byte(want)) + if err != nil { + results <- err + return + } + resolved, err := store.ResolveToken(context.Background(), token) + if err == nil && string(resolved) != want { + err = fmt.Errorf("uploader %d resolved %q, want %q", i, resolved, want) + } + results <- err + }(i) + } + for i := 0; i < uploaders; i++ { + require.NoError(t, <-results) + } +} + +// TestAzureBlobStoreAzuriteRecreatesDeletedContainer verifies that an +// out-of-band container deletion is recovered from within a single upload. +func TestAzureBlobStoreAzuriteRecreatesDeletedContainer(t *testing.T) { + store := newAzuriteStore(t, nil) + _, err := store.StoreToken(context.Background(), []byte("first")) + require.NoError(t, err) + + _, err = store.client.DeleteContainer(context.Background(), store.container, nil) + require.NoError(t, err) + + token, err := store.StoreToken(context.Background(), []byte("second")) + require.NoError(t, err) + resolved, err := store.ResolveToken(context.Background(), token) + require.NoError(t, err) + require.Equal(t, []byte("second"), resolved) +} + +// TestAzureBlobStoreAzuriteMissingBlobIsAReferenceError pins the error surface +// for a token whose blob no longer exists. +func TestAzureBlobStoreAzuriteMissingBlobIsAReferenceError(t *testing.T) { + store := newAzuriteStore(t, nil) + token, err := store.StoreToken(context.Background(), []byte("payload")) + require.NoError(t, err) + _, err = store.client.DeleteBlob( + context.Background(), + store.container, + token[strings.LastIndex(token, "/")+1:], + nil, + ) + require.NoError(t, err) + + _, err = store.ResolveToken(context.Background(), token) + require.Error(t, err) +} diff --git a/payload/azure_blob_container_test.go b/payload/azure_blob_container_test.go new file mode 100644 index 00000000..829cf194 --- /dev/null +++ b/payload/azure_blob_container_test.go @@ -0,0 +1,403 @@ +package payload + +import ( + "context" + "errors" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/stretchr/testify/require" +) + +func blobCodeError(status int, code bloberror.Code) error { + return &azcore.ResponseError{StatusCode: status, ErrorCode: string(code)} +} + +// blobBackend replaces the Azure Blob client with scripted, deterministic +// container and upload behavior. Backoff waits are recorded rather than slept +// so retry tests stay fast and deterministic. +type blobBackend struct { + createResult func(ctx context.Context, call int) error + uploadResult func(ctx context.Context, call int) error + + createCalls atomic.Int64 + uploadCalls atomic.Int64 + + sleepMu sync.Mutex + sleeps []time.Duration +} + +func (b *blobBackend) createContainer(ctx context.Context) error { + call := int(b.createCalls.Add(1)) + if b.createResult == nil { + return nil + } + return b.createResult(ctx, call) +} + +func (b *blobBackend) uploadBlob( + ctx context.Context, + _ string, + _ []byte, + _ *azblob.UploadBufferOptions, +) error { + call := int(b.uploadCalls.Add(1)) + if b.uploadResult == nil { + return nil + } + return b.uploadResult(ctx, call) +} + +func (b *blobBackend) waitForRetry(ctx context.Context, delay time.Duration) error { + if err := ctx.Err(); err != nil { + return err + } + b.sleepMu.Lock() + b.sleeps = append(b.sleeps, delay) + b.sleepMu.Unlock() + return nil +} + +func (b *blobBackend) recordedSleeps() []time.Duration { + b.sleepMu.Lock() + defer b.sleepMu.Unlock() + return append([]time.Duration(nil), b.sleeps...) +} + +func newScriptedAzureBlobStore(t *testing.T, backend *blobBackend) *AzureBlobStore { + t.Helper() + store := newTestAzureBlobStore(t) + store.createContainerHook = backend.createContainer + store.uploadBlobHook = backend.uploadBlob + store.waitHook = backend.waitForRetry + return store +} + +// TestAzureBlobStoreScriptedContainerLifecycle covers the container +// initialization outcomes that can be expressed as a fixed script of create and +// upload results. +func TestAzureBlobStoreScriptedContainerLifecycle(t *testing.T) { + transient := errors.New("service unavailable") + unrelated := blobCodeError(http.StatusInternalServerError, bloberror.InternalError) + missing := blobCodeError(http.StatusNotFound, bloberror.ContainerNotFound) + beingDeleted := blobCodeError(http.StatusConflict, bloberror.ContainerBeingDeleted) + + tests := []struct { + name string + // createResults and uploadResults are indexed by call number, and the + // final entry repeats for any further calls. + createResults []error + uploadResults []error + // uploads describes the expected error of each sequential upload. + uploads []error + expectedCreateCalls int64 + expectedUploadCalls int64 + // expectedBackoffs are the waits between container creation attempts. + expectedBackoffs []time.Duration + }{ + { + name: "cached initialization is reused", + createResults: []error{nil}, + uploadResults: []error{nil}, + uploads: []error{nil, nil, nil}, + expectedCreateCalls: 1, + expectedUploadCalls: 3, + }, + { + name: "failed initialization is retried on the next call", + createResults: []error{transient, nil}, + uploadResults: []error{nil}, + uploads: []error{transient, nil}, + expectedCreateCalls: 2, + expectedUploadCalls: 1, + }, + { + name: "already existing container completes initialization", + createResults: []error{blobCodeError(http.StatusConflict, bloberror.ContainerAlreadyExists)}, + uploadResults: []error{nil}, + uploads: []error{nil, nil}, + expectedCreateCalls: 1, + expectedUploadCalls: 2, + }, + { + name: "deleted container is recreated within the same upload", + createResults: []error{nil}, + uploadResults: []error{missing, nil}, + uploads: []error{nil}, + expectedCreateCalls: 2, + expectedUploadCalls: 2, + }, + { + name: "second missing container is not retried again", + createResults: []error{nil}, + uploadResults: []error{missing}, + uploads: []error{missing}, + expectedCreateCalls: 2, + expectedUploadCalls: 2, + }, + { + name: "unrelated storage error propagates without invalidating the cache", + createResults: []error{nil}, + uploadResults: []error{unrelated, nil}, + uploads: []error{unrelated, nil}, + expectedCreateCalls: 1, + expectedUploadCalls: 2, + }, + { + name: "container being deleted is waited out with capped backoff", + createResults: []error{beingDeleted, beingDeleted, beingDeleted, nil}, + uploadResults: []error{nil}, + uploads: []error{nil, nil}, + expectedCreateCalls: 4, + expectedUploadCalls: 2, + expectedBackoffs: []time.Duration{250 * time.Millisecond, 500 * time.Millisecond, time.Second}, + }, + { + name: "container being deleted gives up after the attempt bound", + createResults: []error{beingDeleted}, + uploadResults: []error{nil}, + uploads: []error{beingDeleted}, + expectedCreateCalls: containerBeingDeletedAttempts, + expectedUploadCalls: 0, + expectedBackoffs: []time.Duration{ + 250 * time.Millisecond, + 500 * time.Millisecond, + time.Second, + 2 * time.Second, + 4 * time.Second, + containerBeingDeletedMaxBackoff, + containerBeingDeletedMaxBackoff, + containerBeingDeletedMaxBackoff, + }, + }, + { + name: "a container being deleted that resolves as already existing succeeds", + createResults: []error{beingDeleted, blobCodeError(http.StatusConflict, bloberror.ContainerAlreadyExists)}, + uploadResults: []error{nil}, + uploads: []error{nil}, + expectedCreateCalls: 2, + expectedUploadCalls: 1, + expectedBackoffs: []time.Duration{250 * time.Millisecond}, + }, + } + + scripted := func(results []error) func(context.Context, int) error { + return func(_ context.Context, call int) error { + if call > len(results) { + call = len(results) + } + return results[call-1] + } + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + backend := &blobBackend{ + createResult: scripted(test.createResults), + uploadResult: scripted(test.uploadResults), + } + store := newScriptedAzureBlobStore(t, backend) + for i, expected := range test.uploads { + token, err := store.StoreToken(context.Background(), []byte("payload")) + if expected == nil { + require.NoErrorf(t, err, "upload %d", i) + require.Regexpf(t, `^blob:v2:.*/[0-9a-f]{32}$`, token, "upload %d", i) + continue + } + require.ErrorIsf(t, err, expected, "upload %d", i) + } + require.Equal(t, test.expectedCreateCalls, backend.createCalls.Load(), "container creations") + require.Equal(t, test.expectedUploadCalls, backend.uploadCalls.Load(), "blob uploads") + require.Equal(t, test.expectedBackoffs, backend.recordedSleeps(), "recreation backoff") + }) + } +} + +// TestAzureBlobStoreContainerBeingDeletedStopsOnCancellation verifies that a +// caller cancelled while the recreation backoff is waiting reports cancellation +// instead of exhausting the attempt bound. +func TestAzureBlobStoreContainerBeingDeletedStopsOnCancellation(t *testing.T) { + beingDeleted := blobCodeError(http.StatusConflict, bloberror.ContainerBeingDeleted) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + backend := &blobBackend{createResult: func(context.Context, int) error { return beingDeleted }} + store := newScriptedAzureBlobStore(t, backend) + store.waitHook = func(ctx context.Context, _ time.Duration) error { + cancel() + return ctx.Err() + } + + _, err := store.StoreToken(ctx, []byte("payload")) + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, int64(1), backend.createCalls.Load(), "cancellation must stop the retry loop") + require.Zero(t, backend.uploadCalls.Load()) +} + +// TestAzureBlobStoreConcurrentUploadsInitializeContainerOnce keeps container +// creation in flight until every uploader has reached the store so the +// single-flight gate is genuinely contended. +func TestAzureBlobStoreConcurrentUploadsInitializeContainerOnce(t *testing.T) { + const uploaders = 16 + var ready sync.WaitGroup + ready.Add(uploaders) + backend := &blobBackend{createResult: func(context.Context, int) error { + ready.Wait() + return nil + }} + store := newScriptedAzureBlobStore(t, backend) + + errs := make(chan error, uploaders) + for i := 0; i < uploaders; i++ { + go func() { + ready.Done() + _, err := store.StoreToken(context.Background(), []byte("payload")) + errs <- err + }() + } + for i := 0; i < uploaders; i++ { + require.NoError(t, <-errs) + } + require.Equal(t, int64(1), backend.createCalls.Load()) + require.Equal(t, int64(uploaders), backend.uploadCalls.Load()) +} + +// TestAzureBlobStoreCancelledInitializerReleasesWaiters verifies that a +// cancelled initializer publishes no generation, so a waiting caller retries +// initialization with its own context. +func TestAzureBlobStoreCancelledInitializerReleasesWaiters(t *testing.T) { + createStarted := make(chan struct{}) + waiterReady := make(chan struct{}) + backend := &blobBackend{createResult: func(ctx context.Context, call int) error { + if call > 1 { + return nil + } + close(createStarted) + <-waiterReady + <-ctx.Done() + return ctx.Err() + }} + store := newScriptedAzureBlobStore(t, backend) + + cancellable, cancel := context.WithCancel(context.Background()) + defer cancel() + cancelled := make(chan error, 1) + go func() { + _, err := store.StoreToken(cancellable, []byte("payload")) + cancelled <- err + }() + <-createStarted + + waiting := make(chan error, 1) + go func() { + close(waiterReady) + _, err := store.StoreToken(context.Background(), []byte("payload")) + waiting <- err + }() + + cancel() + require.ErrorIs(t, <-cancelled, context.Canceled) + require.NoError(t, <-waiting) + require.Equal(t, int64(2), backend.createCalls.Load()) + require.Equal(t, int64(1), backend.uploadCalls.Load(), "the cancelled caller must not upload") +} + +// TestAzureBlobStoreCancelledWaiterDoesNotCancelInitializer verifies that one +// caller abandoning the wait leaves the in-flight initialization untouched. +func TestAzureBlobStoreCancelledWaiterDoesNotCancelInitializer(t *testing.T) { + createStarted := make(chan struct{}) + releaseCreate := make(chan struct{}) + backend := &blobBackend{createResult: func(ctx context.Context, _ int) error { + close(createStarted) + <-releaseCreate + return ctx.Err() + }} + store := newScriptedAzureBlobStore(t, backend) + + initializing := make(chan error, 1) + go func() { + _, err := store.StoreToken(context.Background(), []byte("payload")) + initializing <- err + }() + <-createStarted + + cancellable, cancel := context.WithCancel(context.Background()) + defer cancel() + waiting := make(chan error, 1) + waiterReady := make(chan struct{}) + go func() { + close(waiterReady) + _, err := store.StoreToken(cancellable, []byte("payload")) + waiting <- err + }() + <-waiterReady + cancel() + + require.ErrorIs(t, <-waiting, context.Canceled) + close(releaseCreate) + require.NoError(t, <-initializing) + require.Equal(t, int64(1), backend.createCalls.Load()) + require.Equal(t, int64(1), backend.uploadCalls.Load()) +} + +// TestAzureBlobStoreStaleContainerNotFoundKeepsNewerInitialization pins the +// generation identity contract: a slow upload that only discovers the container +// is gone after a newer initialization was published must not invalidate it. +func TestAzureBlobStoreStaleContainerNotFoundKeepsNewerInitialization(t *testing.T) { + type uploadTag struct{} + missing := blobCodeError(http.StatusNotFound, bloberror.ContainerNotFound) + slowStarted := make(chan struct{}) + releaseSlow := make(chan struct{}) + + var slowUpload sync.Once + backend := &blobBackend{uploadResult: func(ctx context.Context, _ int) error { + switch tag, _ := ctx.Value(uploadTag{}).(string); tag { + case "slow": + // Only the first slow attempt parks; the retry is already ordered + // behind it on the same goroutine. + slowUpload.Do(func() { + close(slowStarted) + <-releaseSlow + }) + return missing + case "missing": + return missing + default: + return nil + } + }} + store := newScriptedAzureBlobStore(t, backend) + + tagged := func(tag string) context.Context { + return context.WithValue(context.Background(), uploadTag{}, tag) + } + + slow := make(chan error, 1) + go func() { + _, err := store.StoreToken(tagged("slow"), []byte("payload")) + slow <- err + }() + <-slowStarted + + // Discovers the deletion first, invalidating the generation the slow upload + // is still holding, then fails again on its single permitted retry. + _, err := store.StoreToken(tagged("missing"), []byte("payload")) + require.ErrorIs(t, err, missing) + require.Equal(t, int64(2), backend.createCalls.Load()) + + _, err = store.StoreToken(tagged("recreated"), []byte("payload")) + require.NoError(t, err) + + close(releaseSlow) + require.ErrorIs(t, <-slow, missing) + + _, err = store.StoreToken(tagged("final"), []byte("payload")) + require.NoError(t, err) + require.Equal(t, int64(2), backend.createCalls.Load(), + "the stale failure must not invalidate the newer initialization") +} diff --git a/payload/azure_blob_test.go b/payload/azure_blob_test.go new file mode 100644 index 00000000..0671c77b --- /dev/null +++ b/payload/azure_blob_test.go @@ -0,0 +1,155 @@ +package payload + +import ( + "context" + "encoding/json" + "os" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/require" +) + +func newTestAzureBlobStore(t *testing.T) *AzureBlobStore { + t.Helper() + store, err := NewAzureBlobStore(AzureBlobStoreOptions{ + AccountURL: "http://127.0.0.1:10000/devstoreaccount1", + Credential: staticCredential{}, + Container: "mycontainer", + AllowInsecureHTTP: true, + AllowedHosts: []string{"myaccount.blob.core.windows.net"}, + }) + require.NoError(t, err) + return store +} + +func TestAzureBlobStoreGoldenTokenVectors(t *testing.T) { + data, err := os.ReadFile("testdata/blob_tokens.json") + require.NoError(t, err) + var vectors []struct { + Token string `json:"token"` + Container string `json:"container"` + Name string `json:"name"` + } + require.NoError(t, json.Unmarshal(data, &vectors)) + + store := newTestAzureBlobStore(t) + for _, vector := range vectors { + t.Run(vector.Token[:7], func(t *testing.T) { + ref, err := store.parseToken(vector.Token) + require.NoError(t, err) + require.Equal(t, vector.Container, ref.container) + require.Equal(t, vector.Name, ref.name) + }) + } +} + +func TestAzureBlobStoreRejectsUntrustedTokens(t *testing.T) { + store := newTestAzureBlobStore(t) + for _, token := range []string{ + "blob:v1:othercontainer:0f8fad5bd9cb469fa16570867728950e", + "blob:v2:https://user@myaccount.blob.core.windows.net/mycontainer/0f8fad5bd9cb469fa16570867728950e", + "blob:v2:https://myaccount.blob.core.windows.net/mycontainer/0f8fad5bd9cb469fa16570867728950e?sig=x", + "blob:v2:https://myaccount.blob.core.windows.net/mycontainer/0f8fad5bd9cb469fa16570867728950e#fragment", + "blob:v2:https://untrusted.blob.core.windows.net/mycontainer/0f8fad5bd9cb469fa16570867728950e", + } { + require.ErrorIs(t, store.ValidateLargePayloadToken(token), api.ErrLargePayloadReference) + } +} + +func TestAzureBlobStoreAllowsExplicitCrossAccountHost(t *testing.T) { + store, err := NewAzureBlobStore(AzureBlobStoreOptions{ + AccountURL: "https://account.blob.core.windows.net", + Credential: staticCredential{}, + Container: "mycontainer", + AllowedHosts: []string{"other.blob.core.windows.net"}, + }) + require.NoError(t, err) + require.NoError(t, store.ValidateLargePayloadToken( + "blob:v2:https://other.blob.core.windows.net/othercontainer/0f8fad5bd9cb469fa16570867728950e", + )) + require.ErrorIs(t, store.ValidateLargePayloadToken( + "blob:v2:https://untrusted.blob.core.windows.net/othercontainer/0f8fad5bd9cb469fa16570867728950e", + ), api.ErrLargePayloadReference) +} + +func TestAzureBlobStoreAcceptsConfiguredHostAndSASConnectionString(t *testing.T) { + store, err := NewAzureBlobStore(AzureBlobStoreOptions{ + AccountURL: "https://account.blob.core.windows.net", + Credential: staticCredential{}, + Container: "mycontainer", + }) + require.NoError(t, err) + require.NoError(t, store.ValidateLargePayloadToken( + "blob:v2:https://account.blob.core.windows.net/mycontainer/0f8fad5bd9cb469fa16570867728950e", + )) + require.NoError(t, store.ValidateLargePayloadToken( + "blob:v2:https://account.blob.core.windows.net:443/mycontainer/0f8fad5bd9cb469fa16570867728950e", + )) + + _, err = NewAzureBlobStore(AzureBlobStoreOptions{ + ConnectionString: "BlobEndpoint=https://account.blob.core.windows.net;SharedAccessSignature=sv=2024-01-01&sig=example", + Container: "mycontainer", + }) + require.NoError(t, err) +} + +func TestAzureBlobStoreDefaults(t *testing.T) { + store := newTestAzureBlobStore(t) + threshold, max := store.LargePayloadDefaults() + require.Equal(t, api.DefaultAzureBlobPayloadThresholdBytes, threshold) + require.Equal(t, api.DefaultAzureBlobPayloadMaxBytes, max) + require.True(t, store.UsesInclusiveLargePayloadThreshold()) + require.True(t, store.compressionEnabled) +} + +func TestAzureBlobStoreIntegrityMetadata(t *testing.T) { + payload := []byte("payload") + size, hash := "7", "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5" + require.NoError(t, verifyMetadata(payload, map[string]*string{ + "Durabletask_Size": &size, "DURABLETASK_SHA256": &hash, + })) + require.NoError(t, verifyMetadata(payload, nil), ".NET blobs have no Go metadata") + hash = "not-a-hash" + require.ErrorIs(t, verifyMetadata(payload, map[string]*string{ + "durabletask_size": &size, "durabletask_sha256": &hash, + }), api.ErrLargePayloadIntegrity) +} + +func TestAzureBlobStoreAzuriteRoundTrip(t *testing.T) { + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run against Azurite") + } + store, err := NewAzureBlobStore(AzureBlobStoreOptions{ + ConnectionString: connectionString, Container: "durabletaskgotest", AllowInsecureHTTP: true, + }) + require.NoError(t, err) + token, err := store.StoreToken(context.Background(), []byte("Azure Blob payload")) + require.NoError(t, err) + require.Regexp(t, `^blob:v2:.*/[0-9a-f]{32}$`, token) + resolved, err := store.ResolveToken(context.Background(), token) + require.NoError(t, err) + require.Equal(t, []byte("Azure Blob payload"), resolved) +} + +func TestAzureBlobStoreAcceptsDotNetGoldenBlobNames(t *testing.T) { + store := newTestAzureBlobStore(t) + for _, token := range []string{ + "blob:v2:https://myaccount.blob.core.windows.net/mycontainer/abc123def456", + "blob:v2:http://127.0.0.1:10000/devstoreaccount1/mycontainer/abc123def456", + "blob:v1:mycontainer:abc123def456", + "blob:v2:https://myaccount.blob.core.windows.net/mycontainer/nested/abc123def456", + } { + require.NoError(t, store.ValidateLargePayloadToken(token)) + } +} + +type staticCredential struct{} + +func (staticCredential) GetToken(context.Context, policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "test", ExpiresOn: time.Now().Add(time.Hour)}, nil +} diff --git a/payload/doc.go b/payload/doc.go new file mode 100644 index 00000000..d261ca4c --- /dev/null +++ b/payload/doc.go @@ -0,0 +1,7 @@ +// Package payload provides large-payload stores for Durable Task Scheduler. +// +// AzureBlobStore is the production interoperable implementation for application +// payloads. The separate exporthistory.AzureBlobHistoryStore writes exported +// history objects. MemoryStore is process-local and non-durable, while FileStore +// requires every client and worker to share the same durable filesystem. +package payload diff --git a/payload/file.go b/payload/file.go new file mode 100644 index 00000000..75fed472 --- /dev/null +++ b/payload/file.go @@ -0,0 +1,141 @@ +package payload + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "github.com/microsoft/durabletask-go/api" +) + +// FileStore stores large payloads beneath one filesystem root. +// +// Every client and worker that may resolve its references must mount the same +// durable filesystem at the same path. Container-local and other ephemeral +// filesystems lose references on restart and are not suitable for production. +type FileStore struct { + root string + maxBytes int64 +} + +// NewFileStore creates a filesystem-backed store with an optional payload limit. +func NewFileStore(root string, maxBytes ...int64) (*FileStore, error) { + if strings.TrimSpace(root) == "" { + return nil, errors.New("file payload store root is required") + } + absoluteRoot, err := filepath.Abs(root) + if err != nil { + return nil, fmt.Errorf("failed to resolve payload store root: %w", err) + } + if err := os.MkdirAll(absoluteRoot, 0o700); err != nil { + return nil, fmt.Errorf("failed to create payload store root: %w", err) + } + limit := int64(api.DefaultLargePayloadMaxBytes) + if len(maxBytes) > 0 && maxBytes[0] > 0 { + limit = maxBytes[0] + } + return &FileStore{root: absoluteRoot, maxBytes: limit}, nil +} + +// Store writes one payload beneath the configured root. +func (s *FileStore) Store(ctx context.Context, payload []byte) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if int64(len(payload)) > s.maxBytes { + return "", fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), s.maxBytes) + } + digest := sha256.Sum256(payload) + hash := hex.EncodeToString(digest[:]) + location := "file://sha256/" + hash + path := filepath.Join(s.root, hash+".payload") + if _, err := os.Stat(path); err == nil { + return location, nil + } else if !errors.Is(err, os.ErrNotExist) { + return "", fmt.Errorf("failed to inspect payload file: %w", err) + } + + temp, err := os.CreateTemp(s.root, ".payload-*") + if err != nil { + return "", fmt.Errorf("failed to create payload file: %w", err) + } + tempPath := temp.Name() + defer os.Remove(tempPath) //nolint:errcheck // best-effort cleanup after rename or failure + if err := temp.Chmod(0o600); err != nil { + _ = temp.Close() + return "", fmt.Errorf("failed to secure payload file: %w", err) + } + if _, err := temp.Write(payload); err != nil { + _ = temp.Close() + return "", fmt.Errorf("failed to write payload file: %w", err) + } + if err := temp.Close(); err != nil { + return "", fmt.Errorf("failed to close payload file: %w", err) + } + if err := ctx.Err(); err != nil { + return "", err + } + if err := os.Rename(tempPath, path); err != nil { + if _, statErr := os.Stat(path); statErr == nil { + return location, nil + } + return "", fmt.Errorf("failed to publish payload file: %w", err) + } + return location, nil +} + +// Resolve reads a payload location created by this store. +func (s *FileStore) Resolve(ctx context.Context, location string) ([]byte, error) { + hash, err := parseFileLocation(location) + if err != nil { + return nil, err + } + file, err := os.Open(filepath.Join(s.root, hash+".payload")) + if err != nil { + return nil, fmt.Errorf("failed to open payload file: %w", err) + } + defer file.Close() //nolint:errcheck // read-only file cleanup + info, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("failed to inspect payload file: %w", err) + } + if info.Size() > s.maxBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, info.Size(), s.maxBytes) + } + payload := make([]byte, info.Size()) + if _, err := io.ReadFull(&contextReader{ctx: ctx, reader: file}, payload); err != nil { + return nil, fmt.Errorf("failed to read payload file: %w", err) + } + return payload, nil +} + +func parseFileLocation(location string) (string, error) { + const prefix = "file://sha256/" + if !strings.HasPrefix(location, prefix) { + return "", errors.New("invalid file payload location") + } + hash := strings.TrimPrefix(location, prefix) + decoded, err := hex.DecodeString(hash) + if err != nil || len(decoded) != sha256.Size { + return "", errors.New("invalid file payload hash") + } + return hash, nil +} + +type contextReader struct { + ctx context.Context + reader io.Reader +} + +func (r *contextReader) Read(buffer []byte) (int, error) { + if err := r.ctx.Err(); err != nil { + return 0, err + } + return r.reader.Read(buffer) +} diff --git a/payload/memory.go b/payload/memory.go new file mode 100644 index 00000000..3d0cb199 --- /dev/null +++ b/payload/memory.go @@ -0,0 +1,72 @@ +package payload + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "strings" + "sync" + + "github.com/microsoft/durabletask-go/api" +) + +// MemoryStore is a process-local, non-durable large-payload store. +// +// References become unreadable when the process restarts and cannot be resolved +// by clients or workers in another process. Use it only for tests and +// single-process experiments; use [AzureBlobStore] for production DTS workloads. +type MemoryStore struct { + mu sync.RWMutex + payloads map[string][]byte + maxBytes int +} + +// NewMemoryStore creates a process-local store with an optional payload limit. +func NewMemoryStore(maxBytes ...int) *MemoryStore { + limit := api.DefaultLargePayloadMaxBytes + if len(maxBytes) > 0 && maxBytes[0] > 0 { + limit = maxBytes[0] + } + return &MemoryStore{ + payloads: make(map[string][]byte), + maxBytes: limit, + } +} + +// Store saves one payload in process memory. +func (s *MemoryStore) Store(ctx context.Context, payload []byte) (string, error) { + if err := ctx.Err(); err != nil { + return "", err + } + if len(payload) > s.maxBytes { + return "", fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), s.maxBytes) + } + digest := sha256.Sum256(payload) + location := "memory://sha256/" + hex.EncodeToString(digest[:]) + s.mu.Lock() + s.payloads[location] = append([]byte(nil), payload...) + s.mu.Unlock() + return location, nil +} + +// Resolve reads a payload previously stored by this process. +func (s *MemoryStore) Resolve(ctx context.Context, location string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + if !strings.HasPrefix(location, "memory://sha256/") { + return nil, errors.New("invalid memory payload location") + } + s.mu.RLock() + payload, ok := s.payloads[location] + s.mu.RUnlock() + if !ok { + return nil, errors.New("large payload was not found") + } + if len(payload) > s.maxBytes { + return nil, fmt.Errorf("%w: %d bytes exceeds %d", api.ErrLargePayloadTooLarge, len(payload), s.maxBytes) + } + return append([]byte(nil), payload...), nil +} diff --git a/payload/store_test.go b/payload/store_test.go new file mode 100644 index 00000000..1de691c6 --- /dev/null +++ b/payload/store_test.go @@ -0,0 +1,42 @@ +package payload + +import ( + "context" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/require" +) + +func TestMemoryStoreRoundTripCopiesPayload(t *testing.T) { + store := NewMemoryStore() + original := []byte("payload") + location, err := store.Store(context.Background(), original) + require.NoError(t, err) + original[0] = 'X' + + resolved, err := store.Resolve(context.Background(), location) + require.NoError(t, err) + require.Equal(t, []byte("payload"), resolved) + resolved[0] = 'Y' + + again, err := store.Resolve(context.Background(), location) + require.NoError(t, err) + require.Equal(t, []byte("payload"), again) +} + +func TestFileStoreRoundTripAndLimit(t *testing.T) { + store, err := NewFileStore(t.TempDir(), 8) + require.NoError(t, err) + location, err := store.Store(context.Background(), []byte("payload")) + require.NoError(t, err) + + resolved, err := store.Resolve(context.Background(), location) + require.NoError(t, err) + require.Equal(t, []byte("payload"), resolved) + + _, err = store.Store(context.Background(), []byte("too-large")) + require.ErrorIs(t, err, api.ErrLargePayloadTooLarge) + _, err = store.Resolve(context.Background(), "file://sha256/not-a-hash") + require.Error(t, err) +} diff --git a/payload/testdata/blob_tokens.json b/payload/testdata/blob_tokens.json new file mode 100644 index 00000000..1caacd8e --- /dev/null +++ b/payload/testdata/blob_tokens.json @@ -0,0 +1,17 @@ +[ + { + "token": "blob:v2:https://myaccount.blob.core.windows.net/mycontainer/abc123def456", + "container": "mycontainer", + "name": "abc123def456" + }, + { + "token": "blob:v2:http://127.0.0.1:10000/devstoreaccount1/mycontainer/abc123def456", + "container": "mycontainer", + "name": "abc123def456" + }, + { + "token": "blob:v1:mycontainer:abc123def456", + "container": "mycontainer", + "name": "abc123def456" + } +] diff --git a/samples/azurefunctions/.gitignore b/samples/azurefunctions/.gitignore deleted file mode 100644 index 03c75a9c..00000000 --- a/samples/azurefunctions/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# Azure Functions build output (when using extensions.csproj) -bin/ -obj/ \ No newline at end of file diff --git a/samples/azurefunctions/HelloCities/function.json b/samples/azurefunctions/HelloCities/function.json deleted file mode 100644 index 5354873f..00000000 --- a/samples/azurefunctions/HelloCities/function.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "bindings": [ - { - "name": "context", - "type": "orchestrationTrigger", - "direction": "in" - } - ] - } \ No newline at end of file diff --git a/samples/azurefunctions/SayHello/function.json b/samples/azurefunctions/SayHello/function.json deleted file mode 100644 index cce00b91..00000000 --- a/samples/azurefunctions/SayHello/function.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "bindings": [ - { - "name": "name", - "type": "activityTrigger", - "direction": "in" - } - ] - } \ No newline at end of file diff --git a/samples/azurefunctions/demo.http b/samples/azurefunctions/demo.http deleted file mode 100644 index ef923f62..00000000 --- a/samples/azurefunctions/demo.http +++ /dev/null @@ -1,3 +0,0 @@ - -// Start a new orchestration instance -POST http://localhost:7071/runtime/webhooks/durabletask/orchestrators/HelloCities \ No newline at end of file diff --git a/samples/azurefunctions/extensions.csproj b/samples/azurefunctions/extensions.csproj deleted file mode 100644 index 07137dc4..00000000 --- a/samples/azurefunctions/extensions.csproj +++ /dev/null @@ -1,12 +0,0 @@ - - - netcoreapp3.1 - - - - - - - - - \ No newline at end of file diff --git a/samples/azurefunctions/host.json b/samples/azurefunctions/host.json deleted file mode 100644 index fa9cd919..00000000 --- a/samples/azurefunctions/host.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "version": "2.0", - "customHandler": { - "description": { - "defaultExecutablePath": "./azurefunctions" - } - }, - "extensions": { - "durableTask": { - "hubName": "golang" - } - } -} \ No newline at end of file diff --git a/samples/azurefunctions/local.settings.json b/samples/azurefunctions/local.settings.json deleted file mode 100644 index 8edbdd6b..00000000 --- a/samples/azurefunctions/local.settings.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "IsEncrypted": false, - "Values": { - "AzureWebJobsStorage": "UseDevelopmentStorage=true", - "FUNCTIONS_WORKER_RUNTIME": "Custom" - } - } \ No newline at end of file diff --git a/samples/azurefunctions/main.go b/samples/azurefunctions/main.go deleted file mode 100644 index b93b64cb..00000000 --- a/samples/azurefunctions/main.go +++ /dev/null @@ -1,49 +0,0 @@ -package main - -import ( - "fmt" - "log" - "net/http" - "os" - - "github.com/microsoft/durabletask-go/task" -) - -// HelloCities is an orchestrator function that generates a "hello" message for several cities. -func HelloCities(ctx *task.OrchestrationContext) (any, error) { - var helloTokyo string - if err := ctx.CallActivity(SayHello, task.WithActivityInput("Tokyo")).Await(&helloTokyo); err != nil { - return nil, err - } - var helloLondon string - if err := ctx.CallActivity(SayHello, task.WithActivityInput("London")).Await(&helloLondon); err != nil { - return nil, err - } - var helloSeattle string - if err := ctx.CallActivity(SayHello, task.WithActivityInput("Seattle")).Await(&helloSeattle); err != nil { - return nil, err - } - return []string{helloTokyo, helloLondon, helloSeattle}, nil -} - -// SayHello is an activity function that takes a city name as an input and returns "Hello, {city}!" -func SayHello(ctx task.ActivityContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return "", err - } - return fmt.Sprintf("Hello, %s!", input), nil -} - -func main() { - listenAddr := ":8080" - if val, ok := os.LookupEnv("FUNCTIONS_CUSTOMHANDLER_PORT"); ok { - listenAddr = ":" + val - } - - http.HandleFunc("/HelloCities", MapOrchestrator(HelloCities)) - http.HandleFunc("/SayHello", MapActivity(SayHello)) - - log.Printf("Listening for function invocations on %s\n", listenAddr) - log.Fatal(http.ListenAndServe(listenAddr, nil)) -} diff --git a/samples/azurefunctions/middleware.go b/samples/azurefunctions/middleware.go deleted file mode 100644 index 980a8349..00000000 --- a/samples/azurefunctions/middleware.go +++ /dev/null @@ -1,159 +0,0 @@ -package main - -import ( - "context" - "encoding/base64" - "encoding/json" - "fmt" - "net/http" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/microsoft/durabletask-go/task" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -// NOTE: For more details on custom handlers, see https://learn.microsoft.com/en-us/azure/azure-functions/functions-custom-handlers. - -type InvokeRequest struct { - Data map[string]interface{} - Metadata map[string]interface{} -} - -type InvokeResponse struct { - Outputs map[string]string - Logs []string - ReturnValue json.RawMessage -} - -func MapOrchestrator(o task.Orchestrator) func(http.ResponseWriter, *http.Request) { - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN("*", o); err != nil { - panic(fmt.Errorf("ERROR: Failed to register the orchestrator function: %w", err)) - } - executor := task.NewTaskExecutor(r) - - return func(w http.ResponseWriter, httpReq *http.Request) { - var invokeRequest InvokeRequest - d := json.NewDecoder(httpReq.Body) - if err := d.Decode(&invokeRequest); err != nil { - fmt.Printf("ERROR: Failed to decode invoke request: %v\n", err) - return - } - - // TODO: Give the schema, construct the context object and invoke the orchestrator - contextParam := invokeRequest.Data["context"] - base64encodedPayload := contextParam.(string) - - if err := json.Unmarshal([]byte(base64encodedPayload), &base64encodedPayload); err != nil { - fmt.Printf("ERROR: Failed to json-decode context payload string: %v\n", err) - return - } - - protoBytes, err := base64.StdEncoding.DecodeString(base64encodedPayload) - if err != nil { - fmt.Printf("ERROR: Failed to base64-decode request string: %v\n", err) - return - } - - var request protos.OrchestratorRequest - if err := proto.Unmarshal(protoBytes, &request); err != nil { - fmt.Printf("ERROR: Failed to deserialize request protobuf: %v\n", err) - return - } - fmt.Printf("Orchestrator request for instance ID '%s': %v\n", request.InstanceId, &request) - - results, err := executor.ExecuteOrchestrator(context.TODO(), api.InstanceID(request.InstanceId), request.PastEvents, request.NewEvents) - if err != nil { - fmt.Printf("ERROR: Unexpected failure executing the orchestrator function: %v\n", err) - return - } - fmt.Printf("Orchestrator returned a response: %v\n", results.Response) - - respBytes, err := proto.Marshal(results.Response) - if err != nil { - fmt.Printf("ERROR: Failed to marshal orchestrator results to protobuf: %v\n", err) - return - } - - base64bytes := base64.StdEncoding.EncodeToString(respBytes) - fmt.Printf("Sending back base64 encoded string: %s\n", base64bytes) - - // Send the response back to the Functions host in a JSON envelope - invokeResponse := &InvokeResponse{ReturnValue: []byte(`"` + base64bytes + `"`)} - responseJson, err := json.Marshal(invokeResponse) - if err != nil { - fmt.Printf("ERROR: Failed to marshal response payload to JSON: %v\n", err) - return - } - fmt.Println("Sending response JSON:", string(responseJson)) - w.Header().Set("Content-Type", "application/json") - if _, err := w.Write(responseJson); err != nil { - fmt.Printf("ERROR: Failed to write response: %v\n", err) - } - } -} - -func MapActivity(a task.Activity) func(http.ResponseWriter, *http.Request) { - r := task.NewTaskRegistry() - if err := r.AddActivityN("*", a); err != nil { - panic(fmt.Errorf("ERROR: Failed to register the activity function: %w", err)) - } - executor := task.NewTaskExecutor(r) - - return func(w http.ResponseWriter, r *http.Request) { - var invokeRequest InvokeRequest - d := json.NewDecoder(r.Body) - if err := d.Decode(&invokeRequest); err != nil { - fmt.Printf("ERROR: Failed to decode invoke request: %v\n", err) - return - } - - fmt.Println("Activity request:", invokeRequest) - - sys := invokeRequest.Metadata["sys"].(map[string]interface{}) - name := sys["MethodName"].(string) - instanceID := invokeRequest.Metadata["instanceId"].(string) - - var rawInput *wrapperspb.StringValue - if data, ok := invokeRequest.Metadata["data"]; ok && data != nil { - rawInputStr := data.(string) - rawInput = wrapperspb.String(rawInputStr) - } - - ts := helpers.NewTaskScheduledEvent(-1, name, nil, rawInput, nil) - e, err := executor.ExecuteActivity(context.TODO(), api.InstanceID(instanceID), ts) - if err != nil { - panic(fmt.Errorf("ERROR: Activity execution failed with an error: %w", err)) - } - - // Send the response back to the Functions host in a JSON envelope - var returnValue string - var statusCode int - if tc := e.GetTaskCompleted(); tc != nil { - fmt.Printf("Task completed: %v\n", tc.Result.GetValue()) - returnValue = tc.Result.GetValue() - statusCode = 200 - } else if tf := e.GetTaskFailed(); tf != nil { - fmt.Printf("Task failed: %v\n", tf.FailureDetails) - statusCode = 500 - } else { - panic(fmt.Errorf("unexpected event type: %v", e)) - } - - invokeResponse := &InvokeResponse{ReturnValue: []byte(returnValue)} - responseJson, err := json.Marshal(invokeResponse) - if err != nil { - fmt.Printf("ERROR: Failed to marshal response payload to JSON: %v\n", err) - return - } - fmt.Printf("Sending %d response: %s\n", statusCode, string(responseJson)) - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(statusCode) - if _, err := w.Write(responseJson); err != nil { - fmt.Printf("ERROR: Failed to write response: %v\n", err) - } - } -} diff --git a/samples/distributedtracing/distributedtracing.go b/samples/distributedtracing/distributedtracing.go index e99d7ffb..8322b6a0 100644 --- a/samples/distributedtracing/distributedtracing.go +++ b/samples/distributedtracing/distributedtracing.go @@ -1,121 +1,122 @@ +// Command distributedtracing starts an application caller span, propagates its +// W3C trace context into Durable Task Scheduler, and exports application-process +// spans to a local OpenTelemetry collector over OTLP/HTTP. DTS emits orchestration, +// activity, and timer telemetry service-side. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// cd samples/distributedtracing && go run . package main import ( "context" "encoding/json" + "fmt" "log" + "net/http" "time" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/exporters/zipkin" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/resource" - "go.opentelemetry.io/otel/sdk/trace" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + oteltrace "go.opentelemetry.io/otel/trace" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { // Tracing can be configured independently of the orchestration code. - tp, err := ConfigureZipkinTracing() + tp, err := ConfigureOTLPTracing(context.Background()) if err != nil { - log.Fatalf("Failed to create tracer: %v", err) + return fmt.Errorf("failed to create tracer: %w", err) } defer func() { if err := tp.Shutdown(context.Background()); err != nil { - log.Fatalf("Failed to stop tracer: %v", err) + log.Printf("Failed to stop tracer: %v", err) } }() // Create a new task registry and add the orchestrator and activities r := task.NewTaskRegistry() - if err := r.AddOrchestrator(DistributedTraceSampleOrchestrator); err != nil { - log.Fatalf("Failed to register orchestrator: %v", err) //nolint:gocritic // Fatalf in sample main() is acceptable + if err := r.AddOrchestratorN("DistributedTraceSampleOrchestrator", DistributedTraceSampleOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) } - if err := r.AddActivity(DoWorkActivity); err != nil { - log.Fatalf("Failed to register activity: %v", err) + if err := r.AddActivityN("DoWorkActivity", DoWorkActivity); err != nil { + return fmt.Errorf("failed to register activity: %w", err) } - if err := r.AddActivity(CallHttpEndpointActivity); err != nil { - log.Fatalf("Failed to register activity: %v", err) + if err := r.AddActivityN("CallHttpEndpointActivity", CallHttpEndpointActivity); err != nil { + return fmt.Errorf("failed to register activity: %w", err) } - // Init the client - ctx := context.Background() - client, worker, err := Init(ctx, r) + // Connect a client and worker to the Durable Task Scheduler task hub + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, r) if err != nil { - log.Fatalf("Failed to initialize the client: %v", err) + return err } defer func() { - if err := worker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) } }() - // Start a new orchestration - id, err := client.ScheduleNewOrchestration(ctx, DistributedTraceSampleOrchestrator) + // A sampled caller span is propagated to DTS when the orchestration is + // scheduled, allowing service-side spans to join the application's trace. + callerCtx, callerSpan := otel.Tracer("durabletask-sample").Start( + ctx, + "schedule_distributed_trace_sample", + oteltrace.WithSpanKind(oteltrace.SpanKindClient), + ) + defer callerSpan.End() + id, err := app.Client.ScheduleNewOrchestration(callerCtx, "DistributedTraceSampleOrchestrator") if err != nil { - log.Fatalf("Failed to schedule new orchestration: %v", err) + return fmt.Errorf("failed to schedule new orchestration: %w", err) } // Wait for the orchestration to complete - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(callerCtx, id) if err != nil { - log.Fatalf("Failed to wait for orchestration to complete: %v", err) + return fmt.Errorf("failed to wait for orchestration to complete: %w", err) } // Print the results metadataEnc, err := json.MarshalIndent(metadata, "", " ") if err != nil { - log.Fatalf("Failed to encode result to JSON: %v", err) + return fmt.Errorf("failed to encode result to JSON: %w", err) } log.Printf("Orchestration completed: %v", string(metadataEnc)) + return nil } -// Init creates and initializes an in-memory client and worker pair with default configuration. -func Init(ctx context.Context, r *task.TaskRegistry) (backend.TaskHubClient, backend.TaskHubWorker, error) { - logger := backend.DefaultLogger() - - // Create an executor - executor := task.NewTaskExecutor(r) - - // Create a new backend - // Use the in-memory sqlite provider by specifying "" - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - - // Start the worker - err := taskHubWorker.Start(ctx) - if err != nil { - return nil, nil, err - } - - // Get the client to the backend - taskHubClient := backend.NewTaskHubClient(be) - - return taskHubClient, taskHubWorker, nil -} - -func ConfigureZipkinTracing() (*trace.TracerProvider, error) { - // Inspired by this sample: https://github.com/open-telemetry/opentelemetry-go/blob/main/example/zipkin/main.go - exp, err := zipkin.New("http://localhost:9411/api/v2/spans") +func ConfigureOTLPTracing(ctx context.Context) (*sdktrace.TracerProvider, error) { + exp, err := otlptracehttp.New( + ctx, + otlptracehttp.WithEndpoint("localhost:4318"), + otlptracehttp.WithInsecure(), + ) if err != nil { return nil, err } // NOTE: The simple span processor is not recommended for production. // Instead, the batch span processor should be used for production. - processor := trace.NewSimpleSpanProcessor(exp) - // processor := trace.NewBatchSpanProcessor(exp) + processor := sdktrace.NewSimpleSpanProcessor(exp) + // processor := sdktrace.NewBatchSpanProcessor(exp) - tp := trace.NewTracerProvider( - trace.WithSpanProcessor(processor), - trace.WithSampler(trace.AlwaysSample()), - trace.WithResource(resource.NewWithAttributes( + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(processor), + sdktrace.WithSampler(sdktrace.AlwaysSample()), + sdktrace.WithResource(resource.NewWithAttributes( "durabletask.io", attribute.KeyValue{Key: "service.name", Value: attribute.StringValue("sample-app")}, )), @@ -125,15 +126,15 @@ func ConfigureZipkinTracing() (*trace.TracerProvider, error) { } // DistributedTraceSampleOrchestrator is a simple orchestration that's intended to generate -// distributed trace output to the configured exporter (e.g. zipkin). +// distributed trace output to the configured exporter. func DistributedTraceSampleOrchestrator(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity(DoWorkActivity, task.WithActivityInput(1*time.Second)).Await(nil); err != nil { + if err := ctx.CallActivity("DoWorkActivity", task.WithActivityInput(1*time.Second)).Await(nil); err != nil { return nil, err } if err := ctx.CreateTimer(2 * time.Second).Await(nil); err != nil { return nil, err } - if err := ctx.CallActivity(CallHttpEndpointActivity, task.WithActivityInput("https://bing.com")).Await(nil); err != nil { + if err := ctx.CallActivity("CallHttpEndpointActivity", task.WithActivityInput("https://bing.com")).Await(nil); err != nil { return nil, err } return nil, nil @@ -163,11 +164,23 @@ func CallHttpEndpointActivity(ctx task.ActivityContext) (any, error) { return "", err } - // ActivityContext.Context() returns a context instrumented with span information. - // The OTel HTTP client will use this to produce child spans accordingly. - _, err := otelhttp.Get(ctx.Context(), url) + // The OTel HTTP client records the outbound request in the worker process. + // DTS owns the service-side activity span. + req, err := http.NewRequestWithContext(ctx.Context(), http.MethodGet, url, nil) + if err != nil { + return nil, err + } + httpClient := &http.Client{ + Transport: otelhttp.NewTransport(http.DefaultTransport), + } + resp, err := httpClient.Do(req) if err != nil { return nil, err } + defer func() { + if err := resp.Body.Close(); err != nil { + log.Printf("Failed to close HTTP response body: %v", err) + } + }() return nil, nil } diff --git a/samples/distributedtracing/go.mod b/samples/distributedtracing/go.mod new file mode 100644 index 00000000..952ca0f7 --- /dev/null +++ b/samples/distributedtracing/go.mod @@ -0,0 +1,45 @@ +module github.com/microsoft/durabletask-go/samples/distributedtracing + +go 1.25.0 + +require ( + github.com/microsoft/durabletask-go v0.0.0 + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 + go.opentelemetry.io/otel v1.46.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 + go.opentelemetry.io/otel/sdk v1.46.0 + go.opentelemetry.io/otel/trace v1.46.0 +) + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 // indirect + go.opentelemetry.io/otel/metric v1.46.0 // indirect + go.opentelemetry.io/proto/otlp v1.11.0 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/net v0.58.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.41.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a // indirect + google.golang.org/grpc v1.83.2 // indirect + google.golang.org/protobuf v1.36.12 // indirect +) + +replace github.com/microsoft/durabletask-go => ../.. diff --git a/samples/distributedtracing/go.sum b/samples/distributedtracing/go.sum new file mode 100644 index 00000000..c7684089 --- /dev/null +++ b/samples/distributedtracing/go.sum @@ -0,0 +1,90 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1 h1:zvXfGJCWvywnCA814d8ZiVyt+fm9nnTE8xSb99zRyfo= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.23.1/go.mod h1:iptorS+VYKFL2N6PnebpS91dubG35eAOEERnT4PJbQU= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1 h1:u93s+zU2JD62im61Bm5CZIc1ZrOJaIAWEg0WOrMVkEo= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.14.1/go.mod h1:oXtinPO4OLj9d1DOTrqrL1oRwGhcqadvAmrl6wTeGlk= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0 h1:xFaZZ+IubdftrDHnGGwZ6QvQ3KHTtWl2MCK+GMt2vxs= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.4.0/go.mod h1:mCBhUhlMjLLJKr5aqw2TNS/VqJOie8MzWq3DAMJeKso= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0 h1:fhqpLE3UEXi9lPaBRpQ6XuRW0nU7hgg4zlmZZa+a9q4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.12.0/go.mod h1:7dCRMLwisfRH3dBupKeNCioWYUZ4SS09Z14H+7i8ZoY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0 h1:irsmOWwkp0KCTTNS5e2hdFeIvSQClQo2No3IaNmL3Vw= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.8.0/go.mod h1:GWcBkQj3MqN7ozHKLaCCAuNLiXoIGv2RtanfAwSjY/Y= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0 h1:MDT4FxAPve5FnYn6vOL1r7RCRDG+l9cI7a5LlCuHsqA= +github.com/AzureAD/microsoft-authentication-library-for-go v1.9.0/go.mod h1:Y33QHnf0FfdVewFFISOGe20mkZbxX4H839o955/PoeI= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0 h1:/Tnpcb2E0Pz/tN9s3bfEY2Q8ePCEX9iuS+cneUwncnw= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.30.0/go.mod h1:zOBXOsUaBSjKgmH4OGzV1esUpR3oUSCPYVd2cUBjKYY= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= +github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0 h1:3g7B90UzBltIDKq1/5mrTGxTnOFDV0ICOhLoxiZ8jlg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.71.0/go.mod h1:Ef8SuTh59BT7+ofpDxN9z+yOlc4t2GjLmKDgYNJL/NU= +go.opentelemetry.io/otel v1.46.0 h1:FHt5/CDyVxi/8IM1CH7VE/rRgq3kLHa2mSTVMO8AWyc= +go.opentelemetry.io/otel v1.46.0/go.mod h1:Gj3SEScelsNC45tp4nSxRYlS+f5iez7W8XPMCt905kE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0 h1:OFnwLJr+pF3iHrlGSzbxyuo6/6HyBlnlN1CWEJmBVcw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.46.0/go.mod h1:716wFneO0ov19A2beH5hjfh9AK5z/VWNAtDijp1Y0/g= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 h1:KrC1YrQeSt46ITMWAbgQx1M1eV1/1TKzttrBzymPmss= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0/go.mod h1:zDSEzoEqsOrgBeGvH66KRgxh90VonFyJqBHA0Pk3+rM= +go.opentelemetry.io/otel/metric v1.46.0 h1:yBnkXvgV7AXFILZc5K6IZe/CBFF3OS7BJ8ov6/lj0K8= +go.opentelemetry.io/otel/metric v1.46.0/go.mod h1:iPmdWqifKUdzziPkvvzIJXITl56fQx2mGM/DHLB3/2o= +go.opentelemetry.io/otel/sdk v1.46.0 h1:h5CNQQjEbuQXY/JfZtgt3i7HVFV3aHPO2OAwO2eTYPI= +go.opentelemetry.io/otel/sdk v1.46.0/go.mod h1:GAERFXFt5SYCEB+YiKUbMBeza6UaDH7GmGOZEfh2gSM= +go.opentelemetry.io/otel/sdk/metric v1.46.0 h1:0piZ26EG4RBfebb2jhDH6ERCYHoVWduc3kLgPCwSnSE= +go.opentelemetry.io/otel/sdk/metric v1.46.0/go.mod h1:I1PbKrdVc8Qu8HYVDNtqVIwLwjNrhsV/uFuxfwg8mO4= +go.opentelemetry.io/otel/trace v1.46.0 h1:OULy7ccdJnZtJ0UDYFOIGaCmiWzJ8Vi2G/Rsu60qs1c= +go.opentelemetry.io/otel/trace v1.46.0/go.mod h1:J7GAXweO77XSFkB/rmAqk9D6ihszhFjLU+d9WuUxDLI= +go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6TbEdMIk= +go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To= +golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= +golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a h1:i3TAXhpKc7TUP1VAPiBBrv45kamjoizCC3rOC0cAbOs= +google.golang.org/genproto/googleapis/api v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:CvYJHpbzPlT0fb/PsgtAamdwru/GVxUsomFdXTpOTI8= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a h1:3Dnd1cDaZlB68lziofO+bJXpjOy8UfRv8Unt+yH8tQ4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260831171406-18b4a7587f8a/go.mod h1:DjtHYE8FKJLivXcBEjGwndXfIC23G0VpXiXKqG179uA= +google.golang.org/grpc v1.83.2 h1:EManeRomTObA0BU7I8vXgg/78uE5MJ9M8B39EX2WscU= +google.golang.org/grpc v1.83.2/go.mod h1:YPI1hK3kDked6iHvgX3tR0y+nX/qpMFKhPgFsokw1S8= +google.golang.org/protobuf v1.36.12 h1:pJOKDDOyeXErUroCihFAd5LQuwXBSpVnKGrj5o/fwxc= +google.golang.org/protobuf v1.36.12/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= diff --git a/samples/durabletaskscheduler/durabletaskscheduler.go b/samples/durabletaskscheduler/durabletaskscheduler.go new file mode 100644 index 00000000..32d137b2 --- /dev/null +++ b/samples/durabletaskscheduler/durabletaskscheduler.go @@ -0,0 +1,136 @@ +// Command durabletaskscheduler demonstrates the Durable Task Scheduler surface: +// versioned registrations, tagged scheduling, orchestration history, and +// recurring scheduled tasks. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/durabletaskscheduler +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + options, err := dtssample.Options() + if err != nil { + return err + } + options.Versioning = &task.VersioningOptions{ + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchNone, + } + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorNVersion("ActivitySequence", "1.0", activitySequence); err != nil { + return err + } + if err := registry.AddActivityNVersion("SayHello", "1.0", sayHello); err != nil { + return err + } + if err := durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion(registry, options.Versioning.DefaultVersion); err != nil { + return err + } + + // Connect a client and worker to the Durable Task Scheduler task hub + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.StartWithOptions(ctx, options, registry, durabletaskscheduler.WithScheduledTasks()) + if err != nil { + return err + } + defer func() { + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) + } + }() + schedulerClient := app.Client + + instanceID, err := schedulerClient.ScheduleNewOrchestration( + ctx, + "ActivitySequence", + api.WithTags(map[string]string{"sample": "durable-task-scheduler"}), + ) + if err != nil { + return err + } + metadata, err := schedulerClient.WaitForOrchestrationCompletion(ctx, instanceID) + if err != nil { + return err + } + output, err := json.MarshalIndent(metadata, "", " ") + if err != nil { + return err + } + fmt.Println(string(output)) + query, err := schedulerClient.QueryInstances(ctx, api.OrchestrationQuery{ + Tags: map[string]string{"sample": "durable-task-scheduler"}, + }) + if err != nil { + return err + } + fmt.Printf("matched %d tagged orchestration(s)\n", len(query.Orchestrations)) + + history, err := schedulerClient.GetOrchestrationHistory(ctx, instanceID, api.HistoryQuery{ + ExecutionID: metadata.ExecutionID, + }) + if err != nil { + return err + } + fmt.Printf("history contains %d event(s)\n", len(history.Events)) + + scheduleID := "sample-hourly" + schedule, err := schedulerClient.ScheduledTasks().Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ + ScheduleID: scheduleID, + OrchestrationName: "ActivitySequence", + Interval: time.Hour, + StartAt: time.Now().UTC().Add(time.Hour), + Tags: map[string]string{"sample": "scheduled-task"}, + }) + if err != nil { + return err + } + description, err := schedule.Describe(ctx) + if err != nil { + return err + } + fmt.Printf("schedule %s is %s; next run: %s\n", description.ScheduleID, description.Status, description.NextRunAt) + if err := schedule.Delete(ctx); err != nil { + return err + } + return nil +} + +func activitySequence(ctx *task.OrchestrationContext) (any, error) { + var results []string + for _, city := range []string{"Tokyo", "London", "Seattle"} { + var result string + if err := ctx.CallActivity("SayHello", task.WithActivityInput(city)).Await(&result); err != nil { + return nil, err + } + results = append(results, result) + } + return results, nil +} + +func sayHello(ctx task.ActivityContext) (any, error) { + var city string + if err := ctx.GetInput(&city); err != nil { + return nil, err + } + return "Hello, " + city + "!", nil +} diff --git a/samples/entity/entity.go b/samples/entity/entity.go new file mode 100644 index 00000000..fb75f5ee --- /dev/null +++ b/samples/entity/entity.go @@ -0,0 +1,266 @@ +// This sample demonstrates how to use durable entities with the Durable Task Go SDK. +// It shows two patterns: +// +// 1. A raw entity function (Counter) with manual operation dispatch +// +// 2. An auto-dispatch entity (BankAccount) where operations map to methods on a struct +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/entity +package main + +import ( + "context" + "fmt" + "log" + "time" + + "github.com/google/uuid" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + r := task.NewTaskRegistry() + + // Pattern 1: Register a raw entity function with manual dispatch + if err := r.AddEntityN("counter", CounterEntity); err != nil { + return fmt.Errorf("failed to register counter entity: %w", err) + } + + // Pattern 2: Register a persistent entity object with separate durable state. + bankAccountFactory := task.NewEntityObjectFactory[BankAccountState, *BankAccount]( + func(task.EntityFactoryContext) (*BankAccount, error) { + return new(BankAccount), nil + }, + ) + if err := r.AddEntityFactoryN("bankaccount", bankAccountFactory); err != nil { + return fmt.Errorf("failed to register bank account entity: %w", err) + } + if err := r.AddOrchestratorN("transfer", TransferOrchestrator); err != nil { + return fmt.Errorf("failed to register transfer orchestrator: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, r) + if err != nil { + return err + } + client := app.Client + defer func() { + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) + } + }() + + // Entity state is durable, so each run uses fresh keys rather than + // accumulating on top of a previous run's balances. + run := uuid.NewString() + + // --- Demo 1: Counter entity (raw function) --- + fmt.Println("=== Counter Entity Demo ===") + counterID := api.NewEntityID("counter", "myCounter-"+run) + + // Signal the entity to perform operations + if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(10)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(5)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(-3)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + + // Query the entity state + meta, err := waitForEntityState(ctx, client, counterID, "12") + if err != nil { + return fmt.Errorf("failed to fetch entity: %w", err) + } + fmt.Printf("Counter state: %s\n", meta.SerializedState) // Expected: 12 + + // --- Demo 2: BankAccount entity (auto-dispatch) --- + fmt.Println("\n=== Bank Account Entity Demo ===") + accountID := api.NewEntityID("bankaccount", "checking-"+run) + + if err := client.SignalEntity(ctx, accountID, "Deposit", api.WithSignalInput(1000)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + if err := client.SignalEntity(ctx, accountID, "Deposit", api.WithSignalInput(500)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + if err := client.SignalEntity(ctx, accountID, "Withdraw", api.WithSignalInput(200)); err != nil { + return fmt.Errorf("failed to signal entity: %w", err) + } + + meta, err = waitForEntityState(ctx, client, accountID, `{"balance":1300}`) + if err != nil { + return fmt.Errorf("failed to fetch entity: %w", err) + } + fmt.Printf("Bank account state: %s\n", meta.SerializedState) // Expected: {"balance":1300} + + savingsID := api.NewEntityID("bankaccount", "savings-"+run) + if err := client.SignalEntity(ctx, savingsID, "Deposit", api.WithSignalInput(100)); err != nil { + return fmt.Errorf("failed to initialize savings account: %w", err) + } + if _, err := waitForEntityState(ctx, client, savingsID, `{"balance":100}`); err != nil { + return fmt.Errorf("failed to initialize savings account: %w", err) + } + transferID, err := client.ScheduleNewOrchestration( + ctx, + "transfer", + api.WithInput(TransferInput{From: accountID, To: savingsID, Amount: 300}), + ) + if err != nil { + return fmt.Errorf("failed to schedule transfer: %w", err) + } + transfer, err := client.WaitForOrchestrationCompletion(ctx, transferID) + if err != nil { + return fmt.Errorf("transfer failed: %w", err) + } + fmt.Printf("Transfer result: %s\n", transfer.SerializedOutput) + + fmt.Println("\nDone!") + return nil +} + +// --- Pattern 1: Raw entity function --- + +// CounterEntity is a simple counter entity that supports "add", "get", and "reset" operations. +func CounterEntity(ctx *task.EntityContext) (any, error) { + var count int + if ctx.HasState() { + if err := ctx.GetState(&count); err != nil { + return nil, err + } + } + + switch ctx.Operation { + case "add": + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + count += amount + case "get": + // just return current value + case "reset": + count = 0 + default: + return nil, fmt.Errorf("unknown operation: %s", ctx.Operation) + } + + if err := ctx.SetState(count); err != nil { + return nil, err + } + return count, nil +} + +// --- Pattern 2: Auto-dispatch entity --- + +// BankAccount is a struct-based entity. Public methods are automatically +// dispatched by operation name (case-insensitive). +type BankAccountState struct { + Balance int `json:"balance"` +} + +type BankAccount struct { + task.EntityObjectBase[BankAccountState] +} + +func (a *BankAccount) Deposit(amount int) (any, error) { + a.State().Balance += amount + return a.State().Balance, nil +} + +func (a *BankAccount) Withdraw(amount int) (any, error) { + if amount > a.State().Balance { + return nil, fmt.Errorf( + "insufficient funds: balance=%d, withdrawal=%d", + a.State().Balance, + amount, + ) + } + a.State().Balance -= amount + return a.State().Balance, nil +} + +func (a *BankAccount) Get() (any, error) { + return a.State().Balance, nil +} + +type TransferInput struct { + From api.EntityID `json:"from"` + To api.EntityID `json:"to"` + Amount int `json:"amount"` +} + +func TransferOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input TransferInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + unlock, err := ctx.LockEntities(input.From, input.To) + if err != nil { + return nil, err + } + defer unlock() + + var fromBalance int + if err := ctx.CallEntity( + input.From, + "Withdraw", + task.WithEntityInput(input.Amount), + ).Await(&fromBalance); err != nil { + return nil, err + } + var toBalance int + if err := ctx.CallEntity( + input.To, + "Deposit", + task.WithEntityInput(input.Amount), + ).Await(&toBalance); err != nil { + return nil, err + } + return map[string]int{"from": fromBalance, "to": toBalance}, nil +} + +func waitForEntityState( + ctx context.Context, + client *durabletaskscheduler.Client, + entityID api.EntityID, + expected string, +) (*api.EntityMetadata, error) { + timeout := time.NewTimer(30 * time.Second) + defer timeout.Stop() + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + metadata, err := client.GetEntity(ctx, entityID) + // The entity does not exist until the service has processed its first + // signal, so a not-found result means "not ready yet", not a failure. + if err != nil { + return nil, err + } + if metadata != nil && metadata.SerializedState == expected { + return metadata, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-timeout.C: + return nil, fmt.Errorf("timed out waiting for %s state %s", entityID.String(), expected) + case <-ticker.C: + } + } +} diff --git a/samples/exporthistory/exporthistory.go b/samples/exporthistory/exporthistory.go new file mode 100644 index 00000000..54717bf9 --- /dev/null +++ b/samples/exporthistory/exporthistory.go @@ -0,0 +1,187 @@ +// Command exporthistory demonstrates the preview export history package: it +// runs a few orchestrations, then creates a batch export job that writes their +// durable histories to Azure Blob Storage as gzip-compressed JSONL. +// +// Configure it with a Durable Task Scheduler connection string and an Azure +// Storage connection string: +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// export EXPORT_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true" +// export EXPORT_CONTAINER="history-exports" +// go run ./samples/exporthistory +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/exporthistory" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { + connectionString := os.Getenv("DTS_CONNECTION_STRING") + if connectionString == "" { + return errors.New("DTS_CONNECTION_STRING is required") + } + storageConnectionString := os.Getenv("EXPORT_STORAGE_CONNECTION_STRING") + if storageConnectionString == "" { + return errors.New("EXPORT_STORAGE_CONNECTION_STRING is required") + } + container := os.Getenv("EXPORT_CONTAINER") + if container == "" { + container = "history-exports" + } + + options, err := durabletaskscheduler.NewOptionsFromConnectionString(connectionString) + if err != nil { + return fmt.Errorf("invalid DTS connection string: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + logger := api.DefaultLogger() + + // The export activities read orchestration metadata and history through the + // same task hub client the application uses. + client, err := durabletaskscheduler.NewClient(ctx, options, logger) + if err != nil { + return fmt.Errorf("failed to create the Durable Task Scheduler client: %w", err) + } + defer func() { + if closeErr := client.Close(); closeErr != nil { + log.Printf("failed to close the client: %v", closeErr) + } + }() + + store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ + ConnectionString: storageConnectionString, + ContainerName: container, + // Azurite serves plaintext HTTP on loopback; production endpoints are HTTPS. + AllowInsecureHTTP: strings.Contains(storageConnectionString, "http://"), + }) + if err != nil { + return fmt.Errorf("failed to create the export store: %w", err) + } + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("Greeting", greeting); err != nil { + return err + } + if err := registry.AddActivityN("SayHello", sayHello); err != nil { + return err + } + if err := exporthistory.Register(registry, exporthistory.WorkerOptions{ + Source: client.TaskHubGrpcClient, + Store: store, + }); err != nil { + return err + } + + worker, err := durabletaskscheduler.NewWorker(options, registry, logger, + durabletaskclient.WithAutoWorkItemFilters(), + // Keeps the unversioned export system tasks routable when the + // application enables strict worker versioning. + exporthistory.WithExportHistory(), + ) + if err != nil { + return fmt.Errorf("failed to create the worker: %w", err) + } + if err := worker.Start(ctx); err != nil { + return fmt.Errorf("failed to start the worker: %w", err) + } + defer func() { + shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer shutdownCancel() + if shutdownErr := worker.Shutdown(shutdownCtx); shutdownErr != nil { + log.Printf("failed to shut down the worker: %v", shutdownErr) + } + }() + + from := time.Now().UTC().Add(-time.Minute) + for _, name := range []string{"Ana", "Bo", "Cyd"} { + id, scheduleErr := client.ScheduleNewOrchestration(ctx, "Greeting", api.WithInput(name)) + if scheduleErr != nil { + return fmt.Errorf("failed to schedule an orchestration: %w", scheduleErr) + } + metadata, waitErr := client.WaitForOrchestrationCompletion(ctx, id) + if waitErr != nil { + return fmt.Errorf("failed to wait for %s: %w", id, waitErr) + } + log.Printf("orchestration %s finished with status %s", id, metadata.RuntimeStatus) + } + + exportClient, err := exporthistory.NewClient(client.TaskHubGrpcClient, exporthistory.ClientOptions{ + ContainerName: container, + }) + if err != nil { + return fmt.Errorf("failed to create the export client: %w", err) + } + + job, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ + Mode: exporthistory.ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: time.Now().UTC(), + }) + if err != nil { + return fmt.Errorf("failed to create the export job: %w", err) + } + log.Printf("created export job %s", job.ID()) + + // Batch jobs complete on their own once the window is drained. + for { + description, describeErr := job.Describe(ctx) + if describeErr != nil { + return fmt.Errorf("failed to describe the export job: %w", describeErr) + } + log.Printf("export job %s is %s (scanned %d, exported %d)", + description.JobID, description.Status, description.ScannedInstances, description.ExportedInstances) + switch description.Status { + case exporthistory.ExportJobStatusCompleted: + log.Printf("history exported to container %q under prefix %q", + description.Config.Destination.Container, description.Config.Destination.Prefix) + return nil + case exporthistory.ExportJobStatusFailed: + return fmt.Errorf("export job failed: %s", description.LastError) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(2 * time.Second): + } + } +} + +func greeting(ctx *task.OrchestrationContext) (any, error) { + var name string + if err := ctx.GetInput(&name); err != nil { + return nil, err + } + var message string + if err := ctx.CallActivity("SayHello", task.WithActivityInput(name)).Await(&message); err != nil { + return nil, err + } + return message, nil +} + +func sayHello(ctx task.ActivityContext) (any, error) { + var name string + if err := ctx.GetInput(&name); err != nil { + return nil, err + } + return "Hello, " + name + "!", nil +} diff --git a/samples/externalevents/externalevents.go b/samples/externalevents/externalevents.go index 2ca4da5e..07548c1c 100644 --- a/samples/externalevents/externalevents.go +++ b/samples/externalevents/externalevents.go @@ -1,3 +1,8 @@ +// Command externalevents demonstrates raising an external event into a running +// orchestration that is blocked on WaitForSingleEvent. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/externalevents package main import ( @@ -7,38 +12,43 @@ import ( "time" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { // Create a new task registry and add the orchestrator and activities r := task.NewTaskRegistry() - if err := r.AddOrchestrator(ExternalEventOrchestrator); err != nil { - log.Fatalf("Failed to register orchestrator: %v", err) + if err := r.AddOrchestratorN("ExternalEventOrchestrator", ExternalEventOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) } - // Init the client - ctx := context.Background() - client, worker, err := Init(ctx, r) + // Connect a client and worker to the Durable Task Scheduler task hub + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, r) if err != nil { - log.Fatalf("Failed to initialize the client: %v", err) + return err } defer func() { - if err := worker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) } }() // Start a new orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrator") + id, err := app.Client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrator") if err != nil { - log.Fatalf("Failed to schedule new orchestration: %v", err) //nolint:gocritic // Fatalf in sample main() is acceptable + return fmt.Errorf("failed to schedule new orchestration: %w", err) } - _, err = client.WaitForOrchestrationStart(ctx, id) - if err != nil { - log.Fatalf("Failed to wait for orchestration to start: %v", err) + if _, err := app.Client.WaitForOrchestrationStart(ctx, id); err != nil { + return fmt.Errorf("failed to wait for orchestration to start: %w", err) } // Prompt the user for their name and send that to the orchestrator @@ -46,49 +56,25 @@ func main() { fmt.Println("Enter your first name: ") var nameInput string if _, err := fmt.Scanln(&nameInput); err != nil { - log.Fatalf("Failed to read input: %v", err) + log.Printf("Failed to read input: %v", err) + return } - if err = client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(nameInput)); err != nil { - log.Fatalf("Failed to raise event: %v", err) + if err := app.Client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(nameInput)); err != nil { + log.Printf("Failed to raise event: %v", err) } }() // After the orchestration receives the event, it should complete on its own - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) if err != nil { - log.Fatalf("Failed to wait for orchestration to complete: %v", err) + return fmt.Errorf("failed to wait for orchestration to complete: %w", err) } if metadata.FailureDetails != nil { log.Println("orchestration failed:", metadata.FailureDetails.ErrorMessage) } else { log.Println("orchestration completed:", metadata.SerializedOutput) } -} - -// Init creates and initializes an in-memory client and worker pair with default configuration. -func Init(ctx context.Context, r *task.TaskRegistry) (backend.TaskHubClient, backend.TaskHubWorker, error) { - logger := backend.DefaultLogger() - - // Create an executor - executor := task.NewTaskExecutor(r) - - // Create a new backend - // Use the in-memory sqlite provider by specifying "" - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - - // Start the worker - err := taskHubWorker.Start(ctx) - if err != nil { - return nil, nil, err - } - - // Get the client to the backend - taskHubClient := backend.NewTaskHubClient(be) - - return taskHubClient, taskHubWorker, nil + return nil } // ExternalEventOrchestrator is an orchestrator function that blocks for 30 seconds or diff --git a/samples/heterogeneous/heterogeneous.go b/samples/heterogeneous/heterogeneous.go deleted file mode 100644 index 01a0129f..00000000 --- a/samples/heterogeneous/heterogeneous.go +++ /dev/null @@ -1,285 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "log" - "net" - "time" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" - "github.com/microsoft/durabletask-go/client" - "github.com/microsoft/durabletask-go/task" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -const ( - doubleActivityName = "double" - tripleActivityName = "triple" - orchestratorName = "sixTimes" - - localExecutorName = "local" - grpcExecutorName = "grpc" -) - -// Config defines the routing configuration for tasks. -// It maps task names to executor names and specifies a default executor. -type Config struct { - Routes map[string]string - DefaultExecutor string -} - -// Resolve returns the executor name for a given task name. -// If no specific route is found, it returns the default executor name. -func (c *Config) Resolve(taskName string) string { - executorName, ok := c.Routes[taskName] - if !ok { - executorName = c.DefaultExecutor - } - return executorName -} - -// RoutingExecutor is a backend.Executor implementation that Routes tasks to different Executors -// based on a configuration. This allows for heterogeneous execution environments where -// different tasks (orchestrators or activities) can be executed by different Executors -// (e.g., local vs gRPC). -type RoutingExecutor struct { - Config Config - Executors map[string]backend.Executor -} - -// getExecutor retrieves the appropriate backend.Executor for a given task name -// based on the routing configuration. -func (e *RoutingExecutor) getExecutor(taskName string) (backend.Executor, error) { - executorName := e.Config.Resolve(taskName) - executor, ok := e.Executors[executorName] - if !ok { - return nil, fmt.Errorf("executor %s for task %s not found", executorName, taskName) - } - return executor, nil -} - -// getOrchestratorName extracts the orchestrator name from the provided history events. -// It iterates through both old and new events to find the ExecutionStarted event, -// which contains the orchestrator's name. This is crucial for routing orchestrations -// to the correct executor based on their name. -func (e *RoutingExecutor) getOrchestratorName(oldEvents, newEvents []*backend.HistoryEvent) string { - - for _, event := range oldEvents { - if x := event.GetExecutionStarted(); x != nil { - return x.Name - } - } - for _, event := range newEvents { - if x := event.GetExecutionStarted(); x != nil { - return x.Name - } - } - return "" -} - -// ExecuteOrchestrator Routes the orchestration execution request to the appropriate executor. -// It determines the orchestrator name from history events and uses it to Resolve the executor. -func (e *RoutingExecutor) ExecuteOrchestrator(ctx context.Context, id api.InstanceID, oldEvents []*backend.HistoryEvent, newEvents []*backend.HistoryEvent) (*backend.ExecutionResults, error) { - name := e.getOrchestratorName(oldEvents, newEvents) - - executor, err := e.getExecutor(name) - if err != nil { - return nil, err - } - - return executor.ExecuteOrchestrator(ctx, id, oldEvents, newEvents) -} - -// ExecuteActivity Routes the activity execution request to the appropriate executor. -// It extracts the activity name from the task scheduled event and uses it to Resolve the executor. -func (e *RoutingExecutor) ExecuteActivity(ctx context.Context, id api.InstanceID, event *backend.HistoryEvent) (*backend.HistoryEvent, error) { - name := event.GetTaskScheduled().GetName() - executor, err := e.getExecutor(name) - if err != nil { - return nil, err - } - return executor.ExecuteActivity(ctx, id, event) -} - -// Shutdown shuts down all registered Executors in parallel. -// It returns a joined error if any of the Executors fail to shut down. -func (e *RoutingExecutor) Shutdown(ctx context.Context) error { - var errs []error - for _, executor := range e.Executors { - err := executor.Shutdown(ctx) - if err != nil { - errs = append(errs, err) - } - } - - if len(errs) == 0 { - return nil - } - - return errors.Join(errs...) -} - -func main() { - - ctx, cancelFunc := context.WithTimeout(context.Background(), 5*time.Second) - defer cancelFunc() - - logger := backend.DefaultLogger() - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - - executor, err := initExecutor(ctx, be, logger) - if err != nil { - panic(err) - } - - workflowWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, workflowWorker, activityWorker, logger) - - if err := taskHubWorker.Start(ctx); err != nil { - panic(err) - } - defer func() { - if err := taskHubWorker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) - } - }() - - taskHubClient := backend.NewTaskHubClient(be) - id, err := taskHubClient.ScheduleNewOrchestration(ctx, orchestratorName, api.WithInput(1)) - if err != nil { - panic(err) - } - metadata, err := taskHubClient.WaitForOrchestrationCompletion(ctx, id) - if err != nil { - panic(err) - } - fmt.Println(metadata.SerializedOutput) -} - -func initExecutor(ctx context.Context, be backend.Backend, logger backend.Logger) (backend.Executor, error) { - localExecutor, err := setupLocalExecutor() - if err != nil { - return nil, err - } - - grpcExecutor, err := setupGrpcExecutor(ctx, be, logger) - if err != nil { - return nil, err - } - - return &RoutingExecutor{ - Config: Config{ - Routes: map[string]string{ - doubleActivityName: localExecutorName, - tripleActivityName: localExecutorName, - }, - DefaultExecutor: grpcExecutorName, - }, - Executors: map[string]backend.Executor{ - localExecutorName: localExecutor, - grpcExecutorName: grpcExecutor, - }, - }, nil -} - -func setupLocalExecutor() (backend.Executor, error) { - timesActivity := func(times int) task.Activity { - return func(ctx task.ActivityContext) (any, error) { - var input int - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - return input * times, nil - } - } - doubleActivity := timesActivity(2) - tripleActivity := timesActivity(3) - - r := task.NewTaskRegistry() - if err := r.AddActivityN(doubleActivityName, doubleActivity); err != nil { - return nil, err - } - - if err := r.AddActivityN(tripleActivityName, tripleActivity); err != nil { - return nil, err - } - - return task.NewTaskExecutor(r), nil -} - -func setupGrpcExecutor(ctx context.Context, be backend.Backend, logger backend.Logger) (backend.Executor, error) { - address := "localhost:0" - grpcServer := grpc.NewServer() - executor, registerFn := backend.NewGrpcExecutor(be, logger) - registerFn(grpcServer) - - lis, err := net.Listen("tcp", address) - if err != nil { - return nil, err - } - go func() { - if err := grpcServer.Serve(lis); err != nil { - panic(err) - } - }() - - go func() { - <-ctx.Done() - grpcServer.GracefulStop() - _ = lis.Close() - }() - - // Create a worker that connects to the gRPC server. - // establish a gRPC connection, blocking until the server is ready or the timeout expires - conn, err := grpc.DialContext( - ctx, - lis.Addr().String(), - grpc.WithTransportCredentials(insecure.NewCredentials()), - grpc.WithBlock(), - ) - if err != nil { - return nil, err - } - - go func() { - <-ctx.Done() - _ = conn.Close() - }() - - orchestrator := func(ctx *task.OrchestrationContext) (any, error) { - var input int - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - - var intermediateResult int - if err := ctx.CallActivity(doubleActivityName, task.WithActivityInput(input)).Await(&intermediateResult); err != nil { - return nil, err - } - - var finalResult int - if err := ctx.CallActivity(tripleActivityName, task.WithActivityInput(intermediateResult)).Await(&finalResult); err != nil { - return nil, err - } - return finalResult, nil - } - - workerClient := client.NewTaskHubGrpcClient(conn, logger) - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN(orchestratorName, orchestrator); err != nil { - return nil, err - } - - // StartWorkItemListener is not blocking - if err := workerClient.StartWorkItemListener(ctx, r); err != nil { - return nil, err - } - - return executor, nil -} diff --git a/samples/internal/dtssample/dtssample.go b/samples/internal/dtssample/dtssample.go new file mode 100644 index 00000000..b019e999 --- /dev/null +++ b/samples/internal/dtssample/dtssample.go @@ -0,0 +1,108 @@ +// Package dtssample holds the Durable Task Scheduler setup that most runnable +// samples in this repository share: reading connection settings from the +// environment, opening a management client, and starting a worker against the +// same task hub. +// +// Point the samples at a scheduler with a connection string, for example the +// local emulator: +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +package dtssample + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/task" +) + +// ConnectionStringVariable names the environment variable used by the +// connection-string-driven samples to locate their DTS task hub. +const ConnectionStringVariable = "DTS_CONNECTION_STRING" + +// App owns a Durable Task Scheduler client and worker pair that are connected to +// the same task hub. +type App struct { + // Client schedules orchestrations and reads their state. + Client *durabletaskscheduler.Client + // worker executes the orchestrators, activities, and entities in the + // registry it was started with, until [App.Shutdown] stops it. + worker *durabletaskclient.TaskHubGrpcWorker +} + +// Options builds scheduler options from ConnectionStringVariable. +func Options() (*durabletaskscheduler.Options, error) { + connectionString := os.Getenv(ConnectionStringVariable) + if connectionString == "" { + return nil, fmt.Errorf("%s is required", ConnectionStringVariable) + } + options, err := durabletaskscheduler.NewOptionsFromConnectionString(connectionString) + if err != nil { + return nil, fmt.Errorf("invalid %s: %w", ConnectionStringVariable, err) + } + return options, nil +} + +// Start connects a client and starts a worker for registry against the task hub +// named by ConnectionStringVariable. The caller owns the returned App and must +// call [App.Shutdown]. +// +// The worker derives its work-item filters from registry; pass additional +// workerOptions to layer more behavior on top. +func Start( + ctx context.Context, + registry *task.TaskRegistry, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) (*App, error) { + options, err := Options() + if err != nil { + return nil, err + } + return StartWithOptions(ctx, options, registry, workerOptions...) +} + +// StartWithOptions is [Start] for a sample that customizes its options, such as +// enabling versioning or large payloads. +func StartWithOptions( + ctx context.Context, + options *durabletaskscheduler.Options, + registry *task.TaskRegistry, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) (*App, error) { + logger := api.DefaultLogger() + client, err := durabletaskscheduler.NewClient(ctx, options, logger) + if err != nil { + return nil, fmt.Errorf("failed to connect the Durable Task Scheduler client: %w", err) + } + // Deriving the work-item filters from the registry is the recommended + // default for every DTS worker, so samples do not repeat it. + workerOptions = append( + []durabletaskclient.TaskHubGrpcWorkerOption{durabletaskclient.WithAutoWorkItemFilters()}, + workerOptions..., + ) + worker, err := durabletaskscheduler.NewWorker(options, registry, logger, workerOptions...) + if err != nil { + return nil, errors.Join(fmt.Errorf("failed to create the worker: %w", err), client.Close()) + } + if err := worker.Start(ctx); err != nil { + return nil, errors.Join(fmt.Errorf("failed to start the worker: %w", err), client.Close()) + } + return &App{Client: client, worker: worker}, nil +} + +// Shutdown stops the worker and closes the client. It uses a background +// deadline so it still runs to completion once the sample's context is done. +func (a *App) Shutdown() error { + if a == nil { + return nil + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return errors.Join(a.worker.Shutdown(shutdownCtx), a.Client.Close()) +} diff --git a/samples/parallel/parallel.go b/samples/parallel/parallel.go index 180eb0da..86837ad7 100644 --- a/samples/parallel/parallel.go +++ b/samples/parallel/parallel.go @@ -1,102 +1,89 @@ +// Command parallel demonstrates fan-out/fan-in: an orchestration starts a +// dynamic number of activities at once and then awaits all of them. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/parallel package main import ( "context" "encoding/json" + "fmt" "log" "math/rand" "time" "github.com/google/uuid" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { // Create a new task registry and add the orchestrator and activities r := task.NewTaskRegistry() - if err := r.AddOrchestrator(UpdateDevicesOrchestrator); err != nil { - log.Fatalf("Failed to register orchestrator: %v", err) + if err := r.AddOrchestratorN("UpdateDevicesOrchestrator", UpdateDevicesOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) } - if err := r.AddActivity(GetDevicesToUpdate); err != nil { - log.Fatalf("Failed to register activity: %v", err) + if err := r.AddActivityN("GetDevicesToUpdate", GetDevicesToUpdate); err != nil { + return fmt.Errorf("failed to register activity: %w", err) } - if err := r.AddActivity(UpdateDevice); err != nil { - log.Fatalf("Failed to register activity: %v", err) + if err := r.AddActivityN("UpdateDevice", UpdateDevice); err != nil { + return fmt.Errorf("failed to register activity: %w", err) } - // Init the client - ctx := context.Background() - client, worker, err := Init(ctx, r) + // Connect a client and worker to the Durable Task Scheduler task hub + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, r) if err != nil { - log.Fatalf("Failed to initialize the client: %v", err) + return err } defer func() { - if err := worker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) } }() // Start a new orchestration - id, err := client.ScheduleNewOrchestration(ctx, UpdateDevicesOrchestrator) + id, err := app.Client.ScheduleNewOrchestration(ctx, "UpdateDevicesOrchestrator") if err != nil { - log.Fatalf("Failed to schedule new orchestration: %v", err) //nolint:gocritic // Fatalf in sample main() is acceptable + return fmt.Errorf("failed to schedule new orchestration: %w", err) } // Wait for the orchestration to complete - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) if err != nil { - log.Fatalf("Failed to wait for orchestration to complete: %v", err) + return fmt.Errorf("failed to wait for orchestration to complete: %w", err) } // Print the results metadataEnc, err := json.MarshalIndent(metadata, "", " ") if err != nil { - log.Fatalf("Failed to encode result to JSON: %v", err) + return fmt.Errorf("failed to encode result to JSON: %w", err) } log.Printf("Orchestration completed: %v", string(metadataEnc)) -} - -// Init creates and initializes an in-memory client and worker pair with default configuration. -func Init(ctx context.Context, r *task.TaskRegistry) (backend.TaskHubClient, backend.TaskHubWorker, error) { - logger := backend.DefaultLogger() - - // Create an executor - executor := task.NewTaskExecutor(r) - - // Create a new backend - // Use the in-memory sqlite provider by specifying "" - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - - // Start the worker - err := taskHubWorker.Start(ctx) - if err != nil { - return nil, nil, err - } - - // Get the client to the backend - taskHubClient := backend.NewTaskHubClient(be) - - return taskHubClient, taskHubWorker, nil + return nil } // UpdateDevicesOrchestrator is an orchestrator that runs activities in parallel func UpdateDevicesOrchestrator(ctx *task.OrchestrationContext) (any, error) { // Get a dynamic list of devices to perform updates on var devices []string - if err := ctx.CallActivity(GetDevicesToUpdate).Await(&devices); err != nil { + if err := ctx.CallActivity("GetDevicesToUpdate").Await(&devices); err != nil { return nil, err } // Start a dynamic number of tasks in parallel, not waiting for any to complete (yet) tasks := make([]task.Task, len(devices)) for i, id := range devices { - tasks[i] = ctx.CallActivity(UpdateDevice, task.WithActivityInput(id)) + tasks[i] = ctx.CallActivity("UpdateDevice", task.WithActivityInput(id)) } // Now that all are started, wait for them to complete and then return the success rate diff --git a/samples/retries/retries.go b/samples/retries/retries.go index f305d185..9c324005 100644 --- a/samples/retries/retries.go +++ b/samples/retries/retries.go @@ -1,88 +1,75 @@ +// Command retries demonstrates activity retry policies: an activity fails +// randomly and the orchestration retries it with exponential backoff. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/retries package main import ( "context" "encoding/json" "errors" + "fmt" "log" "math/rand" "time" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() error { // Create a new task registry and add the orchestrator and activities r := task.NewTaskRegistry() - if err := r.AddOrchestrator(RetryActivityOrchestrator); err != nil { - log.Fatalf("Failed to register orchestrator: %v", err) + if err := r.AddOrchestratorN("RetryActivityOrchestrator", RetryActivityOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) } - if err := r.AddActivity(RandomFailActivity); err != nil { - log.Fatalf("Failed to register activity: %v", err) + if err := r.AddActivityN("RandomFailActivity", RandomFailActivity); err != nil { + return fmt.Errorf("failed to register activity: %w", err) } - // Init the client - ctx := context.Background() - client, worker, err := Init(ctx, r) + // Connect a client and worker to the Durable Task Scheduler task hub + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, r) if err != nil { - log.Fatalf("Failed to initialize the client: %v", err) + return err } defer func() { - if err := worker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) + if err := app.Shutdown(); err != nil { + log.Printf("Failed to shut down: %v", err) } }() // Start a new orchestration - id, err := client.ScheduleNewOrchestration(ctx, RetryActivityOrchestrator) + id, err := app.Client.ScheduleNewOrchestration(ctx, "RetryActivityOrchestrator") if err != nil { - log.Fatalf("Failed to schedule new orchestration: %v", err) //nolint:gocritic // Fatalf in sample main() is acceptable + return fmt.Errorf("failed to schedule new orchestration: %w", err) } // Wait for the orchestration to complete - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) if err != nil { - log.Fatalf("Failed to wait for orchestration to complete: %v", err) + return fmt.Errorf("failed to wait for orchestration to complete: %w", err) } // Print the results metadataEnc, err := json.MarshalIndent(metadata, "", " ") if err != nil { - log.Fatalf("Failed to encode result to JSON: %v", err) + return fmt.Errorf("failed to encode result to JSON: %w", err) } log.Printf("Orchestration completed: %v", string(metadataEnc)) -} - -// Init creates and initializes an in-memory client and worker pair with default configuration. -func Init(ctx context.Context, r *task.TaskRegistry) (backend.TaskHubClient, backend.TaskHubWorker, error) { - logger := backend.DefaultLogger() - - // Create an executor - executor := task.NewTaskExecutor(r) - - // Create a new backend - // Use the in-memory sqlite provider by specifying "" - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - - // Start the worker - err := taskHubWorker.Start(ctx) - if err != nil { - return nil, nil, err - } - - // Get the client to the backend - taskHubClient := backend.NewTaskHubClient(be) - - return taskHubClient, taskHubWorker, nil + return nil } func RetryActivityOrchestrator(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity(RandomFailActivity, task.WithActivityRetryPolicy(&task.RetryPolicy{ + if err := ctx.CallActivity("RandomFailActivity", task.WithActivityRetryPolicy(&task.RetryPolicy{ MaxAttempts: 10, InitialRetryInterval: 100 * time.Millisecond, BackoffCoefficient: 2, diff --git a/samples/sequence/sequence.go b/samples/sequence/sequence.go deleted file mode 100644 index 281d3287..00000000 --- a/samples/sequence/sequence.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "fmt" - "log" - - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" - "github.com/microsoft/durabletask-go/task" -) - -func main() { - // Create a new task registry and add the orchestrator and activities - r := task.NewTaskRegistry() - if err := r.AddOrchestrator(ActivitySequenceOrchestrator); err != nil { - log.Fatalf("Failed to register orchestrator: %v", err) - } - if err := r.AddActivity(SayHelloActivity); err != nil { - log.Fatalf("Failed to register activity: %v", err) - } - - // Init the client - ctx := context.Background() - client, worker, err := Init(ctx, r) - if err != nil { - log.Fatalf("Failed to initialize the client: %v", err) - } - defer func() { - if err := worker.Shutdown(ctx); err != nil { - log.Printf("Failed to shutdown worker: %v", err) - } - }() - - // Start a new orchestration - id, err := client.ScheduleNewOrchestration(ctx, ActivitySequenceOrchestrator) - if err != nil { - log.Fatalf("Failed to schedule new orchestration: %v", err) //nolint:gocritic // Fatalf in sample main() is acceptable - } - - // Wait for the orchestration to complete - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if err != nil { - log.Fatalf("Failed to wait for orchestration to complete: %v", err) - } - - // Print the results - metadataEnc, err := json.MarshalIndent(metadata, "", " ") - if err != nil { - log.Fatalf("Failed to encode result to JSON: %v", err) - } - log.Printf("Orchestration completed: %v", string(metadataEnc)) -} - -// Init creates and initializes an in-memory client and worker pair with default configuration. -func Init(ctx context.Context, r *task.TaskRegistry) (backend.TaskHubClient, backend.TaskHubWorker, error) { - logger := backend.DefaultLogger() - - // Create an executor - executor := task.NewTaskExecutor(r) - - // Create a new backend - // Use the in-memory sqlite provider by specifying "" - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - - // Start the worker - err := taskHubWorker.Start(ctx) - if err != nil { - return nil, nil, err - } - - // Get the client to the backend - taskHubClient := backend.NewTaskHubClient(be) - - return taskHubClient, taskHubWorker, nil -} - -// ActivitySequenceOrchestrator makes three activity calls in sequence and results the results -// as an array. -func ActivitySequenceOrchestrator(ctx *task.OrchestrationContext) (any, error) { - var helloTokyo string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("Tokyo")).Await(&helloTokyo); err != nil { - return nil, err - } - var helloLondon string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("London")).Await(&helloLondon); err != nil { - return nil, err - } - var helloSeattle string - if err := ctx.CallActivity(SayHelloActivity, task.WithActivityInput("Seattle")).Await(&helloSeattle); err != nil { - return nil, err - } - return []string{helloTokyo, helloLondon, helloSeattle}, nil -} - -// SayHelloActivity can be called by an orchestrator function and will return a friendly greeting. -func SayHelloActivity(ctx task.ActivityContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return "", err - } - return fmt.Sprintf("Hello, %s!", input), nil -} diff --git a/task/activity.go b/task/activity.go index bc77429e..3f44cd71 100644 --- a/task/activity.go +++ b/task/activity.go @@ -3,18 +3,42 @@ package task import ( "context" "fmt" + "maps" "math" "time" + "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/internal/protos" "google.golang.org/protobuf/types/known/wrapperspb" ) -type callActivityOption func(*callActivityOptions) error +// CallActivityOption configures an activity invocation. +type CallActivityOption func(*callActivityOptions, api.DataConverter) error type callActivityOptions struct { rawInput *wrapperspb.StringValue + version *wrapperspb.StringValue retryPolicy *RetryPolicy + tags map[string]string +} + +func (options *callActivityOptions) versionOrInherited(inheritedVersion string) *wrapperspb.StringValue { + if options.version != nil { + return options.version + } + // Preserve nil for implicit unversioned calls; an explicit empty version remains non-nil. + if inheritedVersion == "" { + return nil + } + return wrapperspb.String(inheritedVersion) +} + +// WithActivityVersion configures the activity version. +func WithActivityVersion(version string) CallActivityOption { + return func(opt *callActivityOptions, _ api.DataConverter) error { + opt.version = wrapperspb.String(version) + return nil + } } type RetryPolicy struct { @@ -28,40 +52,59 @@ type RetryPolicy struct { MaxRetryInterval time.Duration // Total timeout across all the retries performed RetryTimeout time.Duration - // Optional function to control if retries should proceed - Handle func(error) bool + // Optional deterministic function that controls whether retries should proceed. + Handle func(RetryContext) bool } -func (policy *RetryPolicy) Validate() error { - if policy.InitialRetryInterval <= 0 { - return fmt.Errorf("InitialRetryInterval must be greater than 0") +// RetryContext contains durable inputs for a retry decision. +// Retry handlers execute during replay and must not perform I/O or depend on wall-clock time. +type RetryContext struct { + LastAttemptNumber int + // LastFailure is never nil while Handle is running and must be treated as read-only. + LastFailure *api.FailureDetails + TotalRetryTime time.Duration +} + +// Normalized validates the retry policy and returns an independent copy with +// default values applied. The receiver is not modified. +func (policy *RetryPolicy) Normalized() (RetryPolicy, error) { + if policy == nil { + return RetryPolicy{}, fmt.Errorf("%w: retry policy cannot be nil", api.ErrInvalidArgument) } - if policy.MaxAttempts <= 0 { + normalized := *policy + if normalized.InitialRetryInterval <= 0 { + return RetryPolicy{}, fmt.Errorf("%w: InitialRetryInterval must be greater than 0", api.ErrInvalidArgument) + } + if normalized.MaxAttempts <= 0 { // setting 1 max attempt is equivalent to not retrying - policy.MaxAttempts = 1 + normalized.MaxAttempts = 1 } - if policy.BackoffCoefficient <= 0 { - policy.BackoffCoefficient = 1 + if normalized.BackoffCoefficient <= 0 { + normalized.BackoffCoefficient = 1 } - if policy.MaxRetryInterval <= 0 { - policy.MaxRetryInterval = math.MaxInt64 + if normalized.MaxRetryInterval <= 0 { + normalized.MaxRetryInterval = math.MaxInt64 } - if policy.RetryTimeout <= 0 { - policy.RetryTimeout = math.MaxInt64 + if normalized.RetryTimeout <= 0 { + normalized.RetryTimeout = math.MaxInt64 } - if policy.Handle == nil { - policy.Handle = func(err error) bool { - return true - } + if normalized.Handle == nil { + normalized.Handle = func(RetryContext) bool { return true } } - return nil + return normalized, nil +} + +// Validate reports whether the retry policy is valid without modifying it. +func (policy *RetryPolicy) Validate() error { + _, err := policy.Normalized() + return err } // WithActivityInput configures an input for an activity invocation. -// The specified input must be JSON serializable. -func WithActivityInput(input any) callActivityOption { - return func(opt *callActivityOptions) error { - data, err := marshalData(input) +// The configured data converter must be able to serialize the input. +func WithActivityInput(input any) CallActivityOption { + return func(opt *callActivityOptions, converter api.DataConverter) error { + data, err := marshalData(converter, input) if err != nil { return err } @@ -71,23 +114,38 @@ func WithActivityInput(input any) callActivityOption { } // WithRawActivityInput configures a raw input for an activity invocation. -func WithRawActivityInput(input string) callActivityOption { - return func(opt *callActivityOptions) error { +func WithRawActivityInput(input string) CallActivityOption { + return func(opt *callActivityOptions, _ api.DataConverter) error { opt.rawInput = wrapperspb.String(input) return nil } } -func WithActivityRetryPolicy(policy *RetryPolicy) callActivityOption { - return func(opt *callActivityOptions) error { - if policy == nil { - return nil +// WithActivityTags adds user tags to an activity invocation. Explicit activity +// tags override orchestration tags with the same key. +func WithActivityTags(tags map[string]string) CallActivityOption { + return func(opt *callActivityOptions, _ api.DataConverter) error { + if err := validateUnreservedKeys("activity tag", tags); err != nil { + return err } - err := policy.Validate() + opt.tags = maps.Clone(tags) + return nil + } +} + +// WithActivityRetryPolicy snapshots policy when this option is created. Later +// caller mutations do not affect activity retries. +func WithActivityRetryPolicy(policy *RetryPolicy) CallActivityOption { + if policy == nil { + return func(*callActivityOptions, api.DataConverter) error { return nil } + } + snapshot := *policy + return func(opt *callActivityOptions, _ api.DataConverter) error { + normalized, err := snapshot.Normalized() if err != nil { return err } - opt.retryPolicy = policy + opt.retryPolicy = &normalized return nil } } @@ -102,25 +160,32 @@ type activityContext struct { TaskID int32 Name string - rawInput []byte - ctx context.Context + rawInput []byte + ctx context.Context + converter api.DataConverter } // Activity is the functional interface for activity implementations. type Activity func(ctx ActivityContext) (any, error) -func newTaskActivityContext(ctx context.Context, taskID int32, ts *protos.TaskScheduledEvent) *activityContext { +func newTaskActivityContext( + ctx context.Context, + taskID int32, + ts *protos.TaskScheduledEvent, + converter api.DataConverter, +) *activityContext { return &activityContext{ - TaskID: taskID, - Name: ts.Name, - rawInput: []byte(ts.Input.GetValue()), - ctx: ctx, + TaskID: taskID, + Name: ts.Name, + rawInput: []byte(ts.Input.GetValue()), + ctx: ctx, + converter: converter, } } // GetInput unmarshals the serialized activity input and saves the result into [v]. func (actx *activityContext) GetInput(v any) error { - return unmarshalData(actx.rawInput, v) + return unmarshalData(actx.converter, actx.rawInput, v) } func (actx *activityContext) Context() context.Context { diff --git a/task/cancel.go b/task/cancel.go new file mode 100644 index 00000000..1afbb377 --- /dev/null +++ b/task/cancel.go @@ -0,0 +1,104 @@ +package task + +type cancellationScope struct { + parent *cancellationScope + parentIndex int + children []*cancellationScope + tasks []*completableTask + coroutines []*coroutine + waiters map[*coroutine]struct{} + canceled bool +} + +func newCancellationScope(parent *cancellationScope) *cancellationScope { + scope := &cancellationScope{parent: parent, parentIndex: -1, canceled: parent.isCanceled()} + if parent != nil && !scope.canceled { + scope.parentIndex = len(parent.children) + parent.children = append(parent.children, scope) + } + return scope +} + +// isCanceled reports whether the scope has been canceled. A nil scope is +// treated as never canceled, so callers don't need a separate nil check. +func (s *cancellationScope) isCanceled() bool { + return s != nil && s.canceled +} + +func (s *cancellationScope) addTask(task *completableTask) { + task.scopeIndex = len(s.tasks) + s.tasks = append(s.tasks, task) + if s.canceled { + task.cancel() + } +} + +func (s *cancellationScope) removeTask(task *completableTask) { + if s == nil || task.scopeIndex < 0 || task.scopeIndex >= len(s.tasks) { + return + } + index := task.scopeIndex + last := len(s.tasks) - 1 + if index != last { + replacement := s.tasks[last] + s.tasks[index] = replacement + replacement.scopeIndex = index + } + s.tasks[last] = nil + s.tasks = s.tasks[:last] + task.scopeIndex = -1 +} + +func (s *cancellationScope) addCoroutine(c *coroutine) { + s.coroutines = append(s.coroutines, c) +} + +func (s *cancellationScope) addWaiter(c *coroutine) { + if s == nil { + return + } + if s.waiters == nil { + s.waiters = make(map[*coroutine]struct{}) + } + s.waiters[c] = struct{}{} +} + +func (s *cancellationScope) removeWaiter(c *coroutine) { + if s != nil { + delete(s.waiters, c) + } +} + +func (s *cancellationScope) cancel(scheduler *coroutineScheduler) { + if s.canceled { + return + } + s.canceled = true + for _, child := range s.children { + child.cancel(scheduler) + } + tasks := s.tasks + s.tasks = nil + for _, task := range tasks { + task.scopeIndex = -1 + task.cancel() + } + for _, c := range s.coroutines { + scheduler.makeRunnable(c) + } + for c := range s.waiters { + scheduler.makeRunnable(c) + } + clear(s.waiters) + if s.parent != nil && !s.parent.canceled && s.parentIndex >= 0 { + last := len(s.parent.children) - 1 + if s.parentIndex != last { + replacement := s.parent.children[last] + s.parent.children[s.parentIndex] = replacement + replacement.parentIndex = s.parentIndex + } + s.parent.children[last] = nil + s.parent.children = s.parent.children[:last] + s.parentIndex = -1 + } +} diff --git a/task/cancel_test.go b/task/cancel_test.go new file mode 100644 index 00000000..cba212d8 --- /dev/null +++ b/task/cancel_test.go @@ -0,0 +1,597 @@ +package task + +import ( + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestCanceledScopeNewChildDoesNotScheduleWork(t *testing.T) { + entityID := api.NewEntityID("counter", "canceled") + tests := []struct { + name string + call func(*OrchestrationContext) error + }{ + {"activity", func(ctx *OrchestrationContext) error { return ctx.CallActivity("unused").Await(nil) }}, + {"sub-orchestration", func(ctx *OrchestrationContext) error { return ctx.CallSubOrchestrator("unused").Await(nil) }}, + {"timer", func(ctx *OrchestrationContext) error { return ctx.CreateTimer(time.Hour).Await(nil) }}, + {"entity call", func(ctx *OrchestrationContext) error { return ctx.CallEntity(entityID, "get").Await(nil) }}, + {"entity signal", func(ctx *OrchestrationContext) error { return ctx.SignalEntity(entityID, "add") }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("late-child", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + cancel() + if err := child.WaitForSingleEvent("never", -1).Await(nil); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("parent cancellation was not applied") + } + grandchild, _ := child.WithCancel() + return errors.Is(test.call(grandchild), ErrTaskCanceled), nil + })) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("late-child", "instance", nil, nil, nil, nil), + } + for _, replay := range []bool{false, true} { + var oldEvents, newEvents []*protos.HistoryEvent + if replay { + oldEvents = events + } else { + newEvents = events + } + response := executeOrchestrationTurn(t, registry, "instance", oldEvents, newEvents) + require.Len(t, response.Actions, 1) + require.NotNil(t, response.Actions[0].GetCompleteOrchestration(), "replay=%t", replay) + require.Equal(t, "true", completionResult(t, response), "replay=%t", replay) + } + }) + } +} + +func TestCancellationScopeChildPreservesSchedulerBoundary(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("pending-cancel", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + cancel() + grandchild, _ := child.WithCancel() + if grandchild.scope.isCanceled() { + return nil, errors.New("pending cancellation was applied before the scheduler boundary") + } + task := grandchild.CreateTimer(time.Hour) + if err := task.Await(nil); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("descendant did not receive cancellation") + } + derivedCount := len(ctx.derived) + lateChild, _ := grandchild.WithCancel() + return lateChild.scope.isCanceled() && len(grandchild.scope.children) == 0 && len(ctx.derived) == derivedCount, nil + })) + response := executeOrchestrationTurn(t, registry, "instance", nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("pending-cancel", "instance", nil, nil, nil, nil), + }) + require.Len(t, response.Actions, 2) + require.NotNil(t, response.Actions[0].GetCreateTimer()) + require.Equal(t, "true", completionResult(t, response)) +} + +func TestCancelScopeCancelsTaskButKeepsScheduledAction(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-task", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + timer := child.CreateTimer(time.Hour) + cancel() + return errors.Is(timer.Await(nil), ErrTaskCanceled), nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("cancel-task-instance") + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-task", string(instanceID), nil, nil, nil, nil), + }, + ) + if len(firstTurn.Actions) != 2 { + t.Fatalf("first-turn action count = %d, want timer and completion", len(firstTurn.Actions)) + } + if firstTurn.Actions[0].GetCreateTimer() == nil { + t.Fatal("cancel removed an already-scheduled timer action") + } + if got, want := completionResult(t, firstTurn), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } + + oldEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-task", string(instanceID), nil, nil, nil, nil), + helpers.NewTimerCreatedEvent(0, firstTurn.Actions[0].GetCreateTimer().GetFireAt()), + } + secondTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + oldEvents, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTimerFiredEvent(0, firstTurn.Actions[0].GetCreateTimer().GetFireAt(), nil), + }, + ) + if len(secondTurn.Actions) != 1 || secondTurn.Actions[0].GetCompleteOrchestration() == nil { + t.Fatalf("late timer completion produced unexpected actions: %v", secondTurn.Actions) + } +} + +func TestCancelScopeCancelsNestedScopes(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("nested-cancel", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + grandchild, _ := child.WithCancel() + timer := grandchild.CreateTimer(time.Hour) + cancel() + return errors.Is(timer.Await(nil), ErrTaskCanceled), nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("nested-cancel-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("nested-cancel", string(instanceID), nil, nil, nil, nil), + }, + ) + if got, want := completionResult(t, result), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCancelScopeUnblocksChildCoroutine(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-coroutine", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + wg := ctx.NewWaitGroup() + wg.Add(2) + canceled := false + + child.Go(func(child *OrchestrationContext) { + defer wg.Done() + canceled = errors.Is(child.CreateTimer(time.Hour).Await(nil), ErrTaskCanceled) + }) + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + if err := ctx.CreateTimer(time.Second).Await(nil); err != nil { + panic(err) + } + cancel() + }) + + wg.Wait(ctx) + return canceled, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("cancel-coroutine-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("cancel-coroutine", string(instanceID), nil, nil, nil, nil) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + if len(firstTurn.Actions) != 2 { + t.Fatalf("first-turn action count = %d, want two timers", len(firstTurn.Actions)) + } + + shortTimer := firstTurn.Actions[1].GetCreateTimer() + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, firstTurn.Actions[0].GetCreateTimer().GetFireAt()), + helpers.NewTimerCreatedEvent(1, shortTimer.GetFireAt()), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTimerFiredEvent(1, shortTimer.GetFireAt(), nil), + }, + ) + if got, want := completionResult(t, result), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCancelScopeCompletionOrderIsDeterministic(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-order", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + first := child.CreateTimer(time.Hour) + second := child.CreateTimer(2 * time.Hour) + cancel() + if ctx.WhenAny(first, second) == first { + return "first", nil + } + return "second", nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("cancel-order-instance") + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-order", string(instanceID), nil, nil, nil, nil), + } + for i := 0; i < 200; i++ { + result := executeOrchestrationTurn(t, registry, instanceID, nil, events) + if got, want := completionResult(t, result), `"first"`; got != want { + t.Fatalf("iteration %d result = %s, want %s", i, got, want) + } + } +} + +func TestCanceledChildWaitingOnRootWaitGroupUnwinds(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-waitgroup", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + blocked := ctx.NewWaitGroup() + blocked.Add(1) + completed := ctx.NewWaitGroup() + completed.Add(2) + + child.Go(func(child *OrchestrationContext) { + defer completed.Done() + blocked.Wait(child) + }) + ctx.Go(func(*OrchestrationContext) { + defer completed.Done() + cancel() + }) + completed.Wait(ctx) + return true, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("cancel-waitgroup-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-waitgroup", string(instanceID), nil, nil, nil, nil), + }, + ) + if got, want := completionResult(t, result), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCanceledScopeDoesNotConsumeBufferedEvent(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-buffer", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + completed := ctx.NewWaitGroup() + completed.Add(2) + childCanceled := false + + child.Go(func(child *OrchestrationContext) { + defer completed.Done() + _ = child.CreateTimer(time.Hour).Await(nil) + var ignored string + childCanceled = errors.Is( + child.WaitForSingleEvent("payload", -1).Await(&ignored), + ErrTaskCanceled, + ) + }) + ctx.Go(func(ctx *OrchestrationContext) { + defer completed.Done() + if err := ctx.CreateTimer(time.Second).Await(nil); err != nil { + panic(err) + } + cancel() + }) + + completed.Wait(ctx) + var payload string + if err := ctx.WaitForSingleEvent("payload", 0).Await(&payload); err != nil { + return nil, err + } + return []any{childCanceled, payload}, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("cancel-buffer-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("cancel-buffer", string(instanceID), nil, nil, nil, nil) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + shortTimer := firstTurn.Actions[1].GetCreateTimer() + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, firstTurn.Actions[0].GetCreateTimer().GetFireAt()), + helpers.NewTimerCreatedEvent(1, shortTimer.GetFireAt()), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("payload", wrapperspb.String(`"value"`)), + helpers.NewTimerFiredEvent(1, shortTimer.GetFireAt(), nil), + }, + ) + if got, want := completionResult(t, result), `[true,"value"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCanceledSelectRemovesEventSubscription(t *testing.T) { + var captured *OrchestrationContext + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-select", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + completed := ctx.NewWaitGroup() + completed.Add(2) + child.Go(func(child *OrchestrationContext) { + defer completed.Done() + child.Select(OnEvent(NewEventChannel[int](child, "event"), nil)) + }) + ctx.Go(func(*OrchestrationContext) { + defer completed.Done() + cancel() + }) + completed.Wait(ctx) + return nil, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("cancel-select-instance") + executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-select", string(instanceID), nil, nil, nil, nil), + }, + ) + if len(captured.eventWaiters) != 0 { + t.Fatalf("event subscriptions remain after cancellation: %v", captured.eventWaiters) + } +} + +func TestCanceledPendingEventWaiterDoesNotConsumeEvent(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-pending-event", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + completed := ctx.NewWaitGroup() + completed.Add(2) + + child.Go(func(child *OrchestrationContext) { + defer completed.Done() + _ = child.WaitForSingleEvent("payload", -1).Await(nil) + }) + ctx.Go(func(ctx *OrchestrationContext) { + defer completed.Done() + if err := ctx.CreateTimer(time.Second).Await(nil); err != nil { + panic(err) + } + cancel() + }) + + completed.Wait(ctx) + var payload string + if err := ctx.WaitForSingleEvent("payload", -1).Await(&payload); err != nil { + return nil, err + } + return payload, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("cancel-pending-event-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent( + "cancel-pending-event", + string(instanceID), + nil, + nil, + nil, + nil, + ) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + timer := firstTurn.Actions[0].GetCreateTimer() + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, timer.GetFireAt()), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTimerFiredEvent(0, timer.GetFireAt(), nil), + helpers.NewEventRaisedEvent("payload", wrapperspb.String(`"value"`)), + }, + ) + if got, want := completionResult(t, result), `"value"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestAlreadyCanceledSelectDoesNotWaitOrConsume(t *testing.T) { + for _, scope := range []string{"current", "supplied", "both"} { + for _, buffered := range []bool{false, true} { + t.Run(scope+map[bool]string{false: "/empty", true: "/buffered"}[buffered], func(t *testing.T) { + var root, child *OrchestrationContext + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("canceled-select", func(ctx *OrchestrationContext) (any, error) { + root = ctx + channel := NewEventChannel[int](ctx, "payload") + if err := ctx.WaitForSingleEvent("ready", -1).Await(nil); err != nil { + return nil, err + } + var cancel func() + child, cancel = ctx.WithCancel() + runner, selector := child, child + switch scope { + case "current": + selector = ctx + case "supplied": + runner = ctx + } + done := ctx.NewWaitGroup() + done.Add(1) + canceled, invoked := false, false + runner.Go(func(*OrchestrationContext) { + defer done.Done() + defer func() { canceled = isTaskCanceled(recover()) }() + selector.Select(OnEvent(channel, func(int) { invoked = true })) + }) + cancel() + done.Wait(ctx) + value, received, err := channel.TryReceiveErr() + _, again, nextErr := channel.TryReceiveErr() + return canceled && !invoked && received == buffered && + (!buffered || value == 42) && err == nil && !again && nextErr == nil, nil + })) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("canceled-select", "instance", nil, nil, nil, nil), + } + if buffered { + events = append(events, helpers.NewEventRaisedEvent("payload", wrapperspb.String("42"))) + } + events = append(events, helpers.NewEventRaisedEvent("ready", nil)) + for _, replay := range []bool{false, true} { + var oldEvents, newEvents []*protos.HistoryEvent + if replay { + oldEvents = events + } else { + newEvents = events + } + response := executeOrchestrationTurn(t, registry, "instance", oldEvents, newEvents) + require.Equal(t, "true", completionResult(t, response), "replay=%t", replay) + require.Empty(t, root.eventWaiters) + require.Empty(t, root.scope.waiters) + require.Empty(t, child.scope.waiters) + } + }) + } + } +} + +func TestSelectPreservesPendingCancellationAndRootTaskObservation(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("select-cancellation-boundary", func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "payload") + if err := ctx.WaitForSingleEvent("ready", -1).Await(nil); err != nil { + return nil, err + } + child, cancel := ctx.WithCancel() + canceledTask := child.WaitForSingleEvent("never", -1) + cancel() + value := 0 + child.Select(OnEvent(channel, func(received int) { value = received })) + if err := canceledTask.Await(nil); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("cancellation was not applied") + } + observed := false + ctx.Select(OnTask(canceledTask, func(task Task) { + observed = errors.Is(task.Await(nil), ErrTaskCanceled) + })) + return value == 42 && observed, nil + })) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("select-cancellation-boundary", "instance", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("payload", wrapperspb.String("42")), + helpers.NewEventRaisedEvent("ready", nil), + } + require.Equal(t, "true", completionResult(t, executeOrchestrationTurn(t, registry, "instance", nil, events))) + require.Equal(t, "true", completionResult(t, executeOrchestrationTurn(t, registry, "instance", events, nil))) +} + +func TestCanceledEventChannelReceivePreservesBufferedEvent(t *testing.T) { + for _, method := range []string{"Receive", "ReceiveErr"} { + for _, payload := range []string{"42", `"not an integer"`} { + t.Run(method+"/"+payload, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("canceled-receive", func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "payload") + if err := ctx.WaitForSingleEvent("ready", -1).Await(nil); err != nil { + return nil, err + } + child, cancel := ctx.WithCancel() + cancel() + if err := child.WaitForSingleEvent("never", -1).Await(nil); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("cancellation was not applied") + } + canceled := false + if method == "ReceiveErr" { + value, err := channel.ReceiveErr(child) + canceled = value == 0 && errors.Is(err, ErrTaskCanceled) + } else { + func() { + defer func() { canceled = isTaskCanceled(recover()) }() + channel.Receive(child) + }() + } + value, received, err := channel.TryReceiveErr() + preserved := received && ((payload == "42" && value == 42 && err == nil) || + (payload != "42" && err != nil)) + _, again, nextErr := channel.TryReceiveErr() + return canceled && preserved && !again && nextErr == nil, nil + })) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("canceled-receive", "instance", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("payload", wrapperspb.String(payload)), + helpers.NewEventRaisedEvent("ready", nil), + } + require.Equal(t, "true", completionResult(t, executeOrchestrationTurn(t, registry, "instance", nil, events))) + require.Equal(t, "true", completionResult(t, executeOrchestrationTurn(t, registry, "instance", events, nil))) + }) + } + } +} diff --git a/task/context.go b/task/context.go new file mode 100644 index 00000000..c472e498 --- /dev/null +++ b/task/context.go @@ -0,0 +1,113 @@ +package task + +import ( + "context" + "log/slog" + "maps" + + "github.com/microsoft/durabletask-go/api" +) + +type taskLoggerKey struct{} + +// mergeStringMaps returns base overlaid with overrides, or nil when both are +// empty. The result never aliases either input. +func mergeStringMaps[M ~map[string]string](base, overrides M) M { + if len(base) == 0 && len(overrides) == 0 { + return nil + } + merged := make(M, len(base)+len(overrides)) + maps.Copy(merged, base) + maps.Copy(merged, overrides) + return merged +} + +// Context returns a replay-stable Go context containing only persisted +// orchestration identity and caller fields. Host values, deadlines, and +// cancellation are intentionally excluded from deterministic orchestrator code. +func (ctx *OrchestrationContext) Context() context.Context { + engine := ctx.engineContext() + base := api.ContextWithFields(context.Background(), engine.contextFields) + return api.WithOrchestrationContextInfo(base, api.OrchestrationContextInfo{ + InstanceID: engine.ID, + Name: engine.Name, + Version: engine.Version, + ParentInstanceID: engine.parentInstanceID, + }) +} + +// Logger returns a slog logger that suppresses output while replaying history. +func (ctx *OrchestrationContext) Logger() *slog.Logger { + engine := ctx.engineContext() + logger := engine.logger + if logger == nil { + logger = slog.Default() + } + handler := &replaySafeHandler{ + handler: logger.Handler(), + replaying: func() bool { + return engine.IsReplaying + }, + } + return slog.New(handler).With( + slog.String("durabletask.instance_id", string(engine.ID)), + slog.String("durabletask.orchestration.name", engine.Name), + slog.String("durabletask.orchestration.version", engine.Version), + ) +} + +// LoggerFromContext returns the task logger associated with an activity or +// entity context, or slog.Default. Orchestrators must use +// [OrchestrationContext.Logger]; their replay-stable Context excludes loggers. +func LoggerFromContext(ctx context.Context) *slog.Logger { + if logger, ok := ctx.Value(taskLoggerKey{}).(*slog.Logger); ok && logger != nil { + return logger + } + return slog.Default() +} + +func withActivityLogger(ctx context.Context, logger *slog.Logger) context.Context { + if logger == nil { + logger = slog.Default() + } + attrs := make([]any, 0, 10) + if orchestration, ok := api.OrchestrationContextInfoFromContext(ctx); ok { + attrs = append(attrs, + slog.String("durabletask.instance_id", string(orchestration.InstanceID)), + slog.String("durabletask.orchestration.name", orchestration.Name), + slog.String("durabletask.orchestration.version", orchestration.Version), + ) + } + if activity, ok := api.ActivityContextInfoFromContext(ctx); ok { + attrs = append(attrs, + slog.String("durabletask.activity.name", activity.Name), + slog.String("durabletask.activity.version", activity.Version), + slog.Int("durabletask.activity.task_id", int(activity.TaskID)), + ) + } + return context.WithValue(ctx, taskLoggerKey{}, logger.With(attrs...)) +} + +type replaySafeHandler struct { + handler slog.Handler + replaying func() bool +} + +func (h *replaySafeHandler) Enabled(ctx context.Context, level slog.Level) bool { + return !h.replaying() && h.handler.Enabled(ctx, level) +} + +func (h *replaySafeHandler) Handle(ctx context.Context, record slog.Record) error { + if h.replaying() { + return nil + } + return h.handler.Handle(ctx, record) +} + +func (h *replaySafeHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + return &replaySafeHandler{handler: h.handler.WithAttrs(attrs), replaying: h.replaying} +} + +func (h *replaySafeHandler) WithGroup(name string) slog.Handler { + return &replaySafeHandler{handler: h.handler.WithGroup(name), replaying: h.replaying} +} diff --git a/task/context_test.go b/task/context_test.go new file mode 100644 index 00000000..84381cca --- /dev/null +++ b/task/context_test.go @@ -0,0 +1,272 @@ +package task + +import ( + "bytes" + "context" + "encoding/json" + "log/slog" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/contextprop" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestReplaySafeLoggerSuppressesReplayOutput(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewTextHandler(&output, nil)) + ctx := newTestOrchestrationContext(NewTaskRegistry(), "logging", nil, nil) + ctx.Name = "logger-test" + ctx.Version = "v1" + ctx.logger = logger + + ctx.IsReplaying = true + ctx.Logger().Info("replayed") + ctx.IsReplaying = false + ctx.Logger().Info("live") + + if strings.Contains(output.String(), "replayed") { + t.Fatalf("replay log was emitted: %s", output.String()) + } + if !strings.Contains(output.String(), "live") { + t.Fatalf("live log was not emitted: %s", output.String()) + } +} + +func TestOrchestrationContextPropagatesOnlyPersistedIdentityAndFields(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("context-test", func(ctx *OrchestrationContext) (any, error) { + info, ok := api.OrchestrationContextInfoFromContext(ctx.Context()) + if !ok { + return nil, nil + } + return struct { + Info api.OrchestrationContextInfo + Fields api.ContextFields + }{ + Info: info, + Fields: api.ContextFieldsFromContext(ctx.Context()), + }, nil + }); err != nil { + t.Fatal(err) + } + + fields := api.ContextFields{"tenant": "alpha"} + instanceID := api.InstanceID("context-instance") + parent := helpers.NewParentInfo(4, "parent", "parent-instance") + started := helpers.NewExecutionStartedEvent( + "context-test", + string(instanceID), + nil, + parent, + nil, + nil, + wrapperspb.String("v2"), + ) + started.GetExecutionStarted().Tags = contextprop.Encode(api.OrchestrationContextInfo{}, fields) + fields["tenant"] = "mutated" + events := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), started} + + run := func(workerField string) string { + executor := NewTaskExecutor(registry, WithContextFields(api.ContextFields{"worker": workerField})) + result, err := executor.ExecuteOrchestrator(context.Background(), instanceID, nil, events, nil) + if err != nil { + t.Fatal(err) + } + return completionAction(t, result.Response).GetResult().GetValue() + } + first, second := run("first-worker"), run("second-worker") + if first != second { + t.Fatalf("orchestration output changed across worker context: first=%s second=%s", first, second) + } + + var output struct { + Info api.OrchestrationContextInfo + Fields api.ContextFields + } + if err := json.Unmarshal([]byte(first), &output); err != nil { + t.Fatal(err) + } + if output.Info.InstanceID != instanceID || + output.Info.Name != "context-test" || + output.Info.Version != "v2" || + output.Info.ParentInstanceID != "parent-instance" { + t.Fatalf("unexpected orchestration info: %+v", output.Info) + } + if output.Fields["tenant"] != "alpha" || output.Fields["worker"] != "" { + t.Fatalf("orchestration fields are not purely persisted: %#v", output.Fields) + } +} + +func TestActivityContextPropagatesIdentityFieldsAndLogger(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddActivityN("inspect", func(ctx ActivityContext) (any, error) { + orchestration, _ := api.OrchestrationContextInfoFromContext(ctx.Context()) + activity, _ := api.ActivityContextInfoFromContext(ctx.Context()) + LoggerFromContext(ctx.Context()).Info("activity log") + return struct { + Orchestration api.OrchestrationContextInfo + Activity api.ActivityContextInfo + Fields api.ContextFields + }{ + Orchestration: orchestration, + Activity: activity, + Fields: api.ContextFieldsFromContext(ctx.Context()), + }, nil + }); err != nil { + t.Fatal(err) + } + + var logs bytes.Buffer + executor := NewTaskExecutor( + registry, + WithLogger(slog.New(slog.NewTextHandler(&logs, nil))), + WithContextFields(api.ContextFields{"feature": "enabled"}), + ) + instanceID := api.InstanceID("activity-context") + base := api.WithOrchestrationContextInfo(context.Background(), api.OrchestrationContextInfo{ + InstanceID: instanceID, + Name: "parent", + Version: "v3", + }) + response, err := executor.ExecuteActivity( + base, + instanceID, + helpers.NewTaskScheduledEvent(9, "inspect", wrapperspb.String("a1"), nil, nil), + ) + if err != nil { + t.Fatal(err) + } + + var output struct { + Orchestration api.OrchestrationContextInfo + Activity api.ActivityContextInfo + Fields api.ContextFields + } + if err := json.Unmarshal([]byte(response.GetTaskCompleted().GetResult().GetValue()), &output); err != nil { + t.Fatal(err) + } + if output.Orchestration.Name != "parent" || output.Orchestration.Version != "v3" { + t.Fatalf("unexpected orchestration context: %+v", output.Orchestration) + } + if output.Activity.Name != "inspect" || output.Activity.Version != "a1" || output.Activity.TaskID != 9 { + t.Fatalf("unexpected activity context: %+v", output.Activity) + } + if output.Fields["feature"] != "enabled" { + t.Fatalf("context field = %q, want enabled", output.Fields["feature"]) + } + if !strings.Contains(logs.String(), "activity log") { + t.Fatalf("activity logger output missing: %s", logs.String()) + } +} + +func TestActivityContextDecodesDurableContextTags(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddActivityN("inspect-tags", func(ctx ActivityContext) (any, error) { + orchestration, _ := api.OrchestrationContextInfoFromContext(ctx.Context()) + return struct { + Orchestration api.OrchestrationContextInfo + Fields api.ContextFields + }{ + Orchestration: orchestration, + Fields: api.ContextFieldsFromContext(ctx.Context()), + }, nil + }); err != nil { + t.Fatal(err) + } + + event := helpers.NewTaskScheduledEvent(4, "inspect-tags", nil, nil, nil) + event.GetTaskScheduled().Tags = contextprop.Encode(api.OrchestrationContextInfo{ + InstanceID: "tagged-instance", + Name: "tagged-parent", + Version: "v4", + ParentInstanceID: "root", + }, api.ContextFields{"tenant": "tagged"}) + response, err := NewTaskExecutor(registry).ExecuteActivity( + context.Background(), + "tagged-instance", + event, + ) + if err != nil { + t.Fatal(err) + } + + var output struct { + Orchestration api.OrchestrationContextInfo + Fields api.ContextFields + } + if err := json.Unmarshal([]byte(response.GetTaskCompleted().GetResult().GetValue()), &output); err != nil { + t.Fatal(err) + } + if output.Orchestration.Name != "tagged-parent" || + output.Orchestration.Version != "v4" || + output.Orchestration.ParentInstanceID != "root" { + t.Fatalf("unexpected orchestration identity: %+v", output.Orchestration) + } + if output.Fields["tenant"] != "tagged" { + t.Fatalf("tenant = %q, want tagged", output.Fields["tenant"]) + } +} + +type hostContextKey struct{} + +func TestOrchestrationContextExcludesHostContextState(t *testing.T) { + type contextState struct { + HasHostValue bool + HasDeadline bool + HasDone bool + HasInfo bool + InstanceID api.InstanceID + HasLogger bool + } + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("isolated-context", func(ctx *OrchestrationContext) (any, error) { + deterministic := ctx.Context() + info, hasInfo := api.OrchestrationContextInfoFromContext(deterministic) + _, hasDeadline := deterministic.Deadline() + return contextState{ + HasHostValue: deterministic.Value(hostContextKey{}) != nil, + HasDeadline: hasDeadline, + HasDone: deterministic.Done() != nil, + HasInfo: hasInfo, + InstanceID: info.InstanceID, + HasLogger: LoggerFromContext(deterministic) != slog.Default(), + }, nil + }); err != nil { + t.Fatal(err) + } + + host, cancel := context.WithTimeout( + context.WithValue(context.Background(), hostContextKey{}, "host-only"), + time.Hour, + ) + defer cancel() + instanceID := api.InstanceID("isolated-context-instance") + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + host, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("isolated-context", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + + var output contextState + if err := json.Unmarshal([]byte(completionAction(t, result.Response).GetResult().GetValue()), &output); err != nil { + t.Fatal(err) + } + if output.HasHostValue || output.HasDeadline || output.HasDone || output.HasLogger { + t.Fatalf("host context leaked into orchestrator context: %+v", output) + } + if !output.HasInfo || output.InstanceID != instanceID { + t.Fatalf("persisted orchestration identity missing: %+v", output) + } +} diff --git a/task/converter_test.go b/task/converter_test.go new file mode 100644 index 00000000..b284f850 --- /dev/null +++ b/task/converter_test.go @@ -0,0 +1,195 @@ +package task + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/gob" + "fmt" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type gobDataConverter struct{} + +func (gobDataConverter) Serialize(value any) (string, error) { + var buffer bytes.Buffer + if err := gob.NewEncoder(&buffer).Encode(value); err != nil { + return "", err + } + return base64.RawStdEncoding.EncodeToString(buffer.Bytes()), nil +} + +func (gobDataConverter) Deserialize(payload string, target any) error { + data, err := base64.RawStdEncoding.DecodeString(payload) + if err != nil { + return err + } + return gob.NewDecoder(bytes.NewReader(data)).Decode(target) +} + +type converterPayload struct { + Value int +} + +func TestDataConverterCoversOrchestrationActivityEventsAndContinueAsNew(t *testing.T) { + converter := gobDataConverter{} + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("converter", func(ctx *OrchestrationContext) (any, error) { + var input converterPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if input.Value == 1 { + if err := ctx.SetCustomStatusValue(converterPayload{Value: 2}); err != nil { + return nil, err + } + ctx.CallActivity("activity", WithActivityInput(converterPayload{Value: 3})) + return converterPayload{Value: 4}, nil + } + ctx.ContinueAsNew(converterPayload{Value: input.Value + 1}) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("event", func(ctx *OrchestrationContext) (any, error) { + var payload converterPayload + if err := ctx.WaitForSingleEvent("signal", -1).Await(&payload); err != nil { + return nil, err + } + return payload, nil + })) + + executor := NewTaskExecutor(registry, WithDataConverter(converter)) + input := mustConvert(t, converter, converterPayload{Value: 1}) + result, err := executor.ExecuteOrchestrator( + context.Background(), + "converter-instance", + nil, + []*protos.HistoryEvent{helpers.NewExecutionStartedEvent( + "converter", + "converter-instance", + wrapperspb.String(input), + nil, + nil, + nil, + )}, nil) + + require.NoError(t, err) + requireConvertedValue(t, converter, result.Response.CustomStatus.GetValue(), 2) + for _, action := range result.Response.Actions { + switch { + case action.GetScheduleTask() != nil: + requireConvertedValue(t, converter, action.GetScheduleTask().GetInput().GetValue(), 3) + case action.GetCompleteOrchestration() != nil: + requireConvertedValue(t, converter, action.GetCompleteOrchestration().GetResult().GetValue(), 4) + } + } + + continueInput := mustConvert(t, converter, converterPayload{Value: 10}) + result, err = executor.ExecuteOrchestrator( + context.Background(), + "continue-instance", + nil, + []*protos.HistoryEvent{helpers.NewExecutionStartedEvent( + "converter", + "continue-instance", + wrapperspb.String(continueInput), + nil, + nil, + nil, + )}, nil) + + require.NoError(t, err) + requireConvertedValue(t, converter, completionAction(t, result.Response).GetResult().GetValue(), 11) + + eventPayload := mustConvert(t, converter, converterPayload{Value: 20}) + result, err = executor.ExecuteOrchestrator( + context.Background(), + "event-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("event", "event-instance", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(eventPayload)), + }, nil) + + require.NoError(t, err) + requireConvertedValue(t, converter, completionAction(t, result.Response).GetResult().GetValue(), 20) +} + +func TestDataConverterCoversActivityAndEntityExecution(t *testing.T) { + converter := gobDataConverter{} + registry := NewTaskRegistry() + require.NoError(t, registry.AddActivityN("activity", func(ctx ActivityContext) (any, error) { + var input converterPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return converterPayload{Value: input.Value + 1}, nil + })) + require.NoError(t, registry.AddEntityN("counter", func(ctx *EntityContext) (any, error) { + var state converterPayload + if ctx.HasState() { + if err := ctx.GetState(&state); err != nil { + return nil, err + } + } + var input converterPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + state.Value += input.Value + if err := ctx.SetState(state); err != nil { + return nil, err + } + return state, nil + })) + executor := NewTaskExecutor(registry, WithDataConverter(converter)) + + activity, err := executor.ExecuteActivity( + context.Background(), + "instance", + helpers.NewTaskScheduledEvent( + 1, + "activity", + nil, + wrapperspb.String(mustConvert(t, converter, converterPayload{Value: 4})), + nil, + ), + ) + require.NoError(t, err) + requireConvertedValue(t, converter, activity.GetTaskCompleted().GetResult().GetValue(), 5) + + entity, err := executor.(backendEntityExecutor).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: api.NewEntityID("counter", "one").String(), + EntityState: wrapperspb.String(mustConvert(t, converter, converterPayload{Value: 5})), + Operations: []*protos.OperationRequest{{ + Operation: "add", + Input: wrapperspb.String(mustConvert(t, converter, converterPayload{Value: 7})), + }}, + }) + require.NoError(t, err) + requireConvertedValue(t, converter, entity.EntityState.GetValue(), 12) + requireConvertedValue(t, converter, entity.Results[0].GetSuccess().GetResult().GetValue(), 12) +} + +type backendEntityExecutor interface { + ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) +} + +func mustConvert(t *testing.T, converter api.DataConverter, value any) string { + t.Helper() + payload, err := converter.Serialize(value) + require.NoError(t, err) + return payload +} + +func requireConvertedValue(t *testing.T, converter api.DataConverter, payload string, expected int) { + t.Helper() + var value converterPayload + require.NoError(t, converter.Deserialize(payload, &value), fmt.Sprintf("payload %q", payload)) + require.Equal(t, expected, value.Value) +} diff --git a/task/coroutine.go b/task/coroutine.go new file mode 100644 index 00000000..dacc9b87 --- /dev/null +++ b/task/coroutine.go @@ -0,0 +1,121 @@ +package task + +import ( + "errors" + "fmt" + "runtime/debug" + "sync" + + "github.com/microsoft/durabletask-go/api" +) + +type coroutineState uint8 + +const ( + coroutineRunnable coroutineState = iota + coroutineRunning + coroutineWaiting + coroutineCompleted +) + +type coroutineSignalKind uint8 + +const ( + coroutineYielded coroutineSignalKind = iota + coroutineFinished + coroutineCanceled + coroutinePanicked +) + +type coroutineSignal struct { + kind coroutineSignalKind + panicValue any + panicStack []byte +} + +type coroutine struct { + id uint64 + scheduler *coroutineScheduler + ctx *OrchestrationContext + fn func() + resume chan struct{} + signals chan coroutineSignal + stop chan struct{} + exited chan struct{} + stopOnce sync.Once + state coroutineState + scope *cancellationScope +} + +func (c *coroutine) run() { + defer close(c.exited) + defer func() { + if value := recover(); value != nil { + if c.scheduler.isStopping() && isTaskBlocked(value) { + return + } + if c.scope.isCanceled() && isTaskCanceled(value) { + c.sendSignal(coroutineSignal{kind: coroutineCanceled}) + return + } + c.sendSignal(coroutineSignal{ + kind: coroutinePanicked, + panicValue: value, + panicStack: debug.Stack(), + }) + } + }() + + select { + case <-c.resume: + case <-c.stop: + return + } + + c.fn() + c.sendSignal(coroutineSignal{kind: coroutineFinished}) +} + +func (c *coroutine) yield() { + c.sendSignal(coroutineSignal{kind: coroutineYielded}) + select { + case <-c.resume: + case <-c.stop: + panic(ErrTaskBlocked) + } +} + +func (c *coroutine) sendSignal(signal coroutineSignal) { + select { + case c.signals <- signal: + case <-c.stop: + } +} + +func (c *coroutine) exit() { + c.stopOnce.Do(func() { + close(c.stop) + }) + <-c.exited + c.state = coroutineCompleted +} + +func isTaskBlocked(value any) bool { + err, ok := value.(error) + return ok && errors.Is(err, ErrTaskBlocked) +} + +func isTaskCanceled(value any) bool { + err, ok := value.(error) + return ok && errors.Is(err, ErrTaskCanceled) +} + +func coroutinePanicError(id uint64, value any, stack []byte) error { + var message string + if err, ok := value.(error); ok { + message = fmt.Sprintf("coroutine %d panicked: %v", id, err) + return newPanicFailureError(api.ErrorTypeOrchestratorPanic, message, string(stack), err) + } + message = fmt.Sprintf("coroutine %d panicked: %v", id, value) + return newPanicFailureError(api.ErrorTypeOrchestratorPanic, message, string(stack), nil) +} diff --git a/task/custom_status_test.go b/task/custom_status_test.go new file mode 100644 index 00000000..058b1b42 --- /dev/null +++ b/task/custom_status_test.go @@ -0,0 +1,195 @@ +package task + +import ( + "context" + "encoding/json" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +// SetCustomStatus stores an already-serialized status verbatim. The DTS worker +// forwards OrchestratorResponse.CustomStatus straight to the scheduler, so a +// caller that pre-serializes must not get a second round of serialization. +func TestSetCustomStatusStoresRawStringVerbatim(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN( + "raw-status", + func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus(`{"stage":"halfway"}`) + return nil, nil + }, + )) + + results, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "raw-status-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("raw-status", "raw-status-instance", nil, nil, nil, nil), + }, nil) + + require.NoError(t, err) + require.Equal(t, `{"stage":"halfway"}`, results.Response.GetCustomStatus().GetValue()) +} + +// SetCustomStatusValue applies the configured converter, unlike SetCustomStatus. +func TestSetCustomStatusValueSerializesWithConverter(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN( + "typed-status", + func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.SetCustomStatusValue(map[string]string{"stage": "halfway"}) + }, + )) + + results, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "typed-status-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("typed-status", "typed-status-instance", nil, nil, nil, nil), + }, nil) + + require.NoError(t, err) + require.JSONEq(t, `{"stage":"halfway"}`, results.Response.GetCustomStatus().GetValue()) +} + +// WithSubOrchestratorInput serializes through the configured converter, while +// WithRawSubOrchestratorInput passes the string through untouched. Both end up +// on the CreateSubOrchestration action the worker sends to DTS. +func TestSubOrchestratorInputOptionsProduceExpectedWireInput(t *testing.T) { + type payload struct { + Value int `json:"value"` + } + + for _, test := range []struct { + name string + configure func(*OrchestrationContext) + wantInput string + }{ + { + name: "typed input is serialized", + configure: func(ctx *OrchestrationContext) { + ctx.CallSubOrchestrator("child", WithSubOrchestratorInput(payload{Value: 42})) + }, + wantInput: `{"value":42}`, + }, + { + name: "raw input is passed through", + configure: func(ctx *OrchestrationContext) { + ctx.CallSubOrchestrator("child", WithRawSubOrchestratorInput(`{"value":7}`)) + }, + wantInput: `{"value":7}`, + }, + } { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN( + "parent", + func(ctx *OrchestrationContext) (any, error) { + test.configure(ctx) + return nil, nil + }, + )) + + results, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "parent-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("parent", "parent-instance", nil, nil, nil, nil), + }, nil) + + require.NoError(t, err) + created := createSubOrchestrationAction(results.Response.Actions) + require.NotNil(t, created) + require.Equal(t, "child", created.GetName()) + require.JSONEq(t, test.wantInput, created.GetInput().GetValue()) + }) + } +} + +// A sub-orchestrator input that cannot be serialized must surface as an error on +// the returned task rather than silently scheduling an empty child. +func TestWithSubOrchestratorInputReportsSerializationFailure(t *testing.T) { + registry := NewTaskRegistry() + var awaitErr error + require.NoError(t, registry.AddOrchestratorN( + "parent", + func(ctx *OrchestrationContext) (any, error) { + awaitErr = ctx.CallSubOrchestrator("child", WithSubOrchestratorInput(make(chan int))).Await(nil) + return nil, nil + }, + )) + + results, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "parent-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("parent", "parent-instance", nil, nil, nil, nil), + }, nil) + + require.NoError(t, err) + require.Error(t, awaitErr) + require.ErrorContains(t, awaitErr, "failed to serialize input") + require.Nil(t, createSubOrchestrationAction(results.Response.Actions)) +} + +func createSubOrchestrationAction( + actions []*protos.OrchestratorAction, +) *protos.CreateSubOrchestrationAction { + for _, action := range actions { + if created := action.GetCreateSubOrchestration(); created != nil { + return created + } + } + return nil +} + +// Guards the documented difference between the two custom status setters: the +// raw setter must not double-encode a value the typed setter would encode once. +func TestRawAndTypedCustomStatusDifferForStringValues(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN( + "status", + func(ctx *OrchestrationContext) (any, error) { + ctx.SetCustomStatus("plain") + return nil, nil + }, + )) + require.NoError(t, registry.AddOrchestratorN( + "typed", + func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.SetCustomStatusValue("plain") + }, + )) + + executor := NewTaskExecutor(registry) + statusOf := func(name string) string { + results, err := executor.ExecuteOrchestrator( + context.Background(), + api.InstanceID(name+"-instance"), + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(name, name+"-instance", nil, nil, nil, nil), + }, nil) + + require.NoError(t, err) + return results.Response.GetCustomStatus().GetValue() + } + + require.Equal(t, "plain", statusOf("status")) + encoded, err := json.Marshal("plain") + require.NoError(t, err) + require.Equal(t, string(encoded), statusOf("typed")) +} diff --git a/task/doc.go b/task/doc.go new file mode 100644 index 00000000..3795c6ac --- /dev/null +++ b/task/doc.go @@ -0,0 +1,3 @@ +// Package task implements deterministic orchestrator, activity, and durable +// entity execution for Durable Task Scheduler workers. +package task diff --git a/task/entity.go b/task/entity.go new file mode 100644 index 00000000..ca3aaae4 --- /dev/null +++ b/task/entity.go @@ -0,0 +1,442 @@ +package task + +import ( + "context" + "encoding/hex" + "fmt" + "log/slog" + "reflect" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// Entity is the functional interface for entity implementations. +// An entity function receives an EntityContext and returns a result and error. +type Entity func(ctx *EntityContext) (any, error) + +// EntityFactoryContext identifies the entity batch for which an implementation is created. +type EntityFactoryContext struct { + Context context.Context + ID api.EntityID +} + +// EntityBatch contains one batch-scoped entity implementation and optional cleanup. +type EntityBatch struct { + Entity Entity + Close func(context.Context) error +} + +// EntityBatchCloser can release object resources after an entity batch completes. +type EntityBatchCloser interface { + CloseEntityBatch(context.Context) error +} + +// EntityFactory creates one entity implementation for an operation batch. +// Factories may run concurrently for different entity batches. +type EntityFactory func(EntityFactoryContext) (EntityBatch, error) + +// EntityContext provides the execution context for an entity operation. +type EntityContext struct { + ID api.EntityID + Operation string + RequestID string + IsSignal bool + + rawInput entityPayload + state entityState + stateDirty bool + actions []*protos.OperationAction + actionIDSeq int32 + currentTime time.Time + ctx context.Context + logger *slog.Logger + converter api.DataConverter + parentTrace *protos.TraceContext +} + +type entityPayload struct { + value []byte + present bool +} + +type entityState struct { + value []byte + hasValue bool +} + +// HasInput reports whether the caller supplied an operation input. +func (ctx *EntityContext) HasInput() bool { + return ctx.rawInput.present +} + +// GetInput unmarshals the serialized entity operation input and saves the result into [v]. +func (ctx *EntityContext) GetInput(v any) error { + if !ctx.HasInput() { + return fmt.Errorf("entity operation has no input") + } + if v == nil { + return nil + } + return api.NormalizeDataConverter(ctx.converter).Deserialize(string(ctx.rawInput.value), v) +} + +// SerializeInput serializes a value using the entity's configured data converter. +// It is useful when an entity needs to persist input for a later operation. +func (ctx *EntityContext) SerializeInput(value any) (string, error) { + payload, err := marshalData(ctx.converter, value) + if err != nil { + return "", err + } + return string(payload), nil +} + +// GetRawState returns the entity's pre-serialized state and whether state is set. +// Callers are responsible for deserializing the returned value. +func (ctx *EntityContext) GetRawState() (string, bool) { + return string(ctx.state.value), ctx.state.hasValue +} + +// SetRawState stores pre-serialized entity state. Callers are responsible for +// producing data accepted by all readers of the entity state. +func (ctx *EntityContext) SetRawState(state string) { + if state == "" { + ctx.DeleteState() + return + } + ctx.state.value = []byte(state) + ctx.state.hasValue = true + ctx.stateDirty = true +} + +// HasState returns true if the entity has state set. +func (ctx *EntityContext) HasState() bool { + return ctx.state.hasValue +} + +// GetState unmarshals the entity state and saves the result into [v]. +func (ctx *EntityContext) GetState(v any) error { + if !ctx.state.hasValue { + return fmt.Errorf("entity has no state") + } + if v == nil { + return nil + } + return api.NormalizeDataConverter(ctx.converter).Deserialize(string(ctx.state.value), v) +} + +// SetState serializes and stores entity state with the configured converter. +// Passing nil deletes the entity state. +func (ctx *EntityContext) SetState(state any) error { + if isNilEntityValue(state) { + ctx.DeleteState() + return nil + } + payload, err := marshalData(ctx.converter, state) + if err != nil { + return fmt.Errorf("failed to serialize entity state: %w", err) + } + ctx.state.value = payload + ctx.state.hasValue = true + ctx.stateDirty = true + return nil +} + +// DeleteState removes the entity state. +func (ctx *EntityContext) DeleteState() { + ctx.stateDirty = true + ctx.state.value = nil + ctx.state.hasValue = false +} + +// Context returns the Go context for the current entity operation. +func (ctx *EntityContext) Context() context.Context { + if ctx.ctx == nil { + return context.Background() + } + return ctx.ctx +} + +// Logger returns a logger enriched with entity operation identity. +func (ctx *EntityContext) Logger() *slog.Logger { + if ctx.logger == nil { + ctx.logger = slog.Default() + } + return ctx.logger +} + +// CurrentTimeUTC returns the durable timestamp associated with this operation. +func (ctx *EntityContext) CurrentTimeUTC() time.Time { + return ctx.currentTime +} + +// SignalEntity sends a fire-and-forget signal to another entity. +func (ctx *EntityContext) SignalEntity(entityID api.EntityID, operationName string, input any) error { + return ctx.signalEntity(entityID, operationName, input, time.Time{}) +} + +// SignalEntityAt schedules a fire-and-forget signal to another entity. +func (ctx *EntityContext) SignalEntityAt(entityID api.EntityID, scheduledTime time.Time, operationName string, input any) error { + if scheduledTime.IsZero() { + return fmt.Errorf("scheduled entity signal time must not be zero") + } + return ctx.signalEntity(entityID, operationName, input, scheduledTime) +} + +func (ctx *EntityContext) signalEntity(entityID api.EntityID, operationName string, input any, scheduledTime time.Time) error { + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return err + } + if operationName == "" { + return fmt.Errorf("entity operation name must not be empty") + } + + rawInput, err := marshalEntityInput(ctx.converter, input) + if err != nil { + return fmt.Errorf("failed to serialize signal input: %w", err) + } + + action := &protos.OperationAction{ + Id: ctx.nextActionID(), + OperationActionType: &protos.OperationAction_SendSignal{ + SendSignal: &protos.SendSignalAction{ + InstanceId: entityID.String(), + Name: operationName, + Input: rawInput, + RequestTime: timestampOrNil(ctx.currentTime), + ScheduledTime: timestampOrNil(scheduledTime), + ParentTraceContext: helpers.CloneTraceContext(ctx.parentTrace), + }, + }, + } + ctx.actions = append(ctx.actions, action) + return nil +} + +// StartNewOrchestration schedules a new orchestration from within an entity operation. +func (ctx *EntityContext) StartNewOrchestration(name string, opts ...EntityStartOrchestrationOption) error { + if name == "" { + return fmt.Errorf("orchestration name must not be empty") + } + options := &entityStartOrchestrationOptions{} + for _, configure := range opts { + if err := configure(options, ctx.converter); err != nil { + return err + } + } + if options.instanceID == "" { + seed := fmt.Sprintf("%s|%s|%d|start", ctx.ID.String(), ctx.RequestID, ctx.actionIDSeq) + id := uuid.NewSHA1(entityActionNamespace, []byte(seed)) + options.instanceID = hex.EncodeToString(id[:]) + } else if err := helpers.ValidateOrchestrationInstanceID(options.instanceID); err != nil { + return err + } + + action := &protos.OperationAction{ + Id: ctx.nextActionID(), + OperationActionType: &protos.OperationAction_StartNewOrchestration{ + StartNewOrchestration: &protos.StartNewOrchestrationAction{ + InstanceId: options.instanceID, + Name: name, + Version: options.version, + Input: options.rawInput, + ScheduledTime: timestampOrNil(options.scheduledTime), + RequestTime: timestampOrNil(ctx.currentTime), + ParentTraceContext: helpers.CloneTraceContext(ctx.parentTrace), + }, + }, + } + ctx.actions = append(ctx.actions, action) + return nil +} + +func (ctx *EntityContext) nextActionID() int32 { + id := ctx.actionIDSeq + ctx.actionIDSeq++ + return id +} + +// entityStartOrchestrationOptions holds options for starting orchestrations from entities. +type entityStartOrchestrationOptions struct { + instanceID string + version *wrapperspb.StringValue + rawInput *wrapperspb.StringValue + scheduledTime time.Time +} + +// EntityStartOrchestrationOption is a functional option for StartNewOrchestration. +type EntityStartOrchestrationOption func(*entityStartOrchestrationOptions, api.DataConverter) error + +// WithEntityStartOrchestrationInput sets the input for the new orchestration. +func WithEntityStartOrchestrationInput(input any) EntityStartOrchestrationOption { + return func(opts *entityStartOrchestrationOptions, converter api.DataConverter) error { + rawInput, err := marshalEntityInput(converter, input) + if err != nil { + return fmt.Errorf("failed to serialize orchestration input: %w", err) + } + opts.rawInput = rawInput + return nil + } +} + +// WithRawEntityStartOrchestrationInput sets pre-serialized input for the new +// orchestration. Callers are responsible for ensuring the payload is accepted +// by the target orchestration's data converter. +func WithRawEntityStartOrchestrationInput(input string) EntityStartOrchestrationOption { + return func(opts *entityStartOrchestrationOptions, _ api.DataConverter) error { + opts.rawInput = wrapperspb.String(input) + return nil + } +} + +// WithEntityStartOrchestrationInstanceID sets the instance ID for the new orchestration. +func WithEntityStartOrchestrationInstanceID(instanceID string) EntityStartOrchestrationOption { + return func(opts *entityStartOrchestrationOptions, _ api.DataConverter) error { + opts.instanceID = instanceID + return nil + } +} + +// WithEntityStartOrchestrationVersion sets the version for the new orchestration. +func WithEntityStartOrchestrationVersion(version string) EntityStartOrchestrationOption { + return func(opts *entityStartOrchestrationOptions, _ api.DataConverter) error { + opts.version = wrapperspb.String(version) + return nil + } +} + +// WithEntityStartOrchestrationScheduledTime schedules the new orchestration. +func WithEntityStartOrchestrationScheduledTime(scheduledTime time.Time) EntityStartOrchestrationOption { + return func(opts *entityStartOrchestrationOptions, _ api.DataConverter) error { + if scheduledTime.IsZero() { + return fmt.Errorf("scheduled orchestration time must not be zero") + } + opts.scheduledTime = scheduledTime + return nil + } +} + +// callEntityOption is a functional option type for the CallEntity orchestrator method. +type callEntityOption func(*callEntityOptions, api.DataConverter) error + +type callEntityOptions struct { + rawInput *wrapperspb.StringValue +} + +// WithEntityInput configures an input for an entity operation invocation. +func WithEntityInput(input any) callEntityOption { + return func(opt *callEntityOptions, converter api.DataConverter) error { + rawInput, err := marshalEntityInput(converter, input) + if err != nil { + return err + } + opt.rawInput = rawInput + return nil + } +} + +// WithRawEntityInput configures a raw input for an entity operation invocation. +func WithRawEntityInput(input string) callEntityOption { + return func(opt *callEntityOptions, _ api.DataConverter) error { + opt.rawInput = wrapperspb.String(input) + return nil + } +} + +// signalEntityOption is a functional option type for the SignalEntity orchestrator method. +type signalEntityOption func(*signalEntityOptions, api.DataConverter) error + +type signalEntityOptions struct { + rawInput *wrapperspb.StringValue + scheduledTime *timestamppb.Timestamp +} + +// WithSignalEntityInput configures an input for a signal entity invocation. +func WithSignalEntityInput(input any) signalEntityOption { + return func(opt *signalEntityOptions, converter api.DataConverter) error { + rawInput, err := marshalEntityInput(converter, input) + if err != nil { + return err + } + opt.rawInput = rawInput + return nil + } +} + +// OptionalEntityInput binds an entity method input that may be omitted by the caller. +type OptionalEntityInput[T any] struct { + Value T + Present bool +} + +// Or returns the input value when present, or defaultValue when absent. +func (input OptionalEntityInput[T]) Or(defaultValue T) T { + if input.Present { + return input.Value + } + return defaultValue +} + +type optionalEntityInputBinder interface { + bindEntityInput(*EntityContext) error +} + +func (input *OptionalEntityInput[T]) bindEntityInput(ctx *EntityContext) error { + input.Present = ctx.HasInput() + if !input.Present { + return nil + } + return ctx.GetInput(&input.Value) +} + +func marshalEntityInput(converter api.DataConverter, input any) (*wrapperspb.StringValue, error) { + if isNilEntityValue(input) { + return nil, nil + } + payload, err := marshalData(converter, input) + if err != nil { + return nil, err + } + return wrapperspb.String(string(payload)), nil +} + +func isNilEntityValue(value any) bool { + if value == nil { + return true + } + reflected := reflect.ValueOf(value) + return isNilable(reflected.Kind()) && reflected.IsNil() +} + +// WithRawSignalEntityInput configures a raw input for a signal entity invocation. +func WithRawSignalEntityInput(input string) signalEntityOption { + return func(opt *signalEntityOptions, _ api.DataConverter) error { + opt.rawInput = wrapperspb.String(input) + return nil + } +} + +// WithSignalEntityScheduledTime configures a scheduled time for an entity signal. +func WithSignalEntityScheduledTime(scheduledTime time.Time) signalEntityOption { + return func(opt *signalEntityOptions, _ api.DataConverter) error { + if scheduledTime.IsZero() { + return fmt.Errorf("scheduled entity signal time must not be zero") + } + opt.scheduledTime = timestamppb.New(scheduledTime) + return nil + } +} + +var entityActionNamespace = uuid.MustParse("bd4e8d71-45f8-5f6d-a302-b9a785e665c6") + +func timestampOrNil(value time.Time) *timestamppb.Timestamp { + if value.IsZero() { + return nil + } + return timestamppb.New(value) +} diff --git a/task/entity_dispatch.go b/task/entity_dispatch.go new file mode 100644 index 00000000..45b73452 --- /dev/null +++ b/task/entity_dispatch.go @@ -0,0 +1,249 @@ +package task + +import ( + "fmt" + "reflect" + "strings" + "sync" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" +) + +var ( + entityContextType = reflect.TypeFor[*EntityContext]() + entityIDType = reflect.TypeFor[api.EntityID]() + errorType = reflect.TypeFor[error]() +) + +type entityParameterKind uint8 + +const ( + entityContextParameter entityParameterKind = iota + entityIDParameter + entityInputParameter + optionalEntityInputParameter +) + +type entityParameterBinding struct { + kind entityParameterKind + parameterType reflect.Type +} + +type entityMethodBinding struct { + method reflect.Method + parameters []entityParameterBinding + numOut int + returnsError bool +} + +type entityMethodBindingResult struct { + binding *entityMethodBinding + err error +} + +type entityMethodSet struct { + methods map[string]reflect.Method + bindings sync.Map +} + +func (set *entityMethodSet) call(ctx *EntityContext, receiver reflect.Value) (any, bool, error) { + operation := helpers.ToLowerInvariant(ctx.Operation) + method, found := set.methods[operation] + if !found { + return nil, false, nil + } + + cached, ok := set.bindings.Load(operation) + if !ok { + binding, err := bindEntityMethod(ctx.Operation, method) + cached, _ = set.bindings.LoadOrStore(operation, entityMethodBindingResult{ + binding: binding, + err: err, + }) + } + result := cached.(entityMethodBindingResult) + if result.err != nil { + return nil, true, result.err + } + + output, err := result.binding.call(ctx, receiver) + return output, true, err +} + +// NewEntityFor creates an entity function that dispatches operations to methods +// on a converter-serializable state struct of type S. +// +// Supported method parameters are an optional *EntityContext, an optional +// api.EntityID, and at most one required input value or OptionalEntityInput, +// in any order. Supported return shapes are (), (result), (error), and +// (result, error). +func NewEntityFor[S any]() Entity { + stateType := reflect.TypeFor[S]() + if stateType.Kind() == reflect.Pointer { + panic("NewEntityFor does not support pointer state types") + } + + methods := make(map[string]reflect.Method) + pointerType := reflect.PointerTo(stateType) + for i := 0; i < pointerType.NumMethod(); i++ { + method := pointerType.Method(i) + name := helpers.ToLowerInvariant(method.Name) + if existing, ok := methods[name]; ok { + panic(fmt.Sprintf( + "NewEntityFor found case-insensitive operation collision between %s and %s", + existing.Name, + method.Name, + )) + } + methods[name] = method + } + + methodSet := entityMethodSet{methods: methods} + return func(ctx *EntityContext) (any, error) { + var state S + if ctx.HasState() { + if err := ctx.GetState(&state); err != nil { + return nil, fmt.Errorf("failed to deserialize entity state: %w", err) + } + } + + output, found, err := methodSet.call(ctx, reflect.ValueOf(&state)) + if !found { + if strings.EqualFold(ctx.Operation, "delete") { + ctx.DeleteState() + return nil, nil + } + return nil, fmt.Errorf("entity does not support operation %q", ctx.Operation) + } + if err != nil { + return nil, err + } + if !ctx.stateDirty { + if err := ctx.SetState(state); err != nil { + return nil, fmt.Errorf("failed to save entity state: %w", err) + } + } + return output, nil + } +} + +func bindEntityMethod(operation string, method reflect.Method) (*entityMethodBinding, error) { + methodType := method.Type + if methodType.IsVariadic() { + return nil, fmt.Errorf("entity operation %q must not be variadic", operation) + } + + binding := &entityMethodBinding{ + method: method, + parameters: make([]entityParameterBinding, 0, methodType.NumIn()-1), + numOut: methodType.NumOut(), + } + var sawContext, sawEntityID, sawInput bool + for i := 1; i < methodType.NumIn(); i++ { + parameterType := methodType.In(i) + parameter := entityParameterBinding{parameterType: parameterType} + switch parameterType { + case entityContextType: + if sawContext { + return nil, fmt.Errorf("entity operation %q accepts *EntityContext more than once", operation) + } + sawContext = true + parameter.kind = entityContextParameter + case entityIDType: + if sawEntityID { + return nil, fmt.Errorf("entity operation %q accepts api.EntityID more than once", operation) + } + sawEntityID = true + parameter.kind = entityIDParameter + default: + if sawInput { + return nil, fmt.Errorf("entity operation %q accepts more than one input parameter", operation) + } + sawInput = true + parameter.kind = entityInputParameter + if _, ok := reflect.New(parameterType).Interface().(optionalEntityInputBinder); ok { + parameter.kind = optionalEntityInputParameter + } + } + binding.parameters = append(binding.parameters, parameter) + } + + binding.returnsError = binding.numOut > 0 && methodType.Out(binding.numOut-1).Implements(errorType) + switch binding.numOut { + case 0, 1: + case 2: + if !binding.returnsError { + return nil, fmt.Errorf("entity operation %q second return value must implement error", operation) + } + default: + return nil, fmt.Errorf("entity operation %q has unsupported return count %d", operation, binding.numOut) + } + return binding, nil +} + +func (binding *entityMethodBinding) call(ctx *EntityContext, receiver reflect.Value) (any, error) { + args := make([]reflect.Value, 1, len(binding.parameters)+1) + args[0] = receiver + for _, parameter := range binding.parameters { + switch parameter.kind { + case entityContextParameter: + args = append(args, reflect.ValueOf(ctx)) + case entityIDParameter: + args = append(args, reflect.ValueOf(ctx.ID)) + case optionalEntityInputParameter: + input := reflect.New(parameter.parameterType) + if err := input.Interface().(optionalEntityInputBinder).bindEntityInput(ctx); err != nil { + return nil, fmt.Errorf("failed to deserialize input for operation %q: %w", ctx.Operation, err) + } + args = append(args, input.Elem()) + default: + if !ctx.HasInput() { + return nil, fmt.Errorf( + "failed to bind input for operation %q: the operation expected an input value, but none was provided", + ctx.Operation, + ) + } + input := reflect.New(parameter.parameterType) + if err := ctx.GetInput(input.Interface()); err != nil { + return nil, fmt.Errorf("failed to deserialize input for operation %q: %w", ctx.Operation, err) + } + args = append(args, input.Elem()) + } + } + + results := binding.method.Func.Call(args) + switch { + case binding.numOut == 0: + return nil, nil + case binding.numOut == 1 && binding.returnsError: + return nil, reflectError(results[0]) + case binding.numOut == 1: + return reflectResult(results[0]), nil + default: + return reflectResult(results[0]), reflectError(results[1]) + } +} + +func reflectResult(value reflect.Value) any { + if isNilable(value.Kind()) && value.IsNil() { + return nil + } + return value.Interface() +} + +func reflectError(value reflect.Value) error { + if isNilable(value.Kind()) && value.IsNil() { + return nil + } + return value.Interface().(error) +} + +func isNilable(kind reflect.Kind) bool { + switch kind { + case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Pointer, reflect.Slice: + return true + default: + return false + } +} diff --git a/task/entity_dispatch_test.go b/task/entity_dispatch_test.go new file mode 100644 index 00000000..94eafe59 --- /dev/null +++ b/task/entity_dispatch_test.go @@ -0,0 +1,707 @@ +package task + +import ( + "fmt" + "sync" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type testCounter struct { + Value int `json:"value"` +} + +func Test_EntityDispatcher_ConcurrentFirstUse(t *testing.T) { + entity := NewEntityFor[testCounter]() + var wait sync.WaitGroup + failures := make(chan error, 32) + for i := 0; i < 32; i++ { + wait.Add(1) + go func() { + defer wait.Done() + result, err := entity(&EntityContext{ + ID: api.NewEntityID("counter", "key"), + Operation: "Add", + rawInput: presentEntityPayload("1"), + }) + if err != nil { + failures <- err + } else if result != 1 { + failures <- fmt.Errorf("result = %v, want 1", result) + } + }() + } + wait.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } +} + +func (c *testCounter) Add(amount int) (any, error) { + c.Value += amount + return c.Value, nil +} + +func (c *testCounter) Get() (any, error) { + return c.Value, nil +} + +func (c *testCounter) Reset() (any, error) { + c.Value = 0 + return nil, nil +} + +func Test_EntityDispatcher_BasicOperations(t *testing.T) { + entity := NewEntityFor[testCounter]() + + // Test "Add" operation + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "Add", + rawInput: presentEntityPayload("5"), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 5, result) + assert.True(t, ctx.HasState()) + + // Verify state was saved + var state testCounter + require.NoError(t, ctx.GetState(&state)) + assert.Equal(t, 5, state.Value) +} + +func Test_EntityDispatcher_CaseInsensitive(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "add", // lowercase + rawInput: presentEntityPayload("10"), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 10, result) +} + +func Test_EntityDispatcher_WithExistingState(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "Add", + rawInput: presentEntityPayload("3"), + state: entityState{value: []byte(`{"value":7}`), hasValue: true}, + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 10, result) +} + +func Test_EntityDispatcher_Get(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "Get", + state: entityState{value: []byte(`{"value":42}`), hasValue: true}, + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 42, result) +} + +func Test_EntityDispatcher_Reset(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "Reset", + state: entityState{value: []byte(`{"value":42}`), hasValue: true}, + } + _, err := entity(ctx) + require.NoError(t, err) + + var state testCounter + require.NoError(t, ctx.GetState(&state)) + assert.Equal(t, 0, state.Value) +} + +func Test_EntityDispatcher_ImplicitDelete(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "delete", + state: entityState{value: []byte(`{"value":42}`), hasValue: true}, + } + _, err := entity(ctx) + require.NoError(t, err) + assert.False(t, ctx.HasState()) +} + +func Test_EntityDispatcher_UnknownOperation(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "test"), + Operation: "unknown", + } + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support operation") +} + +// Test with EntityContext parameter +type contextAwareEntity struct { + LastOp string `json:"lastOp"` +} + +func (e *contextAwareEntity) Info(ctx *EntityContext) (any, error) { + e.LastOp = ctx.Operation + return ctx.ID.String(), nil +} + +func Test_EntityDispatcher_WithEntityContext(t *testing.T) { + entity := NewEntityFor[contextAwareEntity]() + + ctx := &EntityContext{ + ID: api.NewEntityID("myentity", "key1"), + Operation: "Info", + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, "@myentity@key1", result) + + var state contextAwareEntity + require.NoError(t, ctx.GetState(&state)) + assert.Equal(t, "Info", state.LastOp) +} + +// OperationNotSupported_Fails: tests rejection of non-existent methods +func Test_EntityDispatcher_OperationNotSupported(t *testing.T) { + entity := NewEntityFor[testCounter]() + + tests := []struct { + name string + op string + }{ + {"non-existent method", "doesNotExist"}, + {"special chars", "add!"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "k"), + Operation: tt.op, + } + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "does not support operation") + }) + } +} + +// Add_Success with case-insensitive matching (lowercase, uppercase, mixed) +func Test_EntityDispatcher_CaseInsensitiveMethodMatching(t *testing.T) { + entity := NewEntityFor[testCounter]() + + cases := []string{"add", "Add", "ADD", "aDd"} + for _, op := range cases { + t.Run(op, func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "k"), + Operation: op, + rawInput: presentEntityPayload("7"), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 7, result) + }) + } +} + +// Get_Success: state retrieval from existing state +func Test_EntityDispatcher_GetFromExistingState(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "k"), + Operation: "Get", + state: entityState{value: []byte(`{"value":42}`), hasValue: true}, + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 42, result) +} + +func Test_EntityDispatcher_MissingRequiredInputFails(t *testing.T) { + entity := NewEntityFor[testCounter]() + + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "k"), + Operation: "Add", + } + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "expected an input value") +} + +// ImplicitDelete_ClearsState: default delete operation clears state +func Test_EntityDispatcher_ImplicitDeleteClearsState(t *testing.T) { + entity := NewEntityFor[testCounter]() + + // "delete" and "Delete" both work (case-insensitive) + for _, op := range []string{"delete", "Delete", "DELETE"} { + t.Run(op, func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "k"), + Operation: op, + state: entityState{value: []byte(`{"value":42}`), hasValue: true}, + } + _, err := entity(ctx) + require.NoError(t, err) + assert.False(t, ctx.HasState(), "state should be cleared after delete") + }) + } +} + +// ExplicitDelete_Overridden: custom delete method takes precedence +type entityWithDelete struct { + Value int `json:"value"` + Deleted bool `json:"deleted"` +} + +func (e *entityWithDelete) Delete() (any, error) { + e.Deleted = true + return "custom delete", nil +} + +func (e *entityWithDelete) Get() (any, error) { + return e.Value, nil +} + +func Test_EntityDispatcher_ExplicitDeleteOverridesImplicit(t *testing.T) { + entity := NewEntityFor[entityWithDelete]() + + ctx := &EntityContext{ + ID: api.NewEntityID("e", "k"), + Operation: "delete", + state: entityState{value: []byte(`{"value":42,"deleted":false}`), hasValue: true}, + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, "custom delete", result) + // State should still exist (not implicitly cleared) since custom Delete ran + assert.True(t, ctx.HasState()) + + var state entityWithDelete + require.NoError(t, ctx.GetState(&state)) + assert.True(t, state.Deleted) + assert.Equal(t, 42, state.Value) +} + +type entityWithContextDelete struct { + Value int `json:"value"` + Deleted bool `json:"deleted"` +} + +func (e *entityWithContextDelete) Delete(ctx *EntityContext) (any, error) { + e.Deleted = true + return "deleted via ctx", ctx.SetState(nil) +} + +func Test_EntityDispatcher_ExplicitContextDeleteCanClearState(t *testing.T) { + entity := NewEntityFor[entityWithContextDelete]() + + ctx := &EntityContext{ + ID: api.NewEntityID("e", "k"), + Operation: "delete", + state: entityState{value: []byte(`{"value":42,"deleted":false}`), hasValue: true}, + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, "deleted via ctx", result) + assert.False(t, ctx.HasState()) +} + +// Throws_ExceptionPreserved: error propagation from entity methods +type errorEntity struct{} + +func (e *errorEntity) Fail() (any, error) { + return nil, fmt.Errorf("entity operation failed: %w", assert.AnError) +} + +func (e *errorEntity) Get() (any, error) { + return "ok", nil +} + +func Test_EntityDispatcher_ErrorPreserved(t *testing.T) { + entity := NewEntityFor[errorEntity]() + + ctx := &EntityContext{ + ID: api.NewEntityID("e", "k"), + Operation: "Fail", + } + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "entity operation failed") + assert.ErrorIs(t, err, assert.AnError) +} + +// State machine pattern: complex state transitions (ported from ExportJobTests) +type jobState struct { + Status string `json:"status"` + Name string `json:"name"` +} + +func (j *jobState) Create(name string) (any, error) { + if j.Status != "" { + return nil, fmt.Errorf("job already exists with status %s", j.Status) + } + j.Status = "active" + j.Name = name + return j.Status, nil +} + +func (j *jobState) Complete() (any, error) { + if j.Status != "active" { + return nil, fmt.Errorf("invalid transition: cannot complete job with status %s", j.Status) + } + j.Status = "completed" + return j.Status, nil +} + +func (j *jobState) Fail() (any, error) { + if j.Status != "active" { + return nil, fmt.Errorf("invalid transition: cannot fail job with status %s", j.Status) + } + j.Status = "failed" + return j.Status, nil +} + +func (j *jobState) Get() (any, error) { + return *j, nil +} + +func Test_EntityDispatcher_StateMachine(t *testing.T) { + entity := NewEntityFor[jobState]() + + t.Run("create then complete", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("job", "1"), Operation: "Create", + rawInput: presentEntityPayload(`"myJob"`), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, "active", result) + + ctx2 := &EntityContext{ + ID: api.NewEntityID("job", "1"), Operation: "Complete", + state: ctx.state, + } + result2, err := entity(ctx2) + require.NoError(t, err) + assert.Equal(t, "completed", result2) + }) + + t.Run("complete without create fails", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("job", "2"), Operation: "Complete", + } + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid transition") + }) + + t.Run("create twice fails", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("job", "3"), Operation: "Create", + rawInput: presentEntityPayload(`"first"`), + } + _, err := entity(ctx) + require.NoError(t, err) + + ctx2 := &EntityContext{ + ID: api.NewEntityID("job", "3"), Operation: "Create", + rawInput: presentEntityPayload(`"second"`), state: ctx.state, + } + _, err = entity(ctx2) + require.Error(t, err) + assert.Contains(t, err.Error(), "already exists") + }) + + t.Run("create then fail then recreate", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("job", "4"), Operation: "Create", + rawInput: presentEntityPayload(`"v1"`), + } + _, err := entity(ctx) + require.NoError(t, err) + + ctx2 := &EntityContext{ + ID: api.NewEntityID("job", "4"), Operation: "Fail", + state: ctx.state, + } + _, err = entity(ctx2) + require.NoError(t, err) + + // After failure, delete state and re-create + ctx3 := &EntityContext{ + ID: api.NewEntityID("job", "4"), Operation: "delete", + state: ctx2.state, + } + _, err = entity(ctx3) + require.NoError(t, err) + assert.False(t, ctx3.HasState()) + + ctx4 := &EntityContext{ + ID: api.NewEntityID("job", "4"), Operation: "Create", + rawInput: presentEntityPayload(`"v2"`), + } + result, err := entity(ctx4) + require.NoError(t, err) + assert.Equal(t, "active", result) + }) +} + +// Multiple return type support +type multiReturnEntity struct{} + +func (e *multiReturnEntity) NoReturn() { + // void method +} + +func (e *multiReturnEntity) ErrorOnly() error { + return nil +} + +func (e *multiReturnEntity) ErrorOnlyFail() error { + return fmt.Errorf("error only") +} + +func (e *multiReturnEntity) ResultOnly() any { + return 42 +} + +func Test_EntityDispatcher_ReturnTypeVariations(t *testing.T) { + entity := NewEntityFor[multiReturnEntity]() + + t.Run("void return", func(t *testing.T) { + ctx := &EntityContext{ID: api.NewEntityID("e", "k"), Operation: "NoReturn"} + result, err := entity(ctx) + require.NoError(t, err) + assert.Nil(t, result) + }) + + t.Run("error-only success", func(t *testing.T) { + ctx := &EntityContext{ID: api.NewEntityID("e", "k"), Operation: "ErrorOnly"} + result, err := entity(ctx) + require.NoError(t, err) + assert.Nil(t, result) + }) + + t.Run("error-only failure", func(t *testing.T) { + ctx := &EntityContext{ID: api.NewEntityID("e", "k"), Operation: "ErrorOnlyFail"} + _, err := entity(ctx) + require.Error(t, err) + assert.Contains(t, err.Error(), "error only") + }) + + t.Run("result-only", func(t *testing.T) { + ctx := &EntityContext{ID: api.NewEntityID("e", "k"), Operation: "ResultOnly"} + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 42, result) + }) +} + +// Entity with context + input parameter binding +type fullBindingEntity struct { + Log []string `json:"log"` +} + +func (e *fullBindingEntity) Process(ctx *EntityContext, msg string) (any, error) { + entry := fmt.Sprintf("%s:%s:%s", ctx.ID.String(), ctx.Operation, msg) + e.Log = append(e.Log, entry) + return len(e.Log), nil +} + +func Test_EntityDispatcher_ContextAndInputBinding(t *testing.T) { + entity := NewEntityFor[fullBindingEntity]() + + ctx := &EntityContext{ + ID: api.NewEntityID("logger", "main"), Operation: "Process", + rawInput: presentEntityPayload(`"hello world"`), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 1, result) + + var state fullBindingEntity + require.NoError(t, ctx.GetState(&state)) + require.Len(t, state.Log, 1) + assert.Equal(t, "@logger@main:Process:hello world", state.Log[0]) +} + +type explicitStateEntity struct { + Value int `json:"value"` + Mode string `json:"mode,omitempty"` +} + +func (e *explicitStateEntity) Replace(ctx *EntityContext, value int) (any, error) { + e.Value = value + e.Mode = "receiver" + return value, ctx.SetState(explicitStateEntity{Value: value * 2, Mode: "context"}) +} + +func Test_EntityDispatcher_ExplicitContextStateWins(t *testing.T) { + entity := NewEntityFor[explicitStateEntity]() + + ctx := &EntityContext{ + ID: api.NewEntityID("explicit", "k"), + Operation: "Replace", + rawInput: presentEntityPayload("5"), + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 5, result) + + var state explicitStateEntity + require.NoError(t, ctx.GetState(&state)) + assert.Equal(t, explicitStateEntity{Value: 10, Mode: "context"}, state) +} + +func Test_EntityDispatcher_RejectsPointerStateType(t *testing.T) { + assert.PanicsWithValue(t, "NewEntityFor does not support pointer state types", func() { + _ = NewEntityFor[*testCounter]() + }) +} + +func Test_EntityDispatcher_ZeroValueInitialization(t *testing.T) { + entity := NewEntityFor[testCounter]() + + // No initial state — should start with zero-value testCounter{Value: 0} + ctx := &EntityContext{ + ID: api.NewEntityID("counter", "new"), Operation: "Get", + } + result, err := entity(ctx) + require.NoError(t, err) + assert.Equal(t, 0, result) // zero-value int +} + +type invalidSignatureEntity struct{} + +func (*invalidSignatureEntity) TooManyInputs(int, string) {} + +func (*invalidSignatureEntity) WrongSecondReturn() (int, string) { + return 0, "" +} + +func (*invalidSignatureEntity) Panics() { + panic("boom") +} + +func Test_EntityDispatcher_InvalidSignaturesReturnErrors(t *testing.T) { + entity := NewEntityFor[invalidSignatureEntity]() + for _, operation := range []string{"TooManyInputs", "WrongSecondReturn"} { + t.Run(operation, func(t *testing.T) { + ctx := &EntityContext{ID: api.NewEntityID("invalid", "key"), Operation: operation} + assert.NotPanics(t, func() { + _, err := entity(ctx) + require.Error(t, err) + }) + }) + } +} + +func Test_EntityDispatcher_UserPanicEscapesReflection(t *testing.T) { + entity := NewEntityFor[invalidSignatureEntity]() + ctx := &EntityContext{ID: api.NewEntityID("invalid", "key"), Operation: "Panics"} + assert.PanicsWithValue(t, "boom", func() { + _, _ = entity(ctx) + }) +} + +type optionalInputEntity struct { + Value int `json:"value"` +} + +func (entity *optionalInputEntity) Add(input OptionalEntityInput[int]) int { + entity.Value += input.Or(5) + return entity.Value +} + +func Test_EntityDispatcher_OptionalInput(t *testing.T) { + entity := NewEntityFor[optionalInputEntity]() + + absent := &EntityContext{ID: api.NewEntityID("optional", "key"), Operation: "Add"} + result, err := entity(absent) + require.NoError(t, err) + require.Equal(t, 5, result) + + present := &EntityContext{ + ID: api.NewEntityID("optional", "key"), + Operation: "Add", + rawInput: presentEntityPayload("3"), + state: absent.state, + } + result, err = entity(present) + require.NoError(t, err) + require.Equal(t, 8, result) +} + +type nullableInputEntity struct{} + +func (*nullableInputEntity) IsNil(value *string) bool { + return value == nil +} + +func Test_EntityDispatcher_JSONNullIsPresent(t *testing.T) { + entity := NewEntityFor[nullableInputEntity]() + ctx := &EntityContext{ + ID: api.NewEntityID("nullable", "key"), + Operation: "IsNil", + rawInput: presentEntityPayload("null"), + } + result, err := entity(ctx) + require.NoError(t, err) + require.Equal(t, true, result) + require.True(t, ctx.HasInput()) +} + +type caseCollisionEntity struct{} + +func (*caseCollisionEntity) Operation() {} +func (*caseCollisionEntity) OPERATION() {} + +func Test_EntityDispatcher_RejectsCaseInsensitiveOperationCollisions(t *testing.T) { + assert.PanicsWithValue(t, "NewEntityFor found case-insensitive operation collision between OPERATION and Operation", func() { + _ = NewEntityFor[caseCollisionEntity]() + }) +} + +type ignoredSetStateErrorEntity struct { + Value int `json:"value"` +} + +func (entity *ignoredSetStateErrorEntity) Mutate(ctx *EntityContext) int { + entity.Value++ + _ = ctx.SetState(make(chan int)) + return entity.Value +} + +func Test_EntityDispatcher_FailedExplicitStateDoesNotSuppressAutoSave(t *testing.T) { + entity := NewEntityFor[ignoredSetStateErrorEntity]() + ctx := &EntityContext{ID: api.NewEntityID("state", "key"), Operation: "Mutate"} + result, err := entity(ctx) + require.NoError(t, err) + require.Equal(t, 1, result) + var state ignoredSetStateErrorEntity + require.NoError(t, ctx.GetState(&state)) + require.Equal(t, 1, state.Value) +} diff --git a/task/entity_lock_test.go b/task/entity_lock_test.go new file mode 100644 index 00000000..06edfc59 --- /dev/null +++ b/task/entity_lock_test.go @@ -0,0 +1,163 @@ +package task + +import ( + "math/rand" + "slices" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestLockEntitiesSortsAndDeduplicatesEveryPermutation(t *testing.T) { + source := []api.EntityID{ + api.NewEntityID("cart", "z"), + api.NewEntityID("account", "b"), + api.NewEntityID("account", "a"), + api.NewEntityID("cart", "z"), + } + expected := []string{"@account@a", "@account@b", "@cart@z"} + random := rand.New(rand.NewSource(42)) + + for iteration := 0; iteration < 100; iteration++ { + entities := append([]api.EntityID(nil), source...) + random.Shuffle(len(entities), func(left, right int) { + entities[left], entities[right] = entities[right], entities[left] + }) + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("lock-order", func(ctx *OrchestrationContext) (any, error) { + _, err := ctx.LockEntities(entities...) + return nil, err + })) + instanceID := api.InstanceID("lock-order-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("lock-order", string(instanceID), nil, nil, nil, nil), + }, + ) + require.Len(t, result.Actions, 1) + lock := result.Actions[0].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, lock) + require.True(t, slices.Equal(expected, lock.LockSet)) + require.Equal(t, int32(0), lock.Position) + } +} + +func TestLockEntitiesCancellationBeforeDispatchDoesNotEmitLock(t *testing.T) { + entityID := api.NewEntityID("counter", "cancel") + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("cancel-lock", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + ctx.Go(func(*OrchestrationContext) { + cancel() + }) + _, err := child.LockEntities(entityID) + return err != nil, nil + })) + instanceID := api.InstanceID("cancel-lock-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-lock", string(instanceID), nil, nil, nil, nil), + }, + ) + for _, action := range result.Actions { + require.Nil(t, action.GetSendEntityMessage()) + } +} + +func TestCallEntityCompletionAfterUnlockDoesNotPanic(t *testing.T) { + entityID := api.NewEntityID("counter", "unlock-before-await") + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("unlock-before-await", func(ctx *OrchestrationContext) (any, error) { + unlock, err := ctx.LockEntities(entityID) + if err != nil { + return nil, err + } + call := ctx.CallEntity(entityID, "get") + unlock() + return "done", call.Await(nil) + })) + instanceID := api.InstanceID("unlock-before-await-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("unlock-before-await", string(instanceID), nil, nil, nil, nil) + first := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + require.Len(t, first.Actions, 1) + lockRequest := first.Actions[0].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, lockRequest) + + lockHistory := &protos.HistoryEvent{ + EventId: first.Actions[0].Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityLockRequested{ + EntityLockRequested: lockRequest, + }, + } + lockGranted := &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityLockGranted{ + EntityLockGranted: &protos.EntityLockGrantedEvent{CriticalSectionId: lockRequest.CriticalSectionId}, + }, + } + secondHistory := []*protos.HistoryEvent{started, executionStarted, lockHistory, lockGranted} + second := executeOrchestrationTurn(t, registry, instanceID, secondHistory, nil) + require.Len(t, second.Actions, 2) + callAction := second.Actions[0].GetSendEntityMessage().GetEntityOperationCalled() + unlockAction := second.Actions[1].GetSendEntityMessage().GetEntityUnlockSent() + require.NotNil(t, callAction) + require.NotNil(t, unlockAction) + + callHistory := &protos.HistoryEvent{ + EventId: second.Actions[0].Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationCalled{ + EntityOperationCalled: callAction, + }, + } + unlockHistory := &protos.HistoryEvent{ + EventId: second.Actions[1].Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityUnlockSent{ + EntityUnlockSent: unlockAction, + }, + } + completed := &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityOperationCompleted{ + EntityOperationCompleted: &protos.EntityOperationCompletedEvent{RequestId: callAction.RequestId}, + }, + } + thirdHistory := append(append([]*protos.HistoryEvent(nil), secondHistory...), callHistory, unlockHistory) + var third *protos.OrchestratorResponse + require.NotPanics(t, func() { + third = executeOrchestrationTurn( + t, + registry, + instanceID, + thirdHistory, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), completed}, + ) + }) + require.Len(t, third.Actions, 1) + require.NotNil(t, third.Actions[0].GetCompleteOrchestration()) +} diff --git a/task/entity_object.go b/task/entity_object.go new file mode 100644 index 00000000..82525275 --- /dev/null +++ b/task/entity_object.go @@ -0,0 +1,230 @@ +package task + +import ( + "errors" + "fmt" + "reflect" + "slices" + "strings" + + "github.com/microsoft/durabletask-go/internal/helpers" +) + +// EntityObjectBase binds a persistent entity object to its current state and operation context. +// Embed it in an entity object used with NewEntityObjectFactory. +type EntityObjectBase[S any] struct { + state *S + context *EntityContext +} + +// State returns the mutable state for the current operation. +func (base *EntityObjectBase[S]) State() *S { + return base.state +} + +// Context returns the context for the current operation. +func (base *EntityObjectBase[S]) Context() *EntityContext { + return base.context +} + +func (base *EntityObjectBase[S]) bindEntityObject(ctx *EntityContext, state *S) { + base.context = ctx + base.state = state +} + +// EntityObjectBinding is implemented by entity objects that embed EntityObjectBase. +type EntityObjectBinding[S any] interface { + bindEntityObject(*EntityContext, *S) +} + +type entityObjectOptions[S any] struct { + initializer func(*EntityContext) (S, error) + allowStateDispatch bool +} + +// EntityObjectOption configures reflected entity-object dispatch. +type EntityObjectOption[S any] func(*entityObjectOptions[S]) + +// WithEntityStateInitializer configures state creation when an operation starts without state. +func WithEntityStateInitializer[S any]( + initializer func(*EntityContext) (S, error), +) EntityObjectOption[S] { + return func(options *entityObjectOptions[S]) { + options.initializer = initializer + } +} + +// WithEntityStateDispatch enables reflected operation dispatch to the state +// after no matching entity-object method is found. +func WithEntityStateDispatch[S any]() EntityObjectOption[S] { + return func(options *entityObjectOptions[S]) { + options.allowStateDispatch = true + } +} + +// NewEntityObjectFactory creates one persistent entity object per batch while +// storing only S as durable state. +// +// Operation binding is validated when the factory is created, so a malformed +// entity object fails at startup instead of failing every batch at runtime. +func NewEntityObjectFactory[S any, E EntityObjectBinding[S]]( + factory func(EntityFactoryContext) (E, error), + configure ...EntityObjectOption[S], +) EntityFactory { + stateType := reflect.TypeFor[S]() + if stateType.Kind() == reflect.Pointer { + panic("NewEntityObjectFactory does not support pointer state types") + } + if factory == nil { + panic("NewEntityObjectFactory requires an entity object factory") + } + options := entityObjectOptions[S]{} + for _, apply := range configure { + apply(&options) + } + + // A concrete type parameter is validated eagerly. An interface type + // parameter only reveals its operations once the object is created. + var staticSets *entityObjectMethodSets + if objectType := reflect.TypeFor[E](); objectType.Kind() != reflect.Interface { + sets, err := newEntityObjectMethodSets(objectType, stateType) + if err != nil { + panic(err.Error()) + } + staticSets = sets + } + + return func(factoryContext EntityFactoryContext) (EntityBatch, error) { + object, err := factory(factoryContext) + if err != nil { + return EntityBatch{}, err + } + if isNilEntityValue(object) { + return EntityBatch{}, fmt.Errorf("entity object factory returned nil") + } + batch := EntityBatch{} + if closer, ok := any(object).(EntityBatchCloser); ok { + batch.Close = closer.CloseEntityBatch + } + + objectValue := reflect.ValueOf(object) + sets := staticSets + if sets == nil { + sets, err = newEntityObjectMethodSets(objectValue.Type(), stateType) + if err != nil { + return closeAfterSetupFailure(factoryContext, batch, err) + } + } + + batch.Entity = func(ctx *EntityContext) (any, error) { + var state S + if ctx.HasState() { + if err := ctx.GetState(&state); err != nil { + return nil, fmt.Errorf("failed to deserialize entity state: %w", err) + } + } else if options.initializer != nil { + initialized, err := options.initializer(ctx) + if err != nil { + return nil, fmt.Errorf("failed to initialize entity state: %w", err) + } + state = initialized + } + object.bindEntityObject(ctx, &state) + + result, found, err := sets.object.call(ctx, objectValue) + if err != nil { + return nil, err + } + if !found && options.allowStateDispatch { + result, found, err = sets.state.call(ctx, reflect.ValueOf(&state)) + if err != nil { + return nil, err + } + } + if !found { + if strings.EqualFold(ctx.Operation, "delete") { + ctx.DeleteState() + return nil, nil + } + return nil, fmt.Errorf("entity does not support operation %q", ctx.Operation) + } + if !ctx.stateDirty { + if err := ctx.SetState(state); err != nil { + return nil, fmt.Errorf("failed to save entity state: %w", err) + } + } + return result, nil + } + return batch, nil + } +} + +// entityObjectMethodSets holds the dispatch tables shared by every batch of one +// registered entity object. +type entityObjectMethodSets struct { + object *entityMethodSet + state *entityMethodSet +} + +func newEntityObjectMethodSets( + objectType reflect.Type, + stateType reflect.Type, +) (*entityObjectMethodSets, error) { + objectMethods, err := collectEntityMethods( + objectType, + "closeentitybatch", + "context", + "state", + ) + if err != nil { + return nil, err + } + stateMethods, err := collectEntityMethods(reflect.PointerTo(stateType)) + if err != nil { + return nil, err + } + return &entityObjectMethodSets{ + object: &entityMethodSet{methods: objectMethods}, + state: &entityMethodSet{methods: stateMethods}, + }, nil +} + +func closeAfterSetupFailure( + factoryContext EntityFactoryContext, + batch EntityBatch, + setupErr error, +) (EntityBatch, error) { + if batch.Close == nil { + return EntityBatch{}, setupErr + } + if closeErr := batch.Close(factoryContext.Context); closeErr != nil { + return EntityBatch{}, errors.Join( + setupErr, + fmt.Errorf("failed to close entity batch after setup failure: %w", closeErr), + ) + } + return EntityBatch{}, setupErr +} + +func collectEntityMethods( + targetType reflect.Type, + excludedNames ...string, +) (map[string]reflect.Method, error) { + methods := make(map[string]reflect.Method) + for i := 0; i < targetType.NumMethod(); i++ { + method := targetType.Method(i) + name := helpers.ToLowerInvariant(method.Name) + if slices.Contains(excludedNames, name) { + continue + } + if existing, ok := methods[name]; ok { + return nil, fmt.Errorf( + "entity object found case-insensitive operation collision between %s and %s", + existing.Name, + method.Name, + ) + } + methods[name] = method + } + return methods, nil +} diff --git a/task/entity_object_test.go b/task/entity_object_test.go new file mode 100644 index 00000000..564afe5b --- /dev/null +++ b/task/entity_object_test.go @@ -0,0 +1,286 @@ +package task + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type objectCounterState struct { + Value int `json:"value"` +} + +func (state *objectCounterState) Read() int { + return state.Value +} + +type objectCounter struct { + EntityObjectBase[objectCounterState] + calls int + closeCalls *int +} + +func (entity *objectCounter) Add(value int) int { + entity.calls++ + entity.State().Value += value + return entity.State().Value +} + +func (entity *objectCounter) Fail(value int) error { + entity.calls++ + entity.State().Value += value + return errors.New("failed") +} + +func (entity *objectCounter) Calls() int { + return entity.calls +} + +func (entity *objectCounter) CloseEntityBatch(context.Context) error { + (*entity.closeCalls)++ + return nil +} + +func Test_EntityObjectFactory_StateLifecycleAndDispatch(t *testing.T) { + registry := NewTaskRegistry() + factoryCalls := 0 + initializations := 0 + closeCalls := 0 + factory := NewEntityObjectFactory[objectCounterState, *objectCounter]( + func(EntityFactoryContext) (*objectCounter, error) { + factoryCalls++ + return &objectCounter{closeCalls: &closeCalls}, nil + }, + WithEntityStateInitializer(func(*EntityContext) (objectCounterState, error) { + initializations++ + return objectCounterState{Value: 10}, nil + }), + WithEntityStateDispatch[objectCounterState](), + ) + require.NoError(t, registry.AddEntityFactoryN("counter", factory)) + + executor := NewTaskExecutor(registry).(EntityExecutor) + result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{ + {Operation: "Add", Input: wrapperspb.String("1")}, + {Operation: "Fail", Input: wrapperspb.String("100")}, + {Operation: "Read"}, + {Operation: "Calls"}, + {Operation: "delete"}, + {Operation: "Read"}, + }, + }) + require.NoError(t, err) + require.Equal(t, 1, factoryCalls) + require.Equal(t, 1, closeCalls) + require.Equal(t, 2, initializations) + require.Len(t, result.Results, 6) + require.Equal(t, "11", result.Results[0].GetSuccess().GetResult().GetValue()) + require.Contains(t, result.Results[1].GetFailure().GetFailureDetails().GetErrorMessage(), "failed") + require.Equal(t, "11", result.Results[2].GetSuccess().GetResult().GetValue()) + require.Equal(t, "2", result.Results[3].GetSuccess().GetResult().GetValue()) + require.Nil(t, result.Results[4].GetSuccess().GetResult()) + require.Equal(t, "10", result.Results[5].GetSuccess().GetResult().GetValue()) + require.JSONEq(t, `{"value":10}`, result.EntityState.GetValue()) +} + +func Test_EntityFactory_RunsAfterStateHandshake(t *testing.T) { + registry := NewTaskRegistry() + factoryCalls := 0 + require.NoError(t, registry.AddEntityFactoryN("counter", func(EntityFactoryContext) (EntityBatch, error) { + factoryCalls++ + return EntityBatch{ + Entity: func(*EntityContext) (any, error) { return nil, nil }, + }, nil + })) + executor := NewTaskExecutor(registry).(EntityExecutor) + + result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Properties: map[string]*structpb.Value{ + "IncludeState": structpb.NewBoolValue(false), + }, + }) + require.NoError(t, err) + require.True(t, result.RequiresState) + require.Zero(t, factoryCalls) + + _, err = executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + }) + require.NoError(t, err) + require.Equal(t, 1, factoryCalls) +} + +func Test_EntityFactory_ErrorIsFrameworkFailure(t *testing.T) { + registry := NewTaskRegistry() + expected := errors.New("factory failed") + require.NoError(t, registry.AddEntityFactoryN("counter", func(EntityFactoryContext) (EntityBatch, error) { + return EntityBatch{}, expected + })) + + _, err := NewTaskExecutor(registry).(EntityExecutor).ExecuteEntity( + context.Background(), + &protos.EntityBatchRequest{InstanceId: "@counter@key"}, + ) + require.ErrorIs(t, err, expected) + require.ErrorContains(t, err, fmt.Sprintf("failed to create entity %q", "counter")) +} + +func Test_EntityFactory_CloseErrorIsFrameworkFailure(t *testing.T) { + registry := NewTaskRegistry() + expected := errors.New("close failed") + require.NoError(t, registry.AddEntityFactoryN("counter", func(EntityFactoryContext) (EntityBatch, error) { + return EntityBatch{ + Entity: func(*EntityContext) (any, error) { return nil, nil }, + Close: func(context.Context) error { return expected }, + }, nil + })) + + _, err := NewTaskExecutor(registry).(EntityExecutor).ExecuteEntity( + context.Background(), + &protos.EntityBatchRequest{InstanceId: "@counter@key"}, + ) + require.ErrorIs(t, err, expected) + require.ErrorContains(t, err, "failed to close entity batch") +} + +type pointerObjectState struct{} + +type pointerStateObject struct { + EntityObjectBase[*pointerObjectState] +} + +func Test_EntityObjectFactory_RejectsPointerState(t *testing.T) { + require.PanicsWithValue( + t, + "NewEntityObjectFactory does not support pointer state types", + func() { + _ = NewEntityObjectFactory[*pointerObjectState, *pointerStateObject]( + func(EntityFactoryContext) (*pointerStateObject, error) { + return new(pointerStateObject), nil + }, + ) + }, + ) +} + +type setupFailureObject struct { + EntityObjectBase[objectCounterState] + closeCalls *int +} + +func (*setupFailureObject) Operation() {} +func (*setupFailureObject) OPERATION() {} + +func (entity *setupFailureObject) CloseEntityBatch(context.Context) error { + (*entity.closeCalls)++ + return nil +} + +// A colliding operation set is a permanent authoring error, so it must fail when +// the factory is created rather than failing every batch as a retriable fault. +func Test_EntityObjectFactory_RejectsCollisionAtConstruction(t *testing.T) { + closeCalls := 0 + require.PanicsWithValue( + t, + "entity object found case-insensitive operation collision between OPERATION and Operation", + func() { + _ = NewEntityObjectFactory[objectCounterState, *setupFailureObject]( + func(EntityFactoryContext) (*setupFailureObject, error) { + return &setupFailureObject{closeCalls: &closeCalls}, nil + }, + ) + }, + ) + require.Zero(t, closeCalls) +} + +// An interface type parameter only reveals its operations once the object +// exists, so that path still reports setup failure per batch and releases it. +func Test_EntityObjectFactory_ClosesAfterDynamicSetupFailure(t *testing.T) { + closeCalls := 0 + factory := NewEntityObjectFactory[objectCounterState, EntityObjectBinding[objectCounterState]]( + func(EntityFactoryContext) (EntityObjectBinding[objectCounterState], error) { + return &setupFailureObject{closeCalls: &closeCalls}, nil + }, + ) + _, err := factory(EntityFactoryContext{Context: context.Background()}) + require.ErrorContains(t, err, "case-insensitive operation collision") + require.Equal(t, 1, closeCalls) +} + +type panicConverter struct{} + +func (panicConverter) Serialize(any) (string, error) { + panic("converter panic") +} + +func (panicConverter) Deserialize(string, any) error { + return nil +} + +func Test_EntityFactory_ClosesWhenExecutionPanics(t *testing.T) { + registry := NewTaskRegistry() + closeCalls := 0 + require.NoError(t, registry.AddEntityFactoryN("counter", func(EntityFactoryContext) (EntityBatch, error) { + return EntityBatch{ + Entity: func(*EntityContext) (any, error) { return 1, nil }, + Close: func(context.Context) error { + closeCalls++ + return nil + }, + }, nil + })) + executor := NewTaskExecutor(registry, WithDataConverter(panicConverter{})).(EntityExecutor) + require.PanicsWithValue(t, "converter panic", func() { + _, _ = executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "get"}}, + }) + }) + require.Equal(t, 1, closeCalls) +} + +func Test_EntitySingleton_CanRunConcurrentBatches(t *testing.T) { + registry := NewTaskRegistry() + var calls atomic.Int32 + require.NoError(t, registry.AddEntityN("counter", func(*EntityContext) (any, error) { + calls.Add(1) + return nil, nil + })) + executor := NewTaskExecutor(registry).(EntityExecutor) + + var wait sync.WaitGroup + failures := make(chan error, 16) + for i := 0; i < 16; i++ { + wait.Add(1) + go func() { + defer wait.Done() + result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "run"}}, + }) + if err != nil { + failures <- err + } else if len(result.Results) != 1 { + failures <- fmt.Errorf("result count = %d, want 1", len(result.Results)) + } + }() + } + wait.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } + require.EqualValues(t, 16, calls.Load()) +} diff --git a/task/entity_parameters_test.go b/task/entity_parameters_test.go new file mode 100644 index 00000000..d03b4d7b --- /dev/null +++ b/task/entity_parameters_test.go @@ -0,0 +1,115 @@ +package task + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/durationpb" +) + +func Test_ValidateEntityParameters(t *testing.T) { + supported, err := validateEntityParameters(nil) + require.NoError(t, err) + require.False(t, supported) + + supported, err = validateEntityParameters(&protos.OrchestratorEntityParameters{ + EntityMessageReorderWindow: durationpb.New(0), + }) + require.NoError(t, err) + require.True(t, supported) + + for name, parameters := range map[string]*protos.OrchestratorEntityParameters{ + "missing-window": {}, + "negative-window": { + EntityMessageReorderWindow: durationpb.New(-time.Second), + }, + "positive-window": { + EntityMessageReorderWindow: durationpb.New(time.Second), + }, + "malformed-window": { + EntityMessageReorderWindow: &durationpb.Duration{Seconds: 1, Nanos: -1}, + }, + } { + t.Run(name, func(t *testing.T) { + supported, err := validateEntityParameters(parameters) + require.Error(t, err) + require.False(t, supported) + var delayed interface{ WorkItemAbandonDelay() time.Duration } + require.True(t, errors.As(err, &delayed)) + }) + } +} + +func Test_OrchestrationEntitySupportGating(t *testing.T) { + entityID := api.NewEntityID("counter", "key") + for _, test := range []struct { + name string + orchestrator Orchestrator + }{ + { + name: "call", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallEntity(entityID, "get").Await(nil) + }, + }, + { + name: "signal", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.SignalEntity(entityID, "add") + }, + }, + { + name: "lock", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + _, err := ctx.LockEntities(entityID) + return nil, err + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN(test.name, test.orchestrator)) + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent(test.name, "instance", nil, nil, nil, nil), + }, + nil, + ) + require.NoError(t, err) + for _, action := range result.Response.Actions { + require.Nil(t, action.GetSendEntityMessage()) + } + require.NotNil(t, completionAction(t, result.Response).FailureDetails) + }) + } +} + +func Test_OrchestrationEntitySupportWithZeroWindow(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("signal", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.SignalEntity(api.NewEntityID("counter", "key"), "add") + })) + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "instance", + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("signal", "instance", nil, nil, nil, nil), + }, + supportedEntityParameters(), + ) + require.NoError(t, err) + require.Len(t, result.Response.Actions, 2) + require.NotNil(t, result.Response.Actions[0].GetSendEntityMessage()) +} diff --git a/task/entity_test.go b/task/entity_test.go new file mode 100644 index 00000000..1235f286 --- /dev/null +++ b/task/entity_test.go @@ -0,0 +1,253 @@ +package task + +import ( + "regexp" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func presentEntityPayload(value string) entityPayload { + return entityPayload{value: []byte(value), present: true} +} + +func Test_EntityContext_State(t *testing.T) { + t.Run("no state initially", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + assert.False(t, ctx.HasState()) + assert.Error(t, ctx.GetState(new(int))) + }) + + t.Run("set and get state", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + require.NoError(t, ctx.SetState(42)) + assert.True(t, ctx.HasState()) + + var val int + require.NoError(t, ctx.GetState(&val)) + assert.Equal(t, 42, val) + }) + + t.Run("delete state with nil", func(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + state: entityState{value: []byte("42"), hasValue: true}, + } + assert.True(t, ctx.HasState()) + require.NoError(t, ctx.SetState(nil)) + assert.False(t, ctx.HasState()) + }) + + t.Run("delete state with typed nil", func(t *testing.T) { + ctx := &EntityContext{ + state: entityState{value: []byte("42"), hasValue: true}, + } + var state *int + require.NoError(t, ctx.SetState(state)) + assert.False(t, ctx.HasState()) + }) + + t.Run("set struct state", func(t *testing.T) { + type MyState struct { + Count int `json:"count"` + Name string `json:"name"` + } + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + require.NoError(t, ctx.SetState(MyState{Count: 5, Name: "hello"})) + assert.True(t, ctx.HasState()) + + var result MyState + require.NoError(t, ctx.GetState(&result)) + assert.Equal(t, 5, result.Count) + assert.Equal(t, "hello", result.Name) + }) +} + +func Test_EntityContext_GetInput(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + rawInput: presentEntityPayload(`"hello"`), + } + + var input string + require.NoError(t, ctx.GetInput(&input)) + assert.Equal(t, "hello", input) + + require.Error(t, (&EntityContext{}).GetInput(&input)) + require.Error(t, (&EntityContext{ + rawInput: presentEntityPayload(""), + }).GetInput(&input)) +} + +func Test_EntityContext_RawState(t *testing.T) { + ctx := &EntityContext{} + ctx.SetRawState(`{"version":1}`) + state, ok := ctx.GetRawState() + require.True(t, ok) + assert.Equal(t, `{"version":1}`, state) + + ctx.SetRawState("") + _, ok = ctx.GetRawState() + assert.False(t, ok) +} + +func Test_EntityContext_SignalEntity(t *testing.T) { + parentTrace := &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + TraceState: wrapperspb.String("vendor=value"), + } + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + parentTrace: parentTrace, + } + + err := ctx.SignalEntity(api.NewEntityID("other", "key2"), "increment", 5) + require.NoError(t, err) + require.Len(t, ctx.actions, 1) + + action := ctx.actions[0] + signal := action.GetSendSignal() + require.NotNil(t, signal) + assert.Equal(t, "@other@key2", signal.InstanceId) + assert.Equal(t, "increment", signal.Name) + assert.Equal(t, "5", signal.Input.GetValue()) + assert.Equal(t, parentTrace, signal.ParentTraceContext) + assert.NotSame(t, parentTrace, signal.ParentTraceContext) +} + +func Test_EntityContext_TypedNilSignalInputIsAbsent(t *testing.T) { + ctx := &EntityContext{} + var input *int + require.NoError(t, ctx.SignalEntity(api.NewEntityID("other", "key"), "operation", input)) + require.Len(t, ctx.actions, 1) + require.Nil(t, ctx.actions[0].GetSendSignal().Input) +} + +func Test_EntityTypedNilOptionsAreAbsent(t *testing.T) { + var input *int + + callOptions := new(callEntityOptions) + require.NoError(t, WithEntityInput(input)(callOptions, api.DefaultDataConverter())) + require.Nil(t, callOptions.rawInput) + + signalOptions := new(signalEntityOptions) + require.NoError(t, WithSignalEntityInput(input)(signalOptions, api.DefaultDataConverter())) + require.Nil(t, signalOptions.rawInput) + + startOptions := new(entityStartOrchestrationOptions) + require.NoError(t, WithEntityStartOrchestrationInput(input)(startOptions, api.DefaultDataConverter())) + require.Nil(t, startOptions.rawInput) +} + +func Test_EntityContext_SignalEntity_RejectsInvalidEntityID(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + + err := ctx.SignalEntity(api.EntityID{Name: "bad@name", Key: "key2"}, "increment", 5) + require.Error(t, err) + require.Empty(t, ctx.actions) +} + +func Test_EntityContext_StartNewOrchestration(t *testing.T) { + parentTrace := &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + } + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + parentTrace: parentTrace, + } + + err := ctx.StartNewOrchestration("MyOrchestrator", + WithEntityStartOrchestrationInput("hello"), + WithEntityStartOrchestrationInstanceID("my-instance"), + ) + require.NoError(t, err) + require.Len(t, ctx.actions, 1) + + action := ctx.actions[0] + startOrch := action.GetStartNewOrchestration() + require.NotNil(t, startOrch) + assert.Equal(t, "MyOrchestrator", startOrch.Name) + assert.Equal(t, "my-instance", startOrch.InstanceId) + assert.Equal(t, `"hello"`, startOrch.Input.GetValue()) + assert.Equal(t, parentTrace, startOrch.ParentTraceContext) + assert.NotSame(t, parentTrace, startOrch.ParentTraceContext) +} + +func Test_EntityContext_StartNewOrchestration_RawInput(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + + err := ctx.StartNewOrchestration("MyOrchestrator", + WithRawEntityStartOrchestrationInput(`{"hello":"world"}`), + WithEntityStartOrchestrationInstanceID("my-instance"), + ) + require.NoError(t, err) + require.Len(t, ctx.actions, 1) + assert.Equal(t, `{"hello":"world"}`, ctx.actions[0].GetStartNewOrchestration().Input.GetValue()) +} + +func Test_EntityContext_StartNewOrchestration_DefaultInstanceID(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + + err := ctx.StartNewOrchestration("MyOrchestrator") + require.NoError(t, err) + require.Len(t, ctx.actions, 1) + + startOrch := ctx.actions[0].GetStartNewOrchestration() + require.NotNil(t, startOrch) + assert.Regexp(t, regexp.MustCompile("^[a-f0-9]{32}$"), startOrch.InstanceId) +} + +func Test_EntityContext_StartNewOrchestration_RejectsEntityInstanceID(t *testing.T) { + ctx := &EntityContext{ + ID: api.NewEntityID("test", "key1"), + Operation: "op", + } + + err := ctx.StartNewOrchestration("MyOrchestrator", WithEntityStartOrchestrationInstanceID("@counter@key1")) + require.Error(t, err) + require.Empty(t, ctx.actions) +} + +func Test_EntityRegistry(t *testing.T) { + r := NewTaskRegistry() + + myEntity := func(ctx *EntityContext) (any, error) { return nil, nil } + require.NoError(t, r.AddEntityN("counter", myEntity)) + + // Duplicate registration should fail + err := r.AddEntityN("counter", myEntity) + require.Error(t, err) + assert.Contains(t, err.Error(), "already registered") + + err = r.AddEntityN("", myEntity) + require.Error(t, err) + + err = r.AddEntityN("bad@name", myEntity) + require.Error(t, err) +} diff --git a/task/errors.go b/task/errors.go new file mode 100644 index 00000000..fdb539eb --- /dev/null +++ b/task/errors.go @@ -0,0 +1,218 @@ +package task + +import ( + "errors" + "fmt" + "time" + + "github.com/microsoft/durabletask-go/api" +) + +const ( + activityTaskNotFoundErrorType = api.ErrorTypeActivityTaskNotFound + entityTaskNotFoundErrorType = api.ErrorTypeEntityTaskNotFound + orchestratorTaskNotFoundErrorType = api.ErrorTypeOrchestratorTaskNotFound + taskFailedErrorType = api.ErrorTypeTaskFailed + entityOperationFailedErrorType = api.ErrorTypeEntityOperationFailed + versionMismatchErrorType = api.ErrorTypeVersionMismatch +) + +var errEntitiesUnsupported = errors.New("durable entities are not supported by the current backend configuration") + +// TaskFailedError is returned when an activity or sub-orchestration fails. +type TaskFailedError struct { + TaskName string + TaskVersion string + TaskID int32 + FailureDetails *api.FailureDetails +} + +func (e *TaskFailedError) Error() string { + if e.FailureDetails == nil || e.FailureDetails.ErrorMessage == "" { + return fmt.Sprintf("Task '%s' (#%d) failed with an unhandled exception.", e.TaskName, e.TaskID) + } + return fmt.Sprintf( + "Task '%s' (#%d) failed with an unhandled exception: %s", + e.TaskName, + e.TaskID, + e.FailureDetails.ErrorMessage, + ) +} + +func (*TaskFailedError) DurableTaskErrorType() api.ErrorType { + return taskFailedErrorType +} + +func (e *TaskFailedError) DurableTaskFailureDetails() *api.FailureDetails { + return e.FailureDetails +} + +func (e *TaskFailedError) NonRetriable() bool { + return e.FailureDetails.NonRetriable() +} + +func (e *TaskFailedError) Is(target error) bool { + return e.FailureDetails.Matches(target) +} + +// EntityOperationFailedError is returned when a called entity operation fails. +type EntityOperationFailedError struct { + EntityID api.EntityID + OperationName string + FailureDetails *api.FailureDetails +} + +func (e *EntityOperationFailedError) Error() string { + message := "" + if e.FailureDetails != nil { + message = e.FailureDetails.ErrorMessage + } + return fmt.Sprintf( + "Operation '%s' of entity '%s' failed: %s", + e.OperationName, + e.EntityID, + message, + ) +} + +func (*EntityOperationFailedError) DurableTaskErrorType() api.ErrorType { + return entityOperationFailedErrorType +} + +func (e *EntityOperationFailedError) DurableTaskFailureDetails() *api.FailureDetails { + return e.FailureDetails +} + +func (e *EntityOperationFailedError) NonRetriable() bool { + return e.FailureDetails.NonRetriable() +} + +func (e *EntityOperationFailedError) Is(target error) bool { + return e.FailureDetails.Matches(target) +} + +func failureDetailsFromError(err error) *api.FailureDetails { + var taskFailure *TaskFailedError + if errors.As(err, &taskFailure) { + return taskFailure.FailureDetails + } + var entityFailure *EntityOperationFailedError + if errors.As(err, &entityFailure) { + return entityFailure.FailureDetails + } + return nil +} + +type taskNotRegisteredError struct { + errorType api.ErrorType + message string +} + +func newTaskNotRegisteredError(errorType api.ErrorType, name, version string) error { + taskKind := "task" + switch errorType { + case activityTaskNotFoundErrorType: + taskKind = "activity task" + case entityTaskNotFoundErrorType: + taskKind = "entity task" + case orchestratorTaskNotFoundErrorType: + taskKind = "orchestrator task" + } + message := fmt.Sprintf("No %s named '%s' was found.", taskKind, name) + if version != "" && errorType != entityTaskNotFoundErrorType { + message = fmt.Sprintf("No %s named '%s' with version '%s' was found.", taskKind, name, version) + } + return &taskNotRegisteredError{ + errorType: errorType, + message: message, + } +} + +func (e *taskNotRegisteredError) Error() string { + return e.message +} + +func (e *taskNotRegisteredError) DurableTaskErrorType() api.ErrorType { + return e.errorType +} + +func (*taskNotRegisteredError) NonRetriable() bool { + return true +} + +func (*taskNotRegisteredError) Is(target error) bool { + return target == api.ErrTaskNotRegistered +} + +func (e *taskNotRegisteredError) WorkItemAbandonDelay() time.Duration { + return time.Second +} + +type panicFailureError struct { + errorType api.ErrorType + message string + stack string + cause error +} + +func newPanicFailureError(errorType api.ErrorType, message, stack string, cause error) error { + return &panicFailureError{ + errorType: errorType, + message: message, + stack: stack, + cause: cause, + } +} + +func (e *panicFailureError) Error() string { + return e.message +} + +func (e *panicFailureError) Unwrap() error { + return e.cause +} + +func (e *panicFailureError) DurableTaskErrorType() api.ErrorType { + return e.errorType +} + +func (e *panicFailureError) DurableTaskStackTrace() string { + return e.stack +} + +func panicCause(value any) error { + err, _ := value.(error) + return err +} + +type workItemAbandonDelayProvider interface { + WorkItemAbandonDelay() time.Duration +} + +type unsupportedEntityParametersError struct { + message string +} + +func (e *unsupportedEntityParametersError) Error() string { + return e.message +} + +func (*unsupportedEntityParametersError) WorkItemAbandonDelay() time.Duration { + return time.Second +} + +var _ error = (*TaskFailedError)(nil) +var _ error = (*EntityOperationFailedError)(nil) +var _ api.DurableTaskErrorTypeProvider = (*TaskFailedError)(nil) +var _ api.DurableTaskFailureDetailsProvider = (*TaskFailedError)(nil) +var _ api.NonRetriable = (*TaskFailedError)(nil) +var _ api.DurableTaskErrorTypeProvider = (*EntityOperationFailedError)(nil) +var _ api.DurableTaskFailureDetailsProvider = (*EntityOperationFailedError)(nil) +var _ api.NonRetriable = (*EntityOperationFailedError)(nil) +var _ api.DurableTaskErrorTypeProvider = (*taskNotRegisteredError)(nil) +var _ api.NonRetriable = (*taskNotRegisteredError)(nil) +var _ workItemAbandonDelayProvider = (*taskNotRegisteredError)(nil) +var _ workItemAbandonDelayProvider = (*VersionMismatchError)(nil) +var _ workItemAbandonDelayProvider = (*unsupportedEntityParametersError)(nil) +var _ api.DurableTaskErrorTypeProvider = (*panicFailureError)(nil) +var _ api.DurableTaskStackTraceProvider = (*panicFailureError)(nil) diff --git a/task/eventchannel.go b/task/eventchannel.go new file mode 100644 index 00000000..751a275c --- /dev/null +++ b/task/eventchannel.go @@ -0,0 +1,141 @@ +package task + +import ( + "fmt" + "reflect" + "strings" + + "github.com/microsoft/durabletask-go/internal/protos" +) + +type bufferedEvent struct { + event *protos.HistoryEvent + order uint64 +} + +// EventChannel receives a named external event repeatedly while preserving +// durable event order. Use ReceiveErr or TryReceiveErr when payload errors +// should be handled by the orchestrator. +type EventChannel[T any] struct { + ctx *OrchestrationContext + name string + key string +} + +// NewEventChannel returns the durable channel for a named external event. +// Repeated calls using the same event name and value type return the same channel. +func NewEventChannel[T any](ctx *OrchestrationContext, name string) *EventChannel[T] { + if ctx == nil { + panic("event channel requires an orchestration context") + } + engine := ctx.engineContext() + key := strings.ToUpper(name) + if existing, ok := engine.eventChannels[key]; ok { + channel, ok := existing.(*EventChannel[T]) + if !ok { + panic(fmt.Sprintf( + "event channel %q was already created with a different value type", + name, + )) + } + return channel + } + channel := &EventChannel[T]{ctx: engine, name: name, key: key} + engine.eventChannels[key] = channel + return channel +} + +// Receive waits for and consumes the next event value. +func (c *EventChannel[T]) Receive(ctx *OrchestrationContext) T { + value, err := c.ReceiveErr(ctx) + if err != nil { + panic(err) + } + return value +} + +// ReceiveErr waits for and consumes the next event value, returning payload +// decoding and cancellation errors to the orchestrator. +func (c *EventChannel[T]) ReceiveErr(ctx *OrchestrationContext) (T, error) { + if ctx.engineContext() != c.ctx { + panic("event channel used with a different orchestration context") + } + var value T + if ctx.scope.isCanceled() { + return value, ErrTaskCanceled + } + if value, ok, err := c.TryReceiveErr(); ok || err != nil { + return value, err + } + if err := ctx.WaitForSingleEvent(c.name, -1).Await(&value); err != nil { + return value, err + } + return value, nil +} + +// TryReceive consumes a buffered event without blocking. +func (c *EventChannel[T]) TryReceive() (T, bool) { + value, ok, err := c.TryReceiveErr() + if err != nil { + panic(err) + } + return value, ok +} + +// TryReceiveErr consumes a buffered event without blocking and returns payload +// decoding errors. +func (c *EventChannel[T]) TryReceiveErr() (T, bool, error) { + var value T + buffered, ok := c.ctx.takeBufferedEvent(c.key) + if !ok { + return value, false, nil + } + raw := []byte(buffered.event.GetEventRaised().GetInput().GetValue()) + if err := unmarshalData(c.ctx.converter, raw, &value); err != nil { + return value, true, fmt.Errorf("failed to decode event %q as %s: %w", c.name, reflect.TypeOf(value), err) + } + return value, true, nil +} + +func (c *EventChannel[T]) peek() (*bufferedEvent, bool) { + return c.ctx.peekBufferedEvent(c.key) +} + +type eventSelectCase[T any] struct { + channel *EventChannel[T] + handler func(T) +} + +// OnEvent creates a Select case that receives from a durable event channel. +func OnEvent[T any](channel *EventChannel[T], handler func(T)) SelectCase { + if channel == nil { + panic("event Select case requires a channel") + } + return &eventSelectCase[T]{channel: channel, handler: handler} +} + +func (c *eventSelectCase[T]) ready() (bool, uint64) { + event, ok := c.channel.peek() + if !ok { + return false, 0 + } + return true, event.order +} + +func (c *eventSelectCase[T]) subscribe(coroutine *coroutine) { + c.channel.ctx.addEventWaiter(c.channel.key, coroutine) +} + +func (c *eventSelectCase[T]) unsubscribe(coroutine *coroutine) { + c.channel.ctx.removeEventWaiter(c.channel.key, coroutine) +} + +func (c *eventSelectCase[T]) invoke() { + value, ok := c.channel.TryReceive() + if !ok { + panic("selected event channel no longer has a buffered value") + } + if c.handler != nil { + c.handler(value) + } +} diff --git a/task/executor.go b/task/executor.go index c210e9be..1b644ce7 100644 --- a/task/executor.go +++ b/task/executor.go @@ -2,72 +2,250 @@ package task import ( "context" - "encoding/json" "fmt" + "log/slog" + "maps" + "runtime/debug" + "strings" + "time" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" + "github.com/microsoft/durabletask-go/internal/contextprop" + "github.com/microsoft/durabletask-go/internal/failure" "github.com/microsoft/durabletask-go/internal/helpers" "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" ) type taskExecutor struct { - Registry *TaskRegistry + Registry *TaskRegistry + versioning *VersioningOptions + orchestratorNotFound OrchestratorNotFoundStrategy + orchestrationOptions OrchestrationOptions + logger *slog.Logger + metrics MetricsHooks + contextFields api.ContextFields + errorProperties api.ErrorPropertiesProvider + converter api.DataConverter + unversionedOrchestrators map[string]struct{} + unversionedActivities map[string]struct{} } -// NewTaskExecutor returns a [backend.Executor] implementation that executes orchestrator and activity functions in-memory. -func NewTaskExecutor(registry *TaskRegistry) backend.Executor { - return &taskExecutor{ - Registry: registry, +// TaskExecutorOption configures the in-memory task executor. +type TaskExecutorOption func(*taskExecutor) + +// OrchestratorNotFoundStrategy determines whether missing orchestrators fail or reject work items. +type OrchestratorNotFoundStrategy int + +const ( + // OrchestratorNotFoundFail completes the orchestration with an OrchestratorTaskNotFound failure. + OrchestratorNotFoundFail OrchestratorNotFoundStrategy = iota + // OrchestratorNotFoundReject abandons the work item so another worker or deployment can process it. + OrchestratorNotFoundReject +) + +// WithVersioning configures version-aware orchestration and activity dispatch. +func WithVersioning(options VersioningOptions) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.versioning = &options + } +} + +// WithUnversionedOrchestratorNames allows named system orchestrators to bypass +// worker version matching when their work item explicitly selects no version. +func WithUnversionedOrchestratorNames(names ...string) TaskExecutorOption { + return func(executor *taskExecutor) { + addUnversionedTaskNames(&executor.unversionedOrchestrators, names) + } +} + +// WithUnversionedActivityNames allows named system activities to bypass worker +// version matching when their work item explicitly selects no version. System +// orchestrations run unversioned, and an activity inherits its caller's version, +// so their activities are dispatched unversioned as well. +func WithUnversionedActivityNames(names ...string) TaskExecutorOption { + return func(executor *taskExecutor) { + addUnversionedTaskNames(&executor.unversionedActivities, names) + } +} + +func addUnversionedTaskNames(allowed *map[string]struct{}, names []string) { + if *allowed == nil { + *allowed = make(map[string]struct{}, len(names)) + } + for _, name := range names { + if name = strings.TrimSpace(name); name != "" { + (*allowed)[strings.ToLower(name)] = struct{}{} + } + } +} + +func allowsUnversionedTask(allowed map[string]struct{}, name, version string) bool { + if version != "" { + return false } + _, ok := allowed[strings.ToLower(name)] + return ok } -// ExecuteActivity implements backend.Executor and executes an activity function in the current goroutine. +// WithErrorPropertiesProvider configures custom durable failure properties. +func WithErrorPropertiesProvider(provider api.ErrorPropertiesProvider) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.errorProperties = provider + } +} + +// WithDataConverter configures application payload serialization. +func WithDataConverter(converter api.DataConverter) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.converter = api.NormalizeDataConverter(converter) + } +} + +// WithOrchestratorNotFoundStrategy configures missing-orchestrator handling. +func WithOrchestratorNotFoundStrategy(strategy OrchestratorNotFoundStrategy) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.orchestratorNotFound = strategy + } +} + +// WithOrchestrationOptions configures deterministic orchestration engine +// policies. It panics when a duration or event limit is negative. +func WithOrchestrationOptions(options OrchestrationOptions) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.orchestrationOptions = normalizeOrchestrationOptions(options) + } +} + +// WithMaximumTimerInterval configures the maximum duration of one physical +// durable timer action. Zero restores [DefaultMaximumTimerInterval]. A negative +// interval panics when the option is applied. +func WithMaximumTimerInterval(interval time.Duration) TaskExecutorOption { + return func(executor *taskExecutor) { + options := executor.orchestrationOptions + options.MaximumTimerInterval = interval + executor.orchestrationOptions = normalizeOrchestrationOptions(options) + } +} + +// WithLogger configures the slog logger exposed to orchestration and activity code. +func WithLogger(logger *slog.Logger) TaskExecutorOption { + return func(executor *taskExecutor) { + if logger != nil { + executor.logger = logger + } + } +} + +// WithMetricsHooks configures optional transport-neutral metric callbacks. +func WithMetricsHooks(hooks MetricsHooks) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.metrics = hooks + } +} + +// WithContextFields configures worker-local fields for activity and entity +// contexts. Root-orchestration fields must use [api.WithContextFields], and +// child fields must use [WithSubOrchestrationContextFields], so they are +// persisted and remain stable across replay and worker deployments. +func WithContextFields(fields api.ContextFields) TaskExecutorOption { + return func(executor *taskExecutor) { + executor.contextFields = make(api.ContextFields, len(fields)) + maps.Copy(executor.contextFields, fields) + } +} + +// NewTaskExecutor returns a [Executor] implementation that executes orchestrator and activity functions in-memory. +func NewTaskExecutor(registry *TaskRegistry, opts ...TaskExecutorOption) Executor { + executor := &taskExecutor{ + Registry: registry, + orchestrationOptions: normalizeOrchestrationOptions(OrchestrationOptions{}), + logger: slog.Default(), + converter: api.DefaultDataConverter(), + } + for _, configure := range opts { + configure(executor) + } + return executor +} + +// ExecuteActivity implements Executor and executes an activity function in the current goroutine. func (te *taskExecutor) ExecuteActivity(ctx context.Context, id api.InstanceID, e *protos.HistoryEvent) (response *protos.HistoryEvent, err error) { ts := e.GetTaskScheduled() if ts == nil { // No clean way to deal with this other than to abandon it return nil, fmt.Errorf("unexpected event type for ExecuteActivity: %v", e.EventType) } - invoker, ok := te.Registry.activities[ts.Name] - if !ok { - // try the wildcard match - invoker, ok = te.Registry.activities["*"] - if !ok { - return helpers.NewTaskFailedEvent(e.EventId, &protos.TaskFailureDetails{ - ErrorType: "TaskActivityNotRegistered", - ErrorMessage: fmt.Sprintf("no task activity named '%s' was registered", ts.Name), - }), nil + activityVersion := ts.GetVersion().GetValue() + if versionErr := te.versioning.check(activityVersion); versionErr != nil && + !allowsUnversionedTask(te.unversionedActivities, ts.GetName(), activityVersion) { + if te.versioning.FailureStrategy == VersionFailureReject { + return nil, versionErr + } + return helpers.NewTaskFailedEvent(e.EventId, versionFailureDetails(versionErr)), nil + } + ctx = api.ContextWithFields(ctx, te.contextFields) + tagInfo, tagFields := contextprop.Decode(ts.GetTags()) + ctx = api.ContextWithFields(ctx, tagFields) + orchestrationInfo, _ := api.OrchestrationContextInfoFromContext(ctx) + if orchestrationInfo.Name == "" { + orchestrationInfo.Name = tagInfo.Name + } + if orchestrationInfo.Version == "" { + orchestrationInfo.Version = tagInfo.Version + } + if orchestrationInfo.ParentInstanceID == "" { + orchestrationInfo.ParentInstanceID = tagInfo.ParentInstanceID + } + if orchestrationInfo.InstanceID == "" { + orchestrationInfo.InstanceID = tagInfo.InstanceID + if orchestrationInfo.InstanceID == "" { + orchestrationInfo.InstanceID = id } } - activityCtx := newTaskActivityContext(ctx, e.EventId, ts) + ctx = api.WithOrchestrationContextInfo(ctx, orchestrationInfo) + ctx = api.WithActivityContextInfo(ctx, api.ActivityContextInfo{ + InstanceID: id, + Name: ts.Name, + Version: ts.GetVersion().GetValue(), + TaskID: e.EventId, + }) + ctx = withActivityLogger(ctx, te.logger) + invoker, ok := te.Registry.getActivity(ts.Name, ts.GetVersion().GetValue()) + if !ok { + notFound := newTaskNotRegisteredError( + activityTaskNotFoundErrorType, + ts.Name, + ts.GetVersion().GetValue(), + ) + return helpers.NewTaskFailedEvent(e.EventId, failure.FromError(notFound, nil)), nil + } + activityCtx := newTaskActivityContext(ctx, e.EventId, ts, te.converter) - // convert panics into activity failures + // Recovery must not re-enter application failure providers or panic causes. defer func() { panicVal := recover() if panicVal != nil { - response = helpers.NewTaskFailedEvent(e.EventId, &protos.TaskFailureDetails{ - ErrorType: "TaskActivityPanic", + details := failure.ToProto(&api.FailureDetails{ + ErrorType: api.ErrorTypeActivityPanic, ErrorMessage: fmt.Sprintf("panic: %v", panicVal), + StackTrace: string(debug.Stack()), }) + response = helpers.NewTaskFailedEvent(e.EventId, details) + err = nil } }() result, err := invoker(activityCtx) if err != nil { - return helpers.NewTaskFailedEvent(e.EventId, &protos.TaskFailureDetails{ - ErrorType: fmt.Sprintf("%T", err), - ErrorMessage: fmt.Sprintf("%+v", err), - }), nil + return helpers.NewTaskFailedEvent(e.EventId, failure.FromError(err, te.errorProperties)), nil } - bytes, err := marshalData(result) + bytes, err := marshalData(te.converter, result) if err != nil { - return helpers.NewTaskFailedEvent(e.EventId, &protos.TaskFailureDetails{ - ErrorType: fmt.Sprintf("%T", err), - ErrorMessage: fmt.Sprintf("%+v", err), - }), nil + return helpers.NewTaskFailedEvent(e.EventId, failure.FromError(err, te.errorProperties)), nil } var rawResult *wrapperspb.StringValue if len(bytes) > 0 { @@ -76,40 +254,374 @@ func (te *taskExecutor) ExecuteActivity(ctx context.Context, id api.InstanceID, return helpers.NewTaskCompletedEvent(e.EventId, rawResult), nil } -// ExecuteOrchestrator implements backend.Executor and executes an orchestrator function in the current goroutine. -func (te *taskExecutor) ExecuteOrchestrator(ctx context.Context, id api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent) (*backend.ExecutionResults, error) { - orchestrationCtx := NewOrchestrationContext(te.Registry, id, oldEvents, newEvents) +// ExecuteOrchestrator implements Executor and executes an orchestrator function in the current goroutine. +func (te *taskExecutor) ExecuteOrchestrator( + _ context.Context, + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, +) (*ExecutionResults, error) { + entitiesSupported, err := validateEntityParameters(entityParameters) + if err != nil { + return nil, err + } + started := startedEvent(oldEvents, newEvents) + name := started.GetName() + version := started.GetVersion().GetValue() + if versionErr := te.versioning.check(version); versionErr != nil && + !allowsUnversionedTask(te.unversionedOrchestrators, name, version) { + if te.versioning.FailureStrategy == VersionFailureReject { + return nil, versionErr + } + return &ExecutionResults{ + Response: &protos.OrchestratorResponse{ + InstanceId: string(id), + Actions: []*protos.OrchestratorAction{ + helpers.NewCompleteOrchestrationAction( + 0, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + nil, + nil, + versionFailureDetails(versionErr), + ), + }, + }, + }, nil + } + if te.orchestratorNotFound == OrchestratorNotFoundReject && name != "" { + if !te.Registry.hasOrchestrator(name, version) { + return nil, newTaskNotRegisteredError( + orchestratorTaskNotFoundErrorType, + name, + version, + ) + } + } + orchestrationCtx := newOrchestrationContext( + te.Registry, + id, + oldEvents, + newEvents, + te.orchestrationOptions, + te.logger, + te.metrics, + te.errorProperties, + te.versioning.defaultVersion(), + te.converter, + entitiesSupported, + ) actions := orchestrationCtx.start() + te.reportHistoryMetric(orchestrationCtx) - results := &backend.ExecutionResults{ + results := &ExecutionResults{ Response: &protos.OrchestratorResponse{ InstanceId: string(id), Actions: actions, CustomStatus: wrapperspb.String(orchestrationCtx.customStatus), }, } + if orchestrationCtx.maxEventsPerTurnExceeded && !orchestrationCtx.hasCompletionAction() { + results.Response.NumEventsProcessed = wrapperspb.Int32(int32(orchestrationCtx.processedEventsThisTurn)) + } return results, nil } +func validateEntityParameters(parameters *protos.OrchestratorEntityParameters) (bool, error) { + if parameters == nil { + return false, nil + } + window := parameters.EntityMessageReorderWindow + if window == nil { + return false, &unsupportedEntityParametersError{ + message: "entity message reorder window must be specified", + } + } + if err := window.CheckValid(); err != nil { + return false, &unsupportedEntityParametersError{ + message: fmt.Sprintf("invalid entity message reorder window: %v", err), + } + } + duration := window.AsDuration() + switch { + case duration < 0: + return false, &unsupportedEntityParametersError{ + message: "entity message reorder window must not be negative", + } + case duration > 0: + return false, &unsupportedEntityParametersError{ + message: "positive entity message reorder windows are not supported by the DTS protocol", + } + default: + return true, nil + } +} + +// ExecuteEntity executes an entity batch in the current goroutine. +func (te *taskExecutor) ExecuteEntity( + ctx context.Context, + req *protos.EntityBatchRequest, +) (result *protos.EntityBatchResult, err error) { + if req == nil { + return nil, fmt.Errorf("entity batch request must not be nil") + } + entityID, err := api.EntityIDFromString(req.InstanceId) + if err != nil { + return nil, fmt.Errorf("invalid entity instance ID: %w", err) + } + if !entityStateIncluded(req.Properties) { + // The scheduler omitted the entity state, so the batch cannot run until + // it is retransmitted. + return &protos.EntityBatchResult{RequiresState: true}, nil + } + factory, ok := te.Registry.getEntityFactory(entityID.Name) + if !ok { + result := &protos.EntityBatchResult{ + EntityState: req.EntityState, + Results: make([]*protos.OperationResult, 0, len(req.Operations)), + } + for range req.Operations { + now := time.Now().UTC() + failureResult := entityOperationFailure( + newTaskNotRegisteredError( + entityTaskNotFoundErrorType, + entityID.Name, + "", + ), + now, + now, + te.errorProperties, + ) + result.Results = append(result.Results, failureResult) + } + return result, nil + } + batch, err := factory(EntityFactoryContext{Context: ctx, ID: entityID}) + if err != nil { + return nil, fmt.Errorf("failed to create entity %q: %w", entityID.Name, err) + } + if batch.Close != nil { + defer func() { + if closeErr := batch.Close(ctx); closeErr != nil { + result = nil + err = fmt.Errorf("failed to close entity batch %q: %w", entityID.Name, closeErr) + } + }() + } + if batch.Entity == nil { + return nil, fmt.Errorf("entity factory %q returned a nil implementation", entityID.Name) + } + + var state entityState + if req.EntityState != nil { + state = entityState{value: []byte(req.EntityState.Value), hasValue: true} + } + result = &protos.EntityBatchResult{ + Results: make([]*protos.OperationResult, 0, len(req.Operations)), + Actions: make([]*protos.OperationAction, 0), + } + nextActionID := int32(0) + for _, operation := range req.Operations { + if operation == nil { + result.Results = append(result.Results, entityOperationFailure( + fmt.Errorf("entity operation request must not be nil"), + time.Now().UTC(), + time.Now().UTC(), + te.errorProperties, + )) + continue + } + startedAt := time.Now().UTC() + isSignal := req.Properties[helpers.EntitySignalProperty(operation.RequestId)].GetBoolValue() + operationCtx := te.newEntityContext(ctx, entityID, operation, state, nextActionID, startedAt, isSignal) + output, operationErr := invokeEntity(batch.Entity, operationCtx) + endedAt := time.Now().UTC() + + var rawResult *wrapperspb.StringValue + if operationErr == nil && !isNilEntityValue(output) { + bytes, marshalErr := marshalData(te.converter, output) + if marshalErr != nil { + operationErr = fmt.Errorf("failed to marshal entity result: %w", marshalErr) + } else if len(bytes) > 0 { + rawResult = wrapperspb.String(string(bytes)) + } + } + + if operationErr != nil { + result.Results = append(result.Results, entityOperationFailure( + operationErr, + startedAt, + endedAt, + te.errorProperties, + )) + continue + } + + state = operationCtx.state + nextActionID = operationCtx.actionIDSeq + result.Actions = append(result.Actions, operationCtx.actions...) + result.Results = append(result.Results, &protos.OperationResult{ + ResultType: &protos.OperationResult_Success{ + Success: &protos.OperationResultSuccess{ + Result: rawResult, + StartTimeUtc: timestamppb.New(startedAt), + EndTimeUtc: timestamppb.New(endedAt), + }, + }, + }) + } + + if state.hasValue { + result.EntityState = wrapperspb.String(string(state.value)) + } + return result, nil +} + +// entityStateIncluded reports whether the scheduler attached entity state to the +// work item. It mirrors .NET's GrpcInstanceRunnerUtils: a missing property, or +// one whose value is not a boolean, means the state was included. +func entityStateIncluded(properties map[string]*structpb.Value) bool { + property, exists := properties["IncludeState"] + if !exists { + return true + } + if _, isBool := property.GetKind().(*structpb.Value_BoolValue); !isBool { + return true + } + return property.GetBoolValue() +} + +func (te *taskExecutor) newEntityContext( + ctx context.Context, + entityID api.EntityID, + operation *protos.OperationRequest, + state entityState, + nextActionID int32, + currentTime time.Time, + isSignal bool, +) *EntityContext { + ctx = api.ContextWithFields(ctx, te.contextFields) + ctx = api.WithEntityContextInfo(ctx, api.EntityContextInfo{ + EntityID: entityID, + Operation: operation.Operation, + RequestID: operation.RequestId, + IsSignal: isSignal, + }) + logger := te.logger + if logger == nil { + logger = slog.Default() + } + logger = logger.With( + slog.String("durabletask.entity.id", entityID.String()), + slog.String("durabletask.entity.operation", operation.Operation), + slog.String("durabletask.entity.request_id", operation.RequestId), + ) + ctx = context.WithValue(ctx, taskLoggerKey{}, logger) + return &EntityContext{ + ID: entityID, + Operation: operation.Operation, + RequestID: operation.RequestId, + IsSignal: isSignal, + rawInput: entityPayload{ + value: []byte(operation.Input.GetValue()), + present: operation.Input != nil, + }, + state: state, + actionIDSeq: nextActionID, + currentTime: currentTime, + ctx: ctx, + logger: logger, + converter: te.converter, + parentTrace: helpers.CloneTraceContext(operation.GetTraceContext()), + } +} + +func invokeEntity(invoker Entity, entityCtx *EntityContext) (result any, err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = newPanicFailureError( + api.ErrorTypeEntityOperationPanic, + fmt.Sprintf("entity operation panic: %v", recovered), + string(debug.Stack()), + panicCause(recovered), + ) + } + }() + return invoker(entityCtx) +} + +func entityOperationFailure( + err error, + startedAt time.Time, + endedAt time.Time, + provider api.ErrorPropertiesProvider, +) *protos.OperationResult { + return &protos.OperationResult{ + ResultType: &protos.OperationResult_Failure{ + Failure: &protos.OperationResultFailure{ + FailureDetails: failure.FromError(err, provider), + StartTimeUtc: timestamppb.New(startedAt), + EndTimeUtc: timestamppb.New(endedAt), + }, + }, + } +} + +func (te *taskExecutor) reportHistoryMetric(ctx *OrchestrationContext) { + if te.metrics.History == nil { + return + } + metric := HistoryMetric{ + InstanceID: ctx.ID, + OrchestrationName: ctx.Name, + OrchestrationVersion: ctx.Version, + HistoryLength: ctx.HistoryLength(), + ProcessedEvents: ctx.processedEventsThisTurn, + HistoryLimitExceeded: ctx.HistoryLimitExceeded(), + } + defer func() { + if recovered := recover(); recovered != nil { + ctx.Logger().Error("history metrics callback panicked", "error", recovered) + } + }() + te.metrics.History(metric) +} + +// startedEvent returns the ExecutionStarted event found across the given event lists, or nil if none is present. +func startedEvent(eventLists ...[]*protos.HistoryEvent) *protos.ExecutionStartedEvent { + for _, events := range eventLists { + for _, event := range events { + if started := event.GetExecutionStarted(); started != nil { + return started + } + } + } + return nil +} + +func versionFailureDetails(err error) *protos.TaskFailureDetails { + return failure.FromError(err) +} + func (te taskExecutor) Shutdown(ctx context.Context) error { // Nothing to do return nil } -func unmarshalData(data []byte, v any) error { - switch { - case v == nil: +func unmarshalData(converter api.DataConverter, data []byte, v any) error { + if v == nil || len(data) == 0 { return nil - case len(data) == 0: - return nil - default: - return json.Unmarshal(data, v) } + return api.NormalizeDataConverter(converter).Deserialize(string(data), v) } -func marshalData(v any) ([]byte, error) { +func marshalData(converter api.DataConverter, v any) ([]byte, error) { if v == nil { return nil, nil } - return json.Marshal(v) + payload, err := api.SerializeData(converter, v) + if err != nil { + return nil, err + } + return []byte(payload), nil } diff --git a/task/executor_contract.go b/task/executor_contract.go new file mode 100644 index 00000000..afb46fd6 --- /dev/null +++ b/task/executor_contract.go @@ -0,0 +1,37 @@ +package task + +import ( + "context" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" +) + +// ExecutionResults carries the orchestrator actions produced by a single +// orchestrator execution turn. It is transport-neutral: the DTS worker sends +// [ExecutionResults.Response] back to the scheduler over gRPC. +type ExecutionResults struct { + Response *protos.OrchestratorResponse +} + +// Executor is the internal collaboration contract between the task runtime and +// the gRPC worker. It is exported only so those packages can share it; its +// internal protobuf parameters intentionally prevent application implementations. +// [NewTaskExecutor] is the implementation used by DTS workers. +type Executor interface { + ExecuteOrchestrator( + ctx context.Context, + iid api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + entityParameters *protos.OrchestratorEntityParameters, + ) (*ExecutionResults, error) + ExecuteActivity(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error) + Shutdown(ctx context.Context) error +} + +// EntityExecutor is implemented by executors that process durable entity +// operation batches. +type EntityExecutor interface { + ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) +} diff --git a/task/external_event_test.go b/task/external_event_test.go new file mode 100644 index 00000000..0ed84da7 --- /dev/null +++ b/task/external_event_test.go @@ -0,0 +1,347 @@ +package task + +import ( + "errors" + "fmt" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestExternalEventNewestLiveWaiterReceivesNextEvent(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-stack", func(ctx *OrchestrationContext) (any, error) { + firstWaiter := ctx.WaitForSingleEvent("signal", -1) + secondWaiter := ctx.WaitForSingleEvent("SIGNAL", -1) + var first, second string + if err := secondWaiter.Await(&second); err != nil { + return nil, err + } + if err := firstWaiter.Await(&first); err != nil { + return nil, err + } + return []string{first, second}, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("event-stack-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("event-stack", string(instanceID), nil, nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event-1"`)), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event-2"`)), + }, + ) + if got, want := completionResult(t, result), `["event-2","event-1"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestExternalEventMultipleLiveWaitersReceiveEventsInReverseAssignmentOrder(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-stack-many", func(ctx *OrchestrationContext) (any, error) { + waiters := []Task{ + ctx.WaitForSingleEvent("signal", -1), + ctx.WaitForSingleEvent("signal", -1), + ctx.WaitForSingleEvent("signal", -1), + } + results := make([]string, len(waiters)) + for i := len(waiters) - 1; i >= 0; i-- { + if err := waiters[i].Await(&results[i]); err != nil { + return nil, err + } + } + return results, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("event-stack-many-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("event-stack-many", string(instanceID), nil, nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event-1"`)), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event-2"`)), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event-3"`)), + }, + ) + if got, want := completionResult(t, result), `["event-3","event-2","event-1"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCanceledExternalEventWaiterDoesNotPoisonDelivery(t *testing.T) { + var captured *OrchestrationContext + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-stack-cancel", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + active := ctx.WaitForSingleEvent("signal", -1) + child, cancel := ctx.WithCancel() + canceled := child.WaitForSingleEvent("signal", -1) + cancel() + if err := canceled.Await(nil); err == nil { + return nil, errors.New("canceled waiter returned no error") + } else if !errors.Is(err, ErrTaskCanceled) { + return nil, fmt.Errorf("canceled waiter returned unexpected error: %w", err) + } + var value string + if err := active.Await(&value); err != nil { + return nil, err + } + return value, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("event-stack-cancel-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent( + "event-stack-cancel", + string(instanceID), + nil, + nil, + nil, + nil, + ) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + if len(firstTurn.Actions) != 0 { + t.Fatalf("first turn actions = %v, want none", firstTurn.Actions) + } + pending := captured.pendingExternalEventTasks["SIGNAL"] + if pending == nil || pending.Len() != 1 { + t.Fatalf("live pending event waiters = %v, want 1", pending) + } + + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{started, executionStarted}, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"delivered"`)), + }, + ) + if got, want := completionResult(t, result), `"delivered"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestExternalEventsBufferedBeforeWaitersRemainFIFO(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-buffer", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(time.Minute).Await(nil); err != nil { + return nil, err + } + var first, second string + if err := ctx.WaitForSingleEvent("signal", -1).Await(&first); err != nil { + return nil, err + } + if err := ctx.WaitForSingleEvent("signal", -1).Await(&second); err != nil { + return nil, err + } + return []string{first, second}, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("event-buffer-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("event-buffer", string(instanceID), nil, nil, nil, nil) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + timer := firstTurn.Actions[0].GetCreateTimer() + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, timer.GetFireAt()), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"buffered-1"`)), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"buffered-2"`)), + helpers.NewTimerFiredEvent(0, timer.GetFireAt(), nil), + }, + ) + if got, want := completionResult(t, result), `["buffered-1","buffered-2"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestTypedEventChannelLiveReceiversUseLIFO(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("typed-event-stack", func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "value") + results := make([]int, 2) + completed := ctx.NewWaitGroup() + completed.Add(len(results)) + for i := range results { + index := i + ctx.Go(func(ctx *OrchestrationContext) { + defer completed.Done() + results[index] = channel.Receive(ctx) + }) + } + completed.Wait(ctx) + return results, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("typed-event-stack-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("typed-event-stack", string(instanceID), nil, nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("value", wrapperspb.String("1")), + helpers.NewEventRaisedEvent("value", wrapperspb.String("2")), + }, + ) + if got, want := completionResult(t, result), `[2,1]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestExternalEventSelectRaceUsesHistoryOrder(t *testing.T) { + permutations := []struct { + name string + order [3]string + expected string + }{ + {name: "event-timeout-cancel", order: [3]string{"event", "timeout", "cancel"}, expected: `"event:value"`}, + {name: "event-cancel-timeout", order: [3]string{"event", "cancel", "timeout"}, expected: `"event:value"`}, + {name: "timeout-event-cancel", order: [3]string{"timeout", "event", "cancel"}, expected: `"timeout"`}, + {name: "timeout-cancel-event", order: [3]string{"timeout", "cancel", "event"}, expected: `"timeout"`}, + {name: "cancel-event-timeout", order: [3]string{"cancel", "event", "timeout"}, expected: `"canceled"`}, + {name: "cancel-timeout-event", order: [3]string{"cancel", "timeout", "event"}, expected: `"canceled"`}, + } + + for _, test := range permutations { + t.Run(test.name, func(t *testing.T) { + var captured *OrchestrationContext + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-select-race", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + channel := NewEventChannel[string](child, "signal") + timer := child.CreateTimer(time.Hour) + selected := "" + completed := ctx.NewWaitGroup() + completed.Add(1) + child.Go(func(child *OrchestrationContext) { + defer completed.Done() + child.Select( + OnEvent(channel, func(value string) { + selected = "event:" + value + }), + OnTask(timer, func(Task) { + selected = "timeout" + }), + ) + }) + ctx.Go(func(ctx *OrchestrationContext) { + if err := ctx.WaitForSingleEvent("cancel", -1).Await(nil); err == nil { + cancel() + } + }) + completed.Wait(ctx) + if selected == "" { + selected = "canceled" + } + return selected, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("event-select-race-" + test.name) + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent( + "event-select-race", + string(instanceID), + nil, + nil, + nil, + nil, + ) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + timer := firstTurn.Actions[0].GetCreateTimer() + events := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent()} + for _, event := range test.order { + switch event { + case "event": + events = append(events, helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"value"`))) + case "timeout": + events = append(events, helpers.NewTimerFiredEvent(0, timer.GetFireAt(), nil)) + case "cancel": + events = append(events, helpers.NewEventRaisedEvent("cancel", nil)) + default: + t.Fatalf("unknown event %q", event) + } + } + + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, timer.GetFireAt()), + }, + events, + ) + if got := completionResult(t, result); got != test.expected { + t.Fatalf("result = %s, want %s", got, test.expected) + } + if len(captured.eventWaiters) != 0 { + t.Fatalf("event subscriptions remain after selection: %v", captured.eventWaiters) + } + }) + } +} diff --git a/task/failure_contract_test.go b/task/failure_contract_test.go new file mode 100644 index 00000000..6d70022d --- /dev/null +++ b/task/failure_contract_test.go @@ -0,0 +1,239 @@ +package task + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +func TestDurableFailureMessagesMatchDotNet(t *testing.T) { + details := &api.FailureDetails{ErrorType: "Example.Error", ErrorMessage: "boom"} + + taskErr := &TaskFailedError{TaskName: "ChargeCard", TaskID: 3, FailureDetails: details} + if got, want := taskErr.Error(), "Task 'ChargeCard' (#3) failed with an unhandled exception: boom"; got != want { + t.Fatalf("TaskFailedError.Error() = %q, want %q", got, want) + } + + entityID := api.NewEntityID("account", "123") + entityErr := &EntityOperationFailedError{ + EntityID: entityID, + OperationName: "withdraw", + FailureDetails: details, + } + want := "Operation 'withdraw' of entity '@account@123' failed: boom" + if got := entityErr.Error(); got != want { + t.Fatalf("EntityOperationFailedError.Error() = %q, want %q", got, want) + } +} + +func TestDurableFailuresMatchWireCategories(t *testing.T) { + notFound := &TaskFailedError{FailureDetails: &api.FailureDetails{ + ErrorType: activityTaskNotFoundErrorType, + ErrorMessage: "missing", + IsNonRetriable: true, + }} + if !errors.Is(notFound, api.ErrTaskNotRegistered) { + t.Fatal("task failure does not match ErrTaskNotRegistered") + } + if !notFound.NonRetriable() { + t.Fatal("task failure should be non-retriable") + } + + version := &TaskFailedError{FailureDetails: &api.FailureDetails{ErrorType: versionMismatchErrorType}} + if !errors.Is(version, api.ErrVersionMismatch) { + t.Fatal("task failure does not match ErrVersionMismatch") + } +} + +func TestLocalTaskErrorsAreNotWrappedAsDurableFailures(t *testing.T) { + ctx := &OrchestrationContext{scope: newCancellationScope(nil)} + task := newTaskInScope(ctx, ctx.scope) + task.failLocal(api.WrapInvalidArgument(errors.New("bad option"))) + + err := task.Await(nil) + if !errors.Is(err, api.ErrInvalidArgument) { + t.Fatalf("Await() error = %v", err) + } + var taskFailure *TaskFailedError + if errors.As(err, &taskFailure) { + t.Fatal("local validation error was wrapped as TaskFailedError") + } +} + +func TestRetryBypassesHandlerForCancellationAndNonRetriableFailure(t *testing.T) { + handlerCalls := 0 + policy := RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: time.Second, + BackoffCoefficient: 1, + MaxRetryInterval: time.Second, + RetryTimeout: time.Minute, + Handle: func(RetryContext) bool { + handlerCalls++ + return true + }, + } + now := time.Now() + nonRetriable := &TaskFailedError{FailureDetails: &api.FailureDetails{ + ErrorType: activityTaskNotFoundErrorType, + ErrorMessage: "missing", + IsNonRetriable: true, + }} + + if delay := computeNextDelay(now, policy, 0, now, nonRetriable); delay != 0 { + t.Fatalf("non-retriable delay = %v", delay) + } + if delay := computeNextDelay(now, policy, 0, now, ErrTaskCanceled); delay != 0 { + t.Fatalf("canceled delay = %v", delay) + } + if handlerCalls != 0 { + t.Fatalf("handler calls = %d", handlerCalls) + } +} + +func TestMissingActivityUsesCanonicalNonRetriableFailure(t *testing.T) { + executor := NewTaskExecutor(NewTaskRegistry()) + event := helpers.NewTaskScheduledEvent(4, "missing", nil, nil, nil) + + result, err := executor.ExecuteActivity(context.Background(), "instance", event) + if err != nil { + t.Fatal(err) + } + details := result.GetTaskFailed().GetFailureDetails() + if details.GetErrorType() != string(activityTaskNotFoundErrorType) { + t.Fatalf("ErrorType = %q", details.GetErrorType()) + } + if !details.GetIsNonRetriable() { + t.Fatal("missing activity failure should be non-retriable") + } +} + +func TestMissingOrchestratorRejectsWorkItem(t *testing.T) { + executor := NewTaskExecutor( + NewTaskRegistry(), + WithOrchestratorNotFoundStrategy(OrchestratorNotFoundReject), + ) + events := []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("missing", "instance", nil, nil, nil, nil), + } + + _, err := executor.ExecuteOrchestrator(context.Background(), "instance", nil, events, nil) + if !errors.Is(err, api.ErrTaskNotRegistered) { + t.Fatalf("ExecuteOrchestrator() error = %v", err) + } +} + +func TestMissingOrchestratorFailsByDefault(t *testing.T) { + executor := NewTaskExecutor(NewTaskRegistry()) + events := []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("missing", "instance", nil, nil, nil, nil), + } + + result, err := executor.ExecuteOrchestrator(context.Background(), "instance", nil, events, nil) + if err != nil { + t.Fatal(err) + } + completed := result.Response.Actions[0].GetCompleteOrchestration() + if completed.GetFailureDetails().GetErrorType() != string(orchestratorTaskNotFoundErrorType) { + t.Fatalf("failure = %#v", completed.GetFailureDetails()) + } + if !completed.GetFailureDetails().GetIsNonRetriable() { + t.Fatal("missing orchestrator failure should be non-retriable") + } +} + +func TestErrorPropertiesProviderEnrichesActivityFailure(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddActivityN("failing", func(ActivityContext) (any, error) { + return nil, errors.New("boom") + }); err != nil { + t.Fatal(err) + } + executor := NewTaskExecutor(registry, WithErrorPropertiesProvider( + api.ErrorPropertiesProviderFunc(func(error) map[string]any { + return map[string]any{"code": "E42"} + }), + )) + event := helpers.NewTaskScheduledEvent(1, "failing", nil, nil, nil) + + result, err := executor.ExecuteActivity(context.Background(), "instance", event) + if err != nil { + t.Fatal(err) + } + if got := result.GetTaskFailed().GetFailureDetails().GetProperties()["code"].GetStringValue(); got != "E42" { + t.Fatalf("code property = %q", got) + } +} + +type hostileActivityPanic struct{} + +func (hostileActivityPanic) Error() string { return strings.Repeat("panic detail ", 4096) } +func (hostileActivityPanic) Unwrap() error { panic("panic cause must not be traversed") } + +func TestActivityPanicRecoveryDoesNotReenterApplicationCode(t *testing.T) { + for _, test := range []struct { + name string + activity Activity + providerCalls int + }{ + { + name: "error properties", + activity: func(ActivityContext) (any, error) { + return nil, errors.New("activity failed") + }, + providerCalls: 1, + }, + { + name: "serialization error properties", + activity: func(ActivityContext) (any, error) { + return make(chan int), nil + }, + providerCalls: 1, + }, + { + name: "activity panic cause", + activity: func(ActivityContext) (any, error) { + panic(hostileActivityPanic{}) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddActivityN("failing", test.activity)) + calls := 0 + executor := NewTaskExecutor(registry, WithErrorPropertiesProvider( + api.ErrorPropertiesProviderFunc(func(error) map[string]any { + calls++ + panic(hostileActivityPanic{}) + }), + )) + var response *protos.HistoryEvent + var err error + require.NotPanics(t, func() { + response, err = executor.ExecuteActivity(context.Background(), "instance", + helpers.NewTaskScheduledEvent(7, "failing", nil, nil, nil)) + }) + require.NoError(t, err) + require.Equal(t, test.providerCalls, calls) + failed := response.GetTaskFailed() + require.NotNil(t, failed) + require.EqualValues(t, 7, failed.TaskScheduledId) + details := failed.GetFailureDetails() + require.Equal(t, string(api.ErrorTypeActivityPanic), details.GetErrorType()) + require.Contains(t, details.GetErrorMessage(), "panic: panic detail") + require.LessOrEqual(t, len(details.GetErrorMessage()), 16*1024) + require.NotEmpty(t, details.GetStackTrace().GetValue()) + require.LessOrEqual(t, len(details.GetStackTrace().GetValue()), 16*1024) + require.Nil(t, details.GetInnerFailure()) + require.Empty(t, details.GetProperties()) + require.False(t, details.GetIsNonRetriable()) + }) + } +} diff --git a/task/finalization_test.go b/task/finalization_test.go new file mode 100644 index 00000000..c0ec130f --- /dev/null +++ b/task/finalization_test.go @@ -0,0 +1,194 @@ +package task + +import ( + "context" + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +type panickingOrchestrationResult struct{} + +func (panickingOrchestrationResult) MarshalJSON() ([]byte, error) { + panic("result serialization panic") +} + +func TestOrchestrationTurnContainsFinalizationPanics(t *testing.T) { + tests := []struct { + name string + orchestrator Orchestrator + options []TaskExecutorOption + message string + providerCalls int + suspended bool + }{ + { + name: "result marshaler", + orchestrator: func(*OrchestrationContext) (any, error) { + return panickingOrchestrationResult{}, nil + }, + message: "result serialization panic", + }, + { + name: "continue as new marshaler", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew(panickingOrchestrationResult{}, WithKeepUnprocessedEvents()) + return nil, nil + }, + message: "result serialization panic", + }, + { + name: "converter", + orchestrator: func(*OrchestrationContext) (any, error) { + return 1, nil + }, + options: []TaskExecutorOption{WithDataConverter(panicConverter{})}, + message: "converter panic", + }, + { + name: "history limit finalization", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }, + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 3, + OnHistoryLimitExceeded: func(HistoryLimitInfo) (any, error) { + return panickingOrchestrationResult{}, nil + }, + })}, + message: "result serialization panic", + }, + { + name: "suspended history limit finalization", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }, + options: []TaskExecutorOption{WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 3, + OnHistoryLimitExceeded: func(HistoryLimitInfo) (any, error) { + return panickingOrchestrationResult{}, nil + }, + })}, + message: "result serialization panic", + suspended: true, + }, + { + name: "failure properties provider", + orchestrator: func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew("unused", WithKeepUnprocessedEvents()) + return nil, errors.New("handler failed") + }, + message: "failure provider panic", + providerCalls: 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + var captured *OrchestrationContext + require.NoError(t, registry.AddOrchestratorN("finalization", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + ctx.CallActivity("pending") + return test.orchestrator(ctx) + })) + providerCalls := 0 + options := append([]TaskExecutorOption(nil), test.options...) + options = append(options, WithErrorPropertiesProvider(api.ErrorPropertiesProviderFunc(func(error) map[string]any { + providerCalls++ + panic("failure provider panic") + }))) + executor := NewTaskExecutor(registry, options...) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("finalization", "instance", nil, nil, nil, nil), + } + if test.suspended { + events = append(events, helpers.NewSuspendOrchestrationEvent("pause")) + } + events = append(events, + helpers.NewEventRaisedEvent("unconsumed", wrapperspb.String("1")), + helpers.NewOrchestratorStartedEvent(), + ) + var result *ExecutionResults + require.NotPanics(t, func() { + var err error + result, err = executor.ExecuteOrchestrator(context.Background(), "instance", nil, events, supportedEntityParameters()) + require.NoError(t, err) + }) + require.Equal(t, test.providerCalls, providerCalls) + require.Nil(t, captured.scheduler) + require.False(t, captured.continuedAsNew) + require.Len(t, result.Response.Actions, 2) + require.NotNil(t, result.Response.Actions[0].GetScheduleTask()) + completed := result.Response.Actions[1].GetCompleteOrchestration() + require.NotNil(t, completed) + require.Equal(t, api.RUNTIME_STATUS_FAILED, completed.OrchestrationStatus) + require.Empty(t, completed.CarryoverEvents) + details := completed.FailureDetails + require.NotNil(t, details) + require.Equal(t, string(api.ErrorTypeOrchestratorPanic), details.ErrorType) + require.Contains(t, details.ErrorMessage, test.message) + require.NotEmpty(t, details.GetStackTrace().GetValue()) + require.LessOrEqual(t, len(details.GetStackTrace().GetValue()), 16*1024) + require.Nil(t, details.InnerFailure) + require.Empty(t, details.Properties) + }) + } +} + +func TestOrchestrationFinalizationPanicReleasesEntityLocks(t *testing.T) { + registry := NewTaskRegistry() + entityID := api.NewEntityID("counter", "locked") + require.NoError(t, registry.AddOrchestratorN("locked-finalization", func(ctx *OrchestrationContext) (any, error) { + if _, err := ctx.LockEntities(entityID); err != nil { + return nil, err + } + ctx.CallActivity("pending") + ctx.ContinueAsNew(panickingOrchestrationResult{}, WithKeepUnprocessedEvents()) + return nil, nil + })) + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("locked-finalization", "instance", nil, nil, nil, nil) + first := executeOrchestrationTurn(t, registry, "instance", nil, []*protos.HistoryEvent{started, executionStarted}) + require.Len(t, first.Actions, 1) + lock := first.Actions[0].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, lock) + + var response *protos.OrchestratorResponse + require.NotPanics(t, func() { + response = executeOrchestrationTurn(t, registry, "instance", []*protos.HistoryEvent{ + started, + executionStarted, + { + EventId: first.Actions[0].Id, + Timestamp: started.Timestamp, + EventType: &protos.HistoryEvent_EntityLockRequested{EntityLockRequested: lock}, + }, + }, []*protos.HistoryEvent{ + { + EventId: -1, + Timestamp: started.Timestamp, + EventType: &protos.HistoryEvent_EntityLockGranted{ + EntityLockGranted: &protos.EntityLockGrantedEvent{CriticalSectionId: lock.CriticalSectionId}, + }, + }, + helpers.NewEventRaisedEvent("unconsumed", wrapperspb.String("1")), + }) + }) + require.Len(t, response.Actions, 3) + require.NotNil(t, response.Actions[0].GetScheduleTask()) + unlock := response.Actions[1].GetSendEntityMessage().GetEntityUnlockSent() + require.NotNil(t, unlock) + require.Equal(t, lock.CriticalSectionId, unlock.CriticalSectionId) + require.Equal(t, entityID.String(), unlock.TargetInstanceId.GetValue()) + completed := response.Actions[2].GetCompleteOrchestration() + require.NotNil(t, completed) + require.Equal(t, api.RUNTIME_STATUS_FAILED, completed.OrchestrationStatus) + require.Empty(t, completed.CarryoverEvents) +} diff --git a/task/history.go b/task/history.go new file mode 100644 index 00000000..c4166f0d --- /dev/null +++ b/task/history.go @@ -0,0 +1,119 @@ +package task + +import ( + "errors" + "fmt" + "time" + + "github.com/microsoft/durabletask-go/api" +) + +// DefaultMaximumTimerInterval is the default maximum duration of one physical +// durable timer action. +const DefaultMaximumTimerInterval = 3 * 24 * time.Hour + +// ErrHistoryLimitExceeded is the sentinel error for deterministic history-budget failures. +var ErrHistoryLimitExceeded = errors.New("orchestration history limit exceeded") + +// HistoryLimitError describes the history budget that was exceeded. +type HistoryLimitError struct { + InstanceID api.InstanceID + HistoryLength int + MaxHistoryEvents int + PolicyError error +} + +func (e *HistoryLimitError) Error() string { + message := fmt.Sprintf( + "%v for instance %q: history=%d max_history=%d", + ErrHistoryLimitExceeded, + e.InstanceID, + e.HistoryLength, + e.MaxHistoryEvents, + ) + if e.PolicyError != nil { + return fmt.Sprintf("%s: history limit handler failed: %v", message, e.PolicyError) + } + return message +} + +func (e *HistoryLimitError) Unwrap() error { + return ErrHistoryLimitExceeded +} + +func (*HistoryLimitError) DurableTaskErrorType() api.ErrorType { + return api.ErrorTypeHistoryLimitExceeded +} + +func (*HistoryLimitError) NonRetriable() bool { + return true +} + +// HistoryLimitInfo is the immutable input passed to a history limit handler. +type HistoryLimitInfo struct { + InstanceID api.InstanceID + OrchestrationName string + OrchestrationVersion string + HistoryLength int + MaxHistoryEvents int + UnprocessedEventCount int + SerializedInput string + Converter api.DataConverter +} + +// GetInput unmarshals the current orchestration input. +func (info HistoryLimitInfo) GetInput(v any) error { + if v == nil || info.SerializedInput == "" { + return nil + } + return api.NormalizeDataConverter(info.Converter).Deserialize(info.SerializedInput, v) +} + +// HistoryLimitHandler supplies safe input for a ContinueAsNew transition. +// It must be deterministic. Unconsumed external events are preserved automatically. +type HistoryLimitHandler func(HistoryLimitInfo) (any, error) + +// OrchestrationOptions configures deterministic orchestration engine policies. +type OrchestrationOptions struct { + // MaximumTimerInterval limits one physical durable timer action. Longer + // timers are split into deterministic sequential actions that retain the + // original deadline. Zero uses [DefaultMaximumTimerInterval]. + // + // Upgrading from a release that did not split long timers is replay-compatible + // once the historical logical deadline has fired. Changing this value between + // two splitting configurations remains replay-breaking for in-flight + // orchestrations that have timers longer than either the old or new value. + MaximumTimerInterval time.Duration + + // MaxEventsPerTurn limits new DTS work-item events processed in one + // execution turn. Orchestration control markers are outside DTS's processed + // event count. Remaining events are left for DTS to redeliver. Zero + // disables the limit. Old replay history is always processed. + MaxEventsPerTurn int + + // MaxHistoryEvents limits the total old and new history supplied to an execution. + // Zero disables the limit. + MaxHistoryEvents int + + // OnHistoryLimitExceeded handles MaxHistoryEvents when the orchestration + // remains incomplete after the turn. It opts into ContinueAsNew by supplying + // serializable state; when nil, the orchestration fails with a non-retriable + // HistoryLimitExceeded failure. + OnHistoryLimitExceeded HistoryLimitHandler +} + +func normalizeOrchestrationOptions(options OrchestrationOptions) OrchestrationOptions { + if options.MaximumTimerInterval < 0 { + panic("maximum timer interval cannot be negative") + } + if options.MaximumTimerInterval == 0 { + options.MaximumTimerInterval = DefaultMaximumTimerInterval + } + if options.MaxEventsPerTurn < 0 { + panic("maximum events per turn cannot be negative") + } + if options.MaxHistoryEvents < 0 { + panic("maximum history events cannot be negative") + } + return options +} diff --git a/task/history_test.go b/task/history_test.go new file mode 100644 index 00000000..0004ecde --- /dev/null +++ b/task/history_test.go @@ -0,0 +1,565 @@ +package task + +import ( + "context" + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestHistoryLimitFailsWithNonRetriableFailureByDefault(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("limited", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("history-limit") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 1}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("limited", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + + completed := completionAction(t, result.Response) + if got := completed.GetOrchestrationStatus(); got != protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { + t.Fatalf("status = %v, want FAILED", got) + } + failure := completed.GetFailureDetails() + if failure.GetErrorType() != "HistoryLimitExceeded" { + t.Fatalf("failure type = %q, want HistoryLimitExceeded", failure.GetErrorType()) + } + if !failure.GetIsNonRetriable() { + t.Fatal("history limit failure must be non-retriable") + } +} + +func TestHistoryLimitHandlerContinuesAsNewAndPreservesExternalEvents(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("limited", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + + handlerCalls := 0 + instanceID := api.InstanceID("history-handler") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 4, + OnHistoryLimitExceeded: func(info HistoryLimitInfo) (any, error) { + handlerCalls++ + var input int + if err := info.GetInput(&input); err != nil { + return nil, err + } + return input + 1, nil + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("limited", string(instanceID), wrapperspb.String("7"), nil, nil, nil), + helpers.NewEventRaisedEvent("first", wrapperspb.String("1")), + helpers.NewEventRaisedEvent("second", wrapperspb.String("2")), + helpers.NewOrchestratorStartedEvent(), + }, nil) + + if err != nil { + t.Fatal(err) + } + if handlerCalls != 1 { + t.Fatalf("handler calls = %d, want 1", handlerCalls) + } + + completed := completionAction(t, result.Response) + if got := completed.GetOrchestrationStatus(); got != protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW { + t.Fatalf("status = %v, want CONTINUED_AS_NEW", got) + } + if got := completed.GetResult().GetValue(); got != "8" { + t.Fatalf("continue-as-new input = %q, want 8", got) + } + if len(completed.GetCarryoverEvents()) != 2 { + t.Fatalf("carryover count = %d, want 2", len(completed.GetCarryoverEvents())) + } + if got := completed.GetCarryoverEvents()[0].GetEventRaised().GetName(); got != "first" { + t.Fatalf("first carryover event = %q, want first", got) + } + if got := completed.GetCarryoverEvents()[1].GetEventRaised().GetName(); got != "second" { + t.Fatalf("second carryover event = %q, want second", got) + } +} + +func TestMaxEventsPerTurnReportsPartialServiceConsumption(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("partial", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("second", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("partial-events") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("partial", string(instanceID), nil, nil, nil, nil), + helpers.NewSuspendOrchestrationEvent("pause"), + helpers.NewResumeOrchestrationEvent("resume"), + helpers.NewEventRaisedEvent("first", wrapperspb.String("1")), + helpers.NewEventRaisedEvent("second", wrapperspb.String("2")), + }, nil) + + if err != nil { + t.Fatal(err) + } + if got := result.Response.GetNumEventsProcessed().GetValue(); got != 1 { + t.Fatalf("numEventsProcessed = %d, want 1 DTS work-item event", got) + } + if len(result.Response.GetActions()) != 0 { + t.Fatalf("partial turn emitted actions: %v", result.Response.GetActions()) + } +} + +func TestMaxEventsPerTurnOmitsCountWhenAllEventsAreConsumed(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("all-consumed", func(*OrchestrationContext) (any, error) { + return "done", nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("all-consumed") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 2}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("all-consumed", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + if result.Response.NumEventsProcessed != nil { + t.Fatalf("numEventsProcessed = %v, want nil for all events", result.Response.NumEventsProcessed) + } +} + +func TestMaxEventsPerTurnSupportsIncrementalRedelivery(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("incremental", func(ctx *OrchestrationContext) (any, error) { + var value string + if err := ctx.WaitForSingleEvent("second", -1).Await(&value); err != nil { + return nil, err + } + return value, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("incremental-events") + allEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("incremental", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("first", wrapperspb.String(`"one"`)), + helpers.NewEventRaisedEvent("second", wrapperspb.String(`"two"`)), + } + executor := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1}), + ) + + first, err := executor.ExecuteOrchestrator(context.Background(), instanceID, nil, allEvents, nil) + if err != nil { + t.Fatal(err) + } + if got := first.Response.GetNumEventsProcessed().GetValue(); got != 1 { + t.Fatalf("first numEventsProcessed = %d, want 1", got) + } + + second, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + allEvents[:2], + allEvents[2:], nil) + + if err != nil { + t.Fatal(err) + } + if got := second.Response.GetNumEventsProcessed().GetValue(); got != 1 { + t.Fatalf("second numEventsProcessed = %d, want 1", got) + } + + third, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + allEvents[:3], + allEvents[3:], nil) + + if err != nil { + t.Fatal(err) + } + if third.Response.NumEventsProcessed != nil { + t.Fatalf("final numEventsProcessed = %v, want nil", third.Response.NumEventsProcessed) + } + completed := completionAction(t, third.Response) + if got := completed.GetResult().GetValue(); got != `"two"` { + t.Fatalf("result = %q, want two", got) + } +} + +func TestOrchestratorCanHandleHistoryLimitExplicitly(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("self-managed", func(ctx *OrchestrationContext) (any, error) { + if !ctx.HistoryLimitExceeded() { + return nil, errors.New("expected history limit to be visible") + } + if ctx.HistoryLength() != 2 { + return nil, errors.New("unexpected history length") + } + ctx.ContinueAsNew("checkpoint", WithKeepUnprocessedEvents()) + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("self-managed-history") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 1}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("self-managed", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + + completed := completionAction(t, result.Response) + if got := completed.GetOrchestrationStatus(); got != protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW { + t.Fatalf("status = %v, want CONTINUED_AS_NEW", got) + } + if got := completed.GetResult().GetValue(); got != `"checkpoint"` { + t.Fatalf("continue-as-new input = %q, want checkpoint", got) + } +} + +func TestHistoryLimitHandlerFailureUsesWellKnownFailure(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("limited", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("history-handler-failure") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 1, + OnHistoryLimitExceeded: func(HistoryLimitInfo) (any, error) { + return nil, errors.New("checkpoint failed") + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("limited", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + + failure := completionAction(t, result.Response).GetFailureDetails() + if failure.GetErrorType() != "HistoryLimitExceeded" || !failure.GetIsNonRetriable() { + t.Fatalf("unexpected failure details: %v", failure) + } + if !errors.Is(&HistoryLimitError{}, ErrHistoryLimitExceeded) { + t.Fatal("HistoryLimitError must unwrap to ErrHistoryLimitExceeded") + } +} + +func TestHistoryMetricReportsTurnUsage(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("metrics", func(*OrchestrationContext) (any, error) { + return nil, nil + }); err != nil { + t.Fatal(err) + } + + var metric HistoryMetric + instanceID := api.InstanceID("history-metric") + _, err := NewTaskExecutor( + registry, + WithMetricsHooks(MetricsHooks{ + History: func(value HistoryMetric) { + metric = value + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("metrics", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + if metric.InstanceID != instanceID || metric.HistoryLength != 2 || metric.ProcessedEvents != 1 { + t.Fatalf("unexpected history metric: %+v", metric) + } +} + +func TestHistoryLimitDoesNotOverrideCompletedOutcome(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("complete-before-limit", func(ctx *OrchestrationContext) (any, error) { + var value string + if err := ctx.WaitForSingleEvent("go", -1).Await(&value); err != nil { + return nil, err + } + + return value, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("complete-before-limit") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 2}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("complete-before-limit", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("go", wrapperspb.String(`"done"`)), + helpers.NewEventRaisedEvent("extra", wrapperspb.String("1")), + }, nil) + + if err != nil { + t.Fatal(err) + } + completed := completionAction(t, result.Response) + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { + t.Fatalf("status = %v, want COMPLETED", completed.GetOrchestrationStatus()) + } + if completed.GetResult().GetValue() != `"done"` { + t.Fatalf("result = %q", completed.GetResult().GetValue()) + } +} + +func TestMaxHistoryLimitDoesNotOverrideCompletedOutcome(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("complete-history-limit", func(*OrchestrationContext) (any, error) { + return "done", nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("complete-history-limit") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 1, + OnHistoryLimitExceeded: func(HistoryLimitInfo) (any, error) { + return "should-not-run", nil + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("complete-history-limit", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + completed := completionAction(t, result.Response) + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED || + completed.GetResult().GetValue() != `"done"` { + t.Fatalf("unexpected completion: %v", completed) + } +} + +func TestHistoryLimitDoesNotOverrideTermination(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("terminate-before-limit", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("terminate-before-limit") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaxEventsPerTurn: 2, + OnHistoryLimitExceeded: func(HistoryLimitInfo) (any, error) { + return "should-not-run", nil + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("terminate-before-limit", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("extra", wrapperspb.String("1")), + helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"killed"`), false), + helpers.NewEventRaisedEvent("after", wrapperspb.String("2")), + }, nil) + + if err != nil { + t.Fatal(err) + } + completed := completionAction(t, result.Response) + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED { + t.Fatalf("status = %v, want TERMINATED", completed.GetOrchestrationStatus()) + } +} + +func TestContinueAsNewCarryoverIncludesUndrainedResumedEvents(t *testing.T) { + resumed := helpers.NewEventRaisedEvent("resumed", wrapperspb.String("1")) + tail := helpers.NewEventRaisedEvent("tail", wrapperspb.String("2")) + ctx := newTestOrchestrationContext(NewTaskRegistry(), "carryover-resumed", nil, []*protos.HistoryEvent{tail}) + ctx.resumedEvents = []replayEvent{{event: resumed}} + events := ctx.unprocessedExternalEvents() + if len(events) != 2 || events[0] != resumed || events[1] != tail { + t.Fatalf("unexpected carryover: %v", events) + } +} + +func TestContinueAsNewCarryoverIncludesSuspendedEvents(t *testing.T) { + suspended := helpers.NewEventRaisedEvent("suspended", wrapperspb.String("1")) + tail := helpers.NewEventRaisedEvent("tail", wrapperspb.String("2")) + ctx := newTestOrchestrationContext(NewTaskRegistry(), "carryover-suspended", nil, []*protos.HistoryEvent{tail}) + ctx.suspendedEvents = []replayEvent{{event: suspended}} + events := ctx.unprocessedExternalEvents() + if len(events) != 2 || events[0] != suspended || events[1] != tail { + t.Fatalf("unexpected carryover: %v", events) + } +} + +func TestMaxHistoryLimitDoesNotOverrideTermination(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("terminated", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("history-terminated") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{MaxHistoryEvents: 1}), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("terminated", string(instanceID), nil, nil, nil, nil), + helpers.NewExecutionTerminatedEvent(wrapperspb.String("stop"), false), + }, nil) + + if err != nil { + t.Fatal(err) + } + if got := completionAction(t, result.Response).GetOrchestrationStatus(); got != protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED { + t.Fatalf("status = %v, want TERMINATED", got) + } +} + +func TestHistoryLimitHandlerFailsWhenCarryoverCannotFit(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("fixed-point", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("never", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("history-fixed-point") + result, err := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaxHistoryEvents: 3, + OnHistoryLimitExceeded: func(info HistoryLimitInfo) (any, error) { + if info.UnprocessedEventCount != 2 { + t.Fatalf("unprocessed event count = %d, want 2", info.UnprocessedEventCount) + } + return "checkpoint", nil + }, + }), + ).ExecuteOrchestrator( + + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("fixed-point", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("one", wrapperspb.String("1")), + helpers.NewEventRaisedEvent("two", wrapperspb.String("2")), + }, nil) + + if err != nil { + t.Fatal(err) + } + completed := completionAction(t, result.Response) + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED || + completed.GetFailureDetails().GetErrorType() != "HistoryLimitExceeded" { + t.Fatalf("unexpected completion: %v", completed) + } +} diff --git a/task/metrics.go b/task/metrics.go new file mode 100644 index 00000000..220fcc94 --- /dev/null +++ b/task/metrics.go @@ -0,0 +1,49 @@ +package task + +import ( + "time" + + "github.com/microsoft/durabletask-go/api" +) + +// WorkItemKind identifies an orchestration, activity, or entity work item. +type WorkItemKind string + +const ( + WorkItemKindOrchestration WorkItemKind = "orchestration" + WorkItemKindActivity WorkItemKind = "activity" + WorkItemKindEntity WorkItemKind = "entity" +) + +// RetryMetric reports a durable task retry that was scheduled. +type RetryMetric struct { + InstanceID api.InstanceID + OrchestrationName string + OrchestrationVersion string + TaskKind WorkItemKind + TaskName string + TaskVersion string + FailedAttempt int + NextAttempt int + MaxAttempts int + Delay time.Duration + ErrorType string + ErrorMessage string +} + +// HistoryMetric reports orchestration history usage for one execution turn. +type HistoryMetric struct { + InstanceID api.InstanceID + OrchestrationName string + OrchestrationVersion string + HistoryLength int + ProcessedEvents int + HistoryLimitExceeded bool +} + +// MetricsHooks contains optional transport-neutral metric callbacks. +// Callbacks must return quickly and must not block worker progress. +type MetricsHooks struct { + Retry func(RetryMetric) + History func(HistoryMetric) +} diff --git a/task/orchestrator.go b/task/orchestrator.go index 8afc3ac6..656c9aef 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -2,60 +2,118 @@ package task import ( "container/list" - "encoding/json" "errors" "fmt" + "log/slog" + "maps" "math" + "runtime/debug" + "sort" "strings" "time" + "github.com/google/uuid" + "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" + "github.com/microsoft/durabletask-go/internal/contextprop" + "github.com/microsoft/durabletask-go/internal/failure" "github.com/microsoft/durabletask-go/internal/helpers" "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/internal/tagcodec" ) // Orchestrator is the functional interface for orchestrator functions. type Orchestrator func(ctx *OrchestrationContext) (any, error) +type replayEvent struct { + event *protos.HistoryEvent + isReplaying bool +} + // OrchestrationContext is the parameter type for orchestrator functions. type OrchestrationContext struct { ID api.InstanceID Name string + Version string IsReplaying bool CurrentTimeUtc time.Time - registry *TaskRegistry - rawInput []byte - oldEvents []*protos.HistoryEvent - newEvents []*protos.HistoryEvent - suspendedEvents []*protos.HistoryEvent - isSuspended bool - historyIndex int - sequenceNumber int32 - pendingActions map[int32]*protos.OrchestratorAction - pendingTasks map[int32]*completableTask - continuedAsNew bool - continuedAsNewInput any - customStatus string + contextFields api.ContextFields + errorProperties api.ErrorPropertiesProvider + orchestrationTags map[string]string + logger *slog.Logger + metrics MetricsHooks + orchestrationOptions OrchestrationOptions + parentInstanceID api.InstanceID + executionID string + registry *TaskRegistry + rawInput []byte + oldEvents []*protos.HistoryEvent + newEvents []*protos.HistoryEvent + suspendedEvents []replayEvent + resumedEvents []replayEvent + isSuspended bool + isTerminated bool + historyIndex int + // processedEventsThisTurn follows DTS numEventsProcessed semantics and + // excludes orchestration control markers. + processedEventsThisTurn int + maxEventsPerTurnExceeded bool + maxHistoryEventsExceeded bool + sequenceNumber int32 + newGuidCounter uint64 + pendingActions map[int32]*protos.OrchestratorAction + pendingTasks map[int32]*completableTask + pendingEntityTasks map[string]*completableTask + continuedAsNew bool + continuedAsNewInput any + continuedAsNewVersion *wrapperspb.StringValue + customStatus string + defaultVersion string + converter api.DataConverter + entitiesSupported bool + scheduler *coroutineScheduler + root *OrchestrationContext + scope *cancellationScope + derived []*OrchestrationContext bufferedExternalEvents map[string]*list.List pendingExternalEventTasks map[string]*list.List + eventChannels map[string]any + eventWaiters map[string]map[*coroutine]struct{} saveBufferedExternalEvents bool + + criticalSectionID string + criticalSectionLocks []string + criticalSectionAvailable map[string]bool + criticalSectionRequestCommitted bool } // callSubOrchestratorOptions is a struct that holds the options for the CallSubOrchestrator orchestrator method. type callSubOrchestratorOptions struct { instanceID string rawInput *wrapperspb.StringValue + version *wrapperspb.StringValue - retryPolicy *RetryPolicy + retryPolicy *RetryPolicy + tags map[string]string + contextFields api.ContextFields +} + +func (options *callSubOrchestratorOptions) versionOrDefault(defaultVersion string) *wrapperspb.StringValue { + if options.version != nil { + return options.version + } + if defaultVersion == "" { + return nil + } + return wrapperspb.String(defaultVersion) } -// subOrchestratorOption is a functional option type for the CallSubOrchestrator orchestrator method. -type subOrchestratorOption func(*callSubOrchestratorOptions) error +// SubOrchestratorOption configures [OrchestrationContext.CallSubOrchestrator]. +type SubOrchestratorOption func(*callSubOrchestratorOptions, api.DataConverter) error // ContinueAsNewOption is a functional option type for the ContinueAsNew orchestrator method. type ContinueAsNewOption func(*OrchestrationContext) @@ -68,13 +126,24 @@ func WithKeepUnprocessedEvents() ContinueAsNewOption { } } +// WithContinueAsNewVersion migrates the next execution to a new orchestration version. +func WithContinueAsNewVersion(version string) ContinueAsNewOption { + return func(ctx *OrchestrationContext) { + if version != "" && strings.TrimSpace(version) == "" { + ctx.continuedAsNewVersion = nil + return + } + ctx.continuedAsNewVersion = wrapperspb.String(version) + } +} + // WithSubOrchestratorInput is a functional option type for the CallSubOrchestrator -// orchestrator method that takes an input value and marshals it to JSON. -func WithSubOrchestratorInput(input any) subOrchestratorOption { - return func(opts *callSubOrchestratorOptions) error { - bytes, err := marshalData(input) +// orchestrator method that serializes an input value with the configured converter. +func WithSubOrchestratorInput(input any) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, converter api.DataConverter) error { + bytes, err := marshalData(converter, input) if err != nil { - return fmt.Errorf("failed to marshal input to JSON: %w", err) + return fmt.Errorf("failed to serialize input: %w", err) } opts.rawInput = wrapperspb.String(string(bytes)) return nil @@ -83,8 +152,8 @@ func WithSubOrchestratorInput(input any) subOrchestratorOption { // WithRawSubOrchestratorInput is a functional option type for the CallSubOrchestrator // orchestrator method that takes a raw input value. -func WithRawSubOrchestratorInput(input string) subOrchestratorOption { - return func(opts *callSubOrchestratorOptions) error { +func WithRawSubOrchestratorInput(input string) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, _ api.DataConverter) error { opts.rawInput = wrapperspb.String(input) return nil } @@ -92,70 +161,280 @@ func WithRawSubOrchestratorInput(input string) subOrchestratorOption { // WithSubOrchestrationInstanceID is a functional option type for the CallSubOrchestrator // orchestrator method that specifies the instance ID of the sub-orchestration. -func WithSubOrchestrationInstanceID(instanceID string) subOrchestratorOption { - return func(opts *callSubOrchestratorOptions) error { +func WithSubOrchestrationInstanceID(instanceID string) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, _ api.DataConverter) error { opts.instanceID = instanceID return nil } } -func WithSubOrchestrationRetryPolicy(policy *RetryPolicy) subOrchestratorOption { - return func(opt *callSubOrchestratorOptions) error { - if policy == nil { - return nil +// WithSubOrchestrationVersion configures the sub-orchestration version. +func WithSubOrchestrationVersion(version string) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, _ api.DataConverter) error { + opts.version = wrapperspb.String(version) + return nil + } +} + +// WithSubOrchestrationTags adds user tags to a sub-orchestration. +func WithSubOrchestrationTags(tags map[string]string) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, _ api.DataConverter) error { + if err := validateUnreservedKeys("sub-orchestration tag", tags); err != nil { + return err + } + opts.tags = maps.Clone(tags) + return nil + } +} + +// WithSubOrchestrationContextFields adds immutable context fields to a +// sub-orchestration. +func WithSubOrchestrationContextFields(fields api.ContextFields) SubOrchestratorOption { + return func(opts *callSubOrchestratorOptions, _ api.DataConverter) error { + if err := validateUnreservedKeys("sub-orchestration context field", fields); err != nil { + return err + } + opts.contextFields = maps.Clone(fields) + return nil + } +} + +// validateUnreservedKeys rejects empty keys and keys that collide with +// reserved wire prefixes, using kind to build caller-facing error messages. +func validateUnreservedKeys[M ~map[string]string](kind string, values M) error { + keys := make([]string, 0, len(values)) + for key := range values { + keys = append(keys, key) + } + sort.Strings(keys) + for _, key := range keys { + if key == "" { + return fmt.Errorf("%s key cannot be empty", kind) + } + if err := checkUnreservedKey(kind, key); err != nil { + return err } - err := policy.Validate() + } + return nil +} + +// checkUnreservedKey rejects caller-supplied keys that collide with the +// reserved prefixes used to carry orchestration context on the wire. +func checkUnreservedKey(kind, key string) error { + if strings.HasPrefix(key, api.ReservedContextFieldPrefix) || + strings.HasPrefix(key, tagcodec.UserTagPrefix) { + return fmt.Errorf("%s %q uses a reserved prefix", kind, key) + } + return nil +} + +// WithSubOrchestrationRetryPolicy snapshots policy when this option is created. +// Later caller mutations do not affect sub-orchestration retries. +func WithSubOrchestrationRetryPolicy(policy *RetryPolicy) SubOrchestratorOption { + if policy == nil { + return func(*callSubOrchestratorOptions, api.DataConverter) error { return nil } + } + snapshot := *policy + return func(opt *callSubOrchestratorOptions, _ api.DataConverter) error { + normalized, err := snapshot.Normalized() if err != nil { return err } - opt.retryPolicy = policy + opt.retryPolicy = &normalized return nil } } -// NewOrchestrationContext returns a new [OrchestrationContext] struct with the specified parameters. -func NewOrchestrationContext(registry *TaskRegistry, id api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent) *OrchestrationContext { - return &OrchestrationContext{ +func newOrchestrationContext( + registry *TaskRegistry, + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, + options OrchestrationOptions, + logger *slog.Logger, + metrics MetricsHooks, + errorProperties api.ErrorPropertiesProvider, + defaultVersion string, + converter api.DataConverter, + entitiesSupported bool, +) *OrchestrationContext { + ctx := &OrchestrationContext{ ID: id, + errorProperties: errorProperties, + logger: logger, + metrics: metrics, + orchestrationOptions: normalizeOrchestrationOptions(options), + defaultVersion: defaultVersion, + converter: api.NormalizeDataConverter(converter), + entitiesSupported: entitiesSupported, registry: registry, oldEvents: oldEvents, newEvents: newEvents, bufferedExternalEvents: make(map[string]*list.List), pendingExternalEventTasks: make(map[string]*list.List), + eventChannels: make(map[string]any), + eventWaiters: make(map[string]map[*coroutine]struct{}), + } + ctx.maxHistoryEventsExceeded = options.MaxHistoryEvents > 0 && + len(oldEvents)+len(newEvents) > options.MaxHistoryEvents + ctx.scope = newCancellationScope(nil) + return ctx +} + +func (ctx *OrchestrationContext) engineContext() *OrchestrationContext { + if ctx.root != nil { + return ctx.root + } + return ctx +} + +func (ctx *OrchestrationContext) syncDerivedContexts() { + active := ctx.derived[:0] + for _, derived := range ctx.derived { + if derived.scope.isCanceled() { + continue + } + derived.ID = ctx.ID + derived.Name = ctx.Name + derived.Version = ctx.Version + derived.IsReplaying = ctx.IsReplaying + derived.CurrentTimeUtc = ctx.CurrentTimeUtc + derived.executionID = ctx.executionID + derived.scheduler = ctx.scheduler + active = append(active, derived) + } + ctx.derived = active +} + +// WithCancel creates a child orchestration context whose tasks, nested scopes, +// and coroutines are canceled together at the next scheduler step. +// A child of an already-canceled scope is canceled immediately. +func (ctx *OrchestrationContext) WithCancel() (*OrchestrationContext, func()) { + engine := ctx.engineContext() + if engine.scheduler == nil { + panic("cancellation scope created outside orchestrator execution") + } + child := &OrchestrationContext{ + ID: engine.ID, + Name: engine.Name, + Version: engine.Version, + IsReplaying: engine.IsReplaying, + CurrentTimeUtc: engine.CurrentTimeUtc, + root: engine, + scope: newCancellationScope(ctx.scope), + scheduler: engine.scheduler, + } + if !child.scope.isCanceled() { + engine.derived = append(engine.derived, child) } + cancel := func() { + scheduler := engine.scheduler + if scheduler == nil { + panic("orchestration cancel called outside orchestrator execution") + } + if scheduler.isStopping() { + return + } + scheduler.mustCurrent() + scheduler.requestCancellation(child.scope) + } + return child, cancel } func (ctx *OrchestrationContext) start() (actions []*protos.OrchestratorAction) { ctx.historyIndex = 0 + ctx.processedEventsThisTurn = 0 ctx.sequenceNumber = 0 + ctx.newGuidCounter = 0 ctx.pendingActions = make(map[int32]*protos.OrchestratorAction) ctx.pendingTasks = make(map[int32]*completableTask) - + ctx.pendingEntityTasks = make(map[string]*completableTask) + ctx.clearCriticalSection() + ctx.scheduler = newCoroutineScheduler(ctx) + defer func() { + ctx.scheduler.shutdown() + ctx.scheduler = nil + }() defer func() { - result := recover() - if resultErr, ok := result.(error); ok && errors.Is(resultErr, ErrTaskBlocked) { - // Expected, normal part of execution + if value := recover(); value != nil { + ctx.continuedAsNew = false + ctx.isSuspended = false + ctx.clearCompletionActions() + // Do not re-enter the converter or failure provider that panicked. + panicErr := newPanicFailureError( + api.ErrorTypeOrchestratorPanic, + fmt.Sprintf("orchestration execution panicked: %v", value), + string(debug.Stack()), + nil, + ) + _ = ctx.setCompleteInternal( + nil, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + failure.FromError(panicErr), + ) actions = ctx.actions() - } else if result != nil { - // Unexpected panic! - panic(result) } }() + terminal := false + markTerminal := func() { + terminal = true + ctx.scheduler.shutdown() + } for { - if ok, err := ctx.processNextEvent(); err != nil { - if setErr := ctx.setFailed(err); setErr != nil { - break + if !terminal && ctx.scheduler.terminalErr != nil { + _ = ctx.setFailed(ctx.scheduler.terminalErr) + markTerminal() + } + + if !terminal && ctx.scheduler.hasRunnable() { + ctx.scheduler.runNext() + if ctx.scheduler.terminalErr != nil { + continue } + if ctx.scheduler.isRootCompleted() && !ctx.scheduler.rootFinalized { + ctx.scheduler.rootFinalized = true + if err := ctx.completeRootCoroutine(); err != nil { + _ = ctx.setFailed(err) + } + markTerminal() + } + continue + } + + ok, err := ctx.processNextEvent() + if err != nil { + _ = ctx.setFailed(err) + markTerminal() break - } else if !ok { - // Orchestrator finished, break out of the loop and return any pending actions + } + if !ok { break } + if ctx.isTerminated && !terminal { + markTerminal() + } + } + if ctx.maxHistoryEventsExceeded && + !ctx.continuedAsNew && + !ctx.isTerminated && + !ctx.hasCompletionAction() { + ctx.enforceHistoryLimit() } return ctx.actions() } +func (ctx *OrchestrationContext) completeRootCoroutine() error { + switch { + case ctx.scheduler.rootErr != nil: + return ctx.setFailed(ctx.scheduler.rootErr) + case ctx.continuedAsNew: + return ctx.setContinuedAsNew() + default: + return ctx.setComplete(ctx.scheduler.rootResult) + } +} + func (ctx *OrchestrationContext) processNextEvent() (bool, error) { e, ok := ctx.getNextHistoryEvent() if !ok { @@ -171,6 +450,13 @@ func (ctx *OrchestrationContext) processNextEvent() (bool, error) { } func (ctx *OrchestrationContext) getNextHistoryEvent() (*protos.HistoryEvent, bool) { + if len(ctx.resumedEvents) > 0 { + next := ctx.resumedEvents[0] + ctx.resumedEvents = ctx.resumedEvents[1:] + ctx.IsReplaying = next.isReplaying + return next.event, true + } + var historyList []*protos.HistoryEvent index := ctx.historyIndex switch { @@ -185,15 +471,55 @@ func (ctx *OrchestrationContext) getNextHistoryEvent() (*protos.HistoryEvent, bo index -= len(ctx.oldEvents) } + event := historyList[index] + if !ctx.IsReplaying && countsTowardTurnBudget(event) { + if ctx.orchestrationOptions.MaxEventsPerTurn > 0 && + ctx.processedEventsThisTurn >= ctx.orchestrationOptions.MaxEventsPerTurn { + ctx.maxEventsPerTurnExceeded = true + return nil, false + } + ctx.processedEventsThisTurn++ + } + ctx.historyIndex++ - e := historyList[index] - return e, true + return event, true +} + +func countsTowardTurnBudget(event *protos.HistoryEvent) bool { + return event.GetOrchestratorStarted() == nil && + event.GetOrchestratorCompleted() == nil && + event.GetGenericEvent() == nil && + event.GetExecutionRewound() == nil && + event.GetExecutionTerminated() == nil && + event.GetExecutionSuspended() == nil && + event.GetExecutionResumed() == nil +} + +// HistoryLength returns the total old and new history supplied to this execution. +func (ctx *OrchestrationContext) HistoryLength() int { + engine := ctx.engineContext() + return len(engine.oldEvents) + len(engine.newEvents) } -func (ctx *OrchestrationContext) processEvent(e *backend.HistoryEvent) error { +// HistoryLimitExceeded reports whether MaxHistoryEvents was exceeded. +func (ctx *OrchestrationContext) HistoryLimitExceeded() bool { + engine := ctx.engineContext() + return engine.maxHistoryEventsExceeded +} + +func (ctx *OrchestrationContext) processEvent(e *protos.HistoryEvent) error { + defer ctx.syncDerivedContexts() // Buffer certain events if we're in a suspended state if ctx.isSuspended && (e.GetExecutionResumed() == nil && e.GetExecutionTerminated() == nil) { - ctx.suspendedEvents = append(ctx.suspendedEvents, e) + if e.GetExecutionSuspended() != nil { + // A redundant suspend is a no-op. Buffering it would re-suspend the + // orchestration when the matching resume drains the buffer. + return nil + } + ctx.suspendedEvents = append(ctx.suspendedEvents, replayEvent{ + event: e, + isReplaying: ctx.IsReplaying, + }) return nil } @@ -219,6 +545,22 @@ func (ctx *OrchestrationContext) processEvent(e *backend.HistoryEvent) error { err = ctx.onTimerCreated(e) } else if tf := e.GetTimerFired(); tf != nil { err = ctx.onTimerFired(tf) + } else if es := e.GetEventSent(); es != nil { + err = ctx.onEventSent(e.EventId, es) + } else if signaled := e.GetEntityOperationSignaled(); signaled != nil { + err = ctx.onEntityOperationSent(e.EventId, signaled.RequestId) + } else if called := e.GetEntityOperationCalled(); called != nil { + err = ctx.onEntityOperationSent(e.EventId, called.RequestId) + } else if completed := e.GetEntityOperationCompleted(); completed != nil { + err = ctx.onEntityOperationCompleted(completed) + } else if failed := e.GetEntityOperationFailed(); failed != nil { + err = ctx.onEntityOperationFailed(failed) + } else if requested := e.GetEntityLockRequested(); requested != nil { + err = ctx.onEntityLockRequested(e.EventId, requested) + } else if granted := e.GetEntityLockGranted(); granted != nil { + err = ctx.onEntityLockGranted(granted) + } else if unlocked := e.GetEntityUnlockSent(); unlocked != nil { + err = ctx.onEntityUnlockSent(e.EventId, unlocked) } else if er := e.GetEventRaised(); er != nil { err = ctx.onExternalEventRaised(e) } else if es := e.GetExecutionSuspended(); es != nil { @@ -229,149 +571,401 @@ func (ctx *OrchestrationContext) processEvent(e *backend.HistoryEvent) error { err = ctx.onExecutionTerminated(et) } else if oc := e.GetOrchestratorCompleted(); oc != nil { // Nothing to do + } else if e.GetGenericEvent() != nil || e.GetExecutionRewound() != nil { + // Service control markers are replay no-ops. } else { err = fmt.Errorf("don't know how to handle event: %v", e) } return err } +// SetCustomStatus stores a raw, pre-serialized custom status string. +// Use SetCustomStatusValue to apply the configured data converter. func (octx *OrchestrationContext) SetCustomStatus(cs string) { - octx.customStatus = cs + octx.engineContext().customStatus = cs +} + +// SetCustomStatusValue serializes and stores a typed custom status value. +func (octx *OrchestrationContext) SetCustomStatusValue(value any) error { + engine := octx.engineContext() + payload, err := api.SerializeData(engine.converter, value) + if err != nil { + return fmt.Errorf("failed to serialize custom status: %w", err) + } + engine.customStatus = payload + return nil +} + +// SetRawCustomStatus stores a pre-serialized custom status value. +func (octx *OrchestrationContext) SetRawCustomStatus(payload string) { + octx.engineContext().customStatus = payload +} + +var guidNamespace = uuid.MustParse("9e952958-5e33-4daf-827f-2fa12937b875") + +// NewGuid returns a deterministic UUID that is stable across orchestration replay. +func (ctx *OrchestrationContext) NewGuid() string { + engine := ctx.engineContext() + timestamp := engine.CurrentTimeUtc.UTC().Format("2006-01-02T15:04:05.0000000Z") + name := fmt.Sprintf("%s_%s_%d", engine.ID, timestamp, engine.newGuidCounter) + engine.newGuidCounter++ + return uuid.NewSHA1(guidNamespace, []byte(name)).String() } // GetInput unmarshals the serialized orchestration input and stores it in [v]. func (octx *OrchestrationContext) GetInput(v any) error { - return unmarshalData(octx.rawInput, v) + engine := octx.engineContext() + return unmarshalData(engine.converter, engine.rawInput, v) } // CallActivity schedules an asynchronous invocation of an activity function. The [activity] // parameter can be either the name of an activity as a string or can be a pointer to the function // that implements the activity, in which case the name is obtained via reflection. -func (ctx *OrchestrationContext) CallActivity(activity any, opts ...callActivityOption) Task { +func (ctx *OrchestrationContext) CallActivity(activity any, opts ...CallActivityOption) Task { + engine := ctx.engineContext() options := new(callActivityOptions) for _, configure := range opts { - if err := configure(options); err != nil { - failedTask := newTask(ctx) - failedTask.fail(helpers.NewTaskFailureDetails(err)) - return failedTask + if err := configure(options, engine.converter); err != nil { + return ctx.newFailedTask(engine, api.WrapInvalidArgument(err)) } } + if ctx.scope.isCanceled() { + return newTaskInScope(engine, ctx.scope) + } if options.retryPolicy != nil { - return ctx.internalScheduleTaskWithRetries(ctx.CurrentTimeUtc, func() Task { - return ctx.internalScheduleActivity(activity, options) - }, *options.retryPolicy, 0) + retryInfo := retryTaskInfo{ + kind: WorkItemKindActivity, + name: helpers.GetTaskFunctionName(activity), + version: options.versionOrInherited(engine.Version).GetValue(), + } + return engine.internalScheduleTaskWithRetries(engine.CurrentTimeUtc, func() Task { + return engine.internalScheduleActivity(activity, options, ctx.scope) + }, *options.retryPolicy, 0, ctx, retryInfo) + } + + return engine.internalScheduleActivity(activity, options, ctx.scope) +} + +// newFailedTask creates a completed, already-failed task in the calling context's +// cancellation scope. Used when option validation fails before any action is scheduled. +func (ctx *OrchestrationContext) newFailedTask(engine *OrchestrationContext, err error) Task { + failedTask := newTaskInScope(engine, ctx.scope) + failedTask.failLocal(err) + return failedTask +} + +// Go starts a coroutine that is cooperatively scheduled with the orchestration. +// Only one orchestration coroutine runs at a time, in monotonically increasing ID order. +func (ctx *OrchestrationContext) Go(fn func(ctx *OrchestrationContext)) { + if fn == nil { + panic("orchestration coroutine function must be non-nil") } + engine := ctx.engineContext() + if engine.scheduler == nil { + panic("orchestration coroutine started outside orchestrator execution") + } + engine.scheduler.spawn(ctx, func() { + fn(ctx) + }) +} - return ctx.internalScheduleActivity(activity, options) +// NewWaitGroup creates a deterministic coroutine wait group. +func (ctx *OrchestrationContext) NewWaitGroup() WaitGroup { + engine := ctx.engineContext() + if engine.scheduler == nil { + panic("orchestration wait group created outside orchestrator execution") + } + return newOrchestrationWaitGroup(engine.scheduler) } -func (ctx *OrchestrationContext) internalScheduleActivity(activity any, options *callActivityOptions) Task { +func (ctx *OrchestrationContext) internalScheduleActivity( + activity any, + options *callActivityOptions, + scope *cancellationScope, +) Task { + if scope.isCanceled() { + return newTaskInScope(ctx, scope) + } scheduleTaskAction := helpers.NewScheduleTaskAction( ctx.getNextSequenceNumber(), helpers.GetTaskFunctionName(activity), - options.rawInput) + options.rawInput, + options.versionOrInherited(ctx.Version)) + scheduleTaskAction.GetScheduleTask().Tags = contextprop.Encode(api.OrchestrationContextInfo{ + InstanceID: ctx.ID, + Name: ctx.Name, + Version: ctx.Version, + ParentInstanceID: ctx.parentInstanceID, + }, ctx.contextFields, mergeStringMaps(ctx.orchestrationTags, options.tags)) ctx.pendingActions[scheduleTaskAction.Id] = scheduleTaskAction - task := newTask(ctx) + task := newTaskInScope(ctx, scope) + task.taskName = scheduleTaskAction.GetScheduleTask().GetName() + task.taskVersion = scheduleTaskAction.GetScheduleTask().GetVersion().GetValue() + task.taskID = scheduleTaskAction.Id ctx.pendingTasks[scheduleTaskAction.Id] = task return task } -func (ctx *OrchestrationContext) CallSubOrchestrator(orchestrator any, opts ...subOrchestratorOption) Task { +func (ctx *OrchestrationContext) CallSubOrchestrator(orchestrator any, opts ...SubOrchestratorOption) Task { + engine := ctx.engineContext() + if engine.criticalSectionID != "" { + return ctx.newFailedTask(engine, fmt.Errorf("sub-orchestrations cannot be started while holding entity locks")) + } options := new(callSubOrchestratorOptions) for _, configure := range opts { - if err := configure(options); err != nil { - failedTask := newTask(ctx) - failedTask.fail(helpers.NewTaskFailureDetails(err)) - return failedTask + if err := configure(options, engine.converter); err != nil { + return ctx.newFailedTask(engine, api.WrapInvalidArgument(err)) } } + if ctx.scope.isCanceled() { + return newTaskInScope(engine, ctx.scope) + } if options.retryPolicy != nil { - return ctx.internalScheduleTaskWithRetries(ctx.CurrentTimeUtc, func() Task { - return ctx.internalCallSubOrchestrator(orchestrator, options) - }, *options.retryPolicy, 0) + retryInfo := retryTaskInfo{ + kind: WorkItemKindOrchestration, + name: helpers.GetTaskFunctionName(orchestrator), + version: options.versionOrDefault(engine.defaultVersion).GetValue(), + } + return engine.internalScheduleTaskWithRetries(engine.CurrentTimeUtc, func() Task { + return engine.internalCallSubOrchestrator(orchestrator, options, ctx.scope) + }, *options.retryPolicy, 0, ctx, retryInfo) } - return ctx.internalCallSubOrchestrator(orchestrator, options) + return engine.internalCallSubOrchestrator(orchestrator, options, ctx.scope) } -func (ctx *OrchestrationContext) internalCallSubOrchestrator(orchestrator any, options *callSubOrchestratorOptions) Task { +func (ctx *OrchestrationContext) internalCallSubOrchestrator( + orchestrator any, + options *callSubOrchestratorOptions, + scope *cancellationScope, +) Task { + if scope.isCanceled() { + return newTaskInScope(ctx, scope) + } createSubOrchestrationAction := helpers.NewCreateSubOrchestrationAction( ctx.getNextSequenceNumber(), helpers.GetTaskFunctionName(orchestrator), options.instanceID, options.rawInput, + options.versionOrDefault(ctx.defaultVersion), ) + createSubOrchestrationAction.GetCreateSubOrchestration().Tags = contextprop.Encode( + api.OrchestrationContextInfo{ + InstanceID: ctx.ID, + Name: ctx.Name, + Version: ctx.Version, + ParentInstanceID: ctx.parentInstanceID, + }, + mergeStringMaps(ctx.contextFields, options.contextFields), + mergeStringMaps(ctx.orchestrationTags, options.tags), + ) + if createSubOrchestrationAction.GetCreateSubOrchestration().GetInstanceId() == "" { + createSubOrchestrationAction.GetCreateSubOrchestration().InstanceId = fmt.Sprintf( + "%s:%04x", + ctx.ID, + createSubOrchestrationAction.Id, + ) + } ctx.pendingActions[createSubOrchestrationAction.Id] = createSubOrchestrationAction - task := newTask(ctx) + task := newTaskInScope(ctx, scope) + task.taskName = createSubOrchestrationAction.GetCreateSubOrchestration().GetName() + task.taskVersion = createSubOrchestrationAction.GetCreateSubOrchestration().GetVersion().GetValue() + task.taskID = createSubOrchestrationAction.Id ctx.pendingTasks[createSubOrchestrationAction.Id] = task return task } -func (ctx *OrchestrationContext) internalScheduleTaskWithRetries(initialAttempt time.Time, schedule func() Task, policy RetryPolicy, retryCount int) Task { - return &taskWrapper{ - delegate: schedule(), - onAwaitResult: func(v any, err error) error { +func (ctx *OrchestrationContext) internalScheduleTaskWithRetries( + initialAttempt time.Time, + schedule func() Task, + policy RetryPolicy, + retryCount int, + owner *OrchestrationContext, + retryInfo retryTaskInfo, +) Task { + result := newTaskInScope(ctx, owner.scope) + attempt := schedule() + ctx.scheduler.spawn(owner, func() { + current := attempt + count := retryCount + for { + err := current.Await(nil) if err == nil { - return nil + result.completeFrom(current, nil) + return } - - if retryCount+1 >= policy.MaxAttempts { - // next try will exceed the max attempts, dont continue - return err + if count+1 >= policy.MaxAttempts { + result.completeFrom(current, err) + return } - nextDelay := computeNextDelay(ctx.CurrentTimeUtc, policy, retryCount, initialAttempt, err) + nextDelay := computeNextDelay(ctx.CurrentTimeUtc, policy, count, initialAttempt, err) if nextDelay == 0 { - return err + result.completeFrom(current, err) + return } - - timerErr := ctx.createTimerInternal(nextDelay).Await(nil) - if timerErr != nil { - return errors.Join(timerErr, err) + ctx.reportRetry(retryInfo, count+1, policy, nextDelay, err) + if timerErr := ctx.createTimerInternal(nextDelay, owner.scope).Await(nil); timerErr != nil { + if errors.Is(timerErr, ErrTaskCanceled) { + result.cancel() + } else { + result.failLocal(errors.Join(timerErr, err)) + } + return } + count++ + current = schedule() + } + }) + return result +} - err = ctx.internalScheduleTaskWithRetries(initialAttempt, schedule, policy, retryCount+1).Await(v) - if err == nil { - return nil - } - return err - }, +func (t *completableTask) completeFrom(source Task, fallback error) { + state, ok := taskState(source) + if !ok { + if fallback != nil { + t.failLocal(fallback) + } else { + t.complete(nil) + } + return + } + t.taskName = state.taskName + t.taskVersion = state.taskVersion + t.taskID = state.taskID + t.entityID = state.entityID + t.entityOperation = state.entityOperation + switch { + case state.failureDetails != nil: + t.fail(state.failureDetails) + case state.localErr != nil: + t.failLocal(state.localErr) + case state.isCanceled: + t.cancel() + default: + t.complete(state.rawResult) } } +// computeNextDelay returns the delay before the next retry attempt, or zero to +// stop retrying. Every input is derived from replayed history, so the same +// failure event always yields the same decision. A delay that would push the +// next attempt past RetryTimeout stops the retries instead. func computeNextDelay(currentTimeUtc time.Time, policy RetryPolicy, attempt int, firstAttempt time.Time, err error) time.Duration { - if policy.Handle(err) { - isExpired := false - if policy.RetryTimeout != math.MaxInt64 { - isExpired = currentTimeUtc.After(firstAttempt.Add(policy.RetryTimeout)) - } - if !isExpired { - nextDelayMs := float64(policy.InitialRetryInterval.Milliseconds()) * math.Pow(policy.BackoffCoefficient, float64(attempt)) - if nextDelayMs < float64(policy.MaxRetryInterval.Milliseconds()) { - return time.Duration(int64(nextDelayMs) * int64(time.Millisecond)) - } - return policy.MaxRetryInterval + if errors.Is(err, ErrTaskCanceled) { + return 0 + } + details := failureDetailsFromError(err) + if details == nil || details.NonRetriable() { + return 0 + } + totalRetryTime := currentTimeUtc.Sub(firstAttempt) + if totalRetryTime < 0 { + totalRetryTime = 0 + } + remaining := time.Duration(math.MaxInt64) + if policy.RetryTimeout != math.MaxInt64 { + remaining = firstAttempt.Add(policy.RetryTimeout).Sub(currentTimeUtc) + if remaining <= 0 { + return 0 + } + } + if policy.Handle != nil && !policy.Handle(RetryContext{ + LastAttemptNumber: attempt + 1, + LastFailure: details, + TotalRetryTime: totalRetryTime, + }) { + return 0 + } + nextDelay := min(policy.InitialRetryInterval, policy.MaxRetryInterval) + if attempt != 0 && policy.BackoffCoefficient != 1 { + scaled := float64(policy.InitialRetryInterval) * math.Pow(policy.BackoffCoefficient, float64(attempt)) + nextDelay = policy.MaxRetryInterval + // Cap before converting to avoid overflow; a positive backoff must not + // underflow to the zero sentinel that stops retries. + if scaled < float64(nextDelay) { + nextDelay = max(time.Nanosecond, time.Duration(scaled)) } } - return 0 + if nextDelay > remaining { + return 0 + } + return nextDelay } // CreateTimer schedules a durable timer that expires after the specified delay. func (ctx *OrchestrationContext) CreateTimer(delay time.Duration) Task { - return ctx.createTimerInternal(delay) + engine := ctx.engineContext() + if ctx.scope.isCanceled() { + return newTaskInScope(engine, ctx.scope) + } + return engine.createTimerInternal(delay, ctx.scope) +} + +func (ctx *OrchestrationContext) createTimerInternal( + delay time.Duration, + scope *cancellationScope, +) *completableTask { + if scope.isCanceled() { + return newTaskInScope(ctx, scope) + } + + logicalTimer := newTaskInScope(ctx, scope) + deadline := ctx.CurrentTimeUtc.Add(delay) + if err := timestamppb.New(deadline).CheckValid(); err != nil { + logicalTimer.failLocal(api.WrapInvalidArgument(fmt.Errorf("timer deadline %v is out of range: %w", deadline, err))) + return logicalTimer + } + + var scheduleNextChunk func() + scheduleNextChunk = func() { + if logicalTimer.isCompleted { + return + } + + fireAt := deadline + isFinalChunk := true + maximumInterval := ctx.orchestrationOptions.MaximumTimerInterval + if deadline.Sub(ctx.CurrentTimeUtc) > maximumInterval { + fireAt = ctx.CurrentTimeUtc.Add(maximumInterval) + isFinalChunk = false + } + chunk := ctx.createTimerAction(fireAt, scope) + chunk.onCompleted(func() { + if logicalTimer.isCompleted { + return + } + // A pre-splitting worker recorded the logical deadline as one timer. + // If replay reaches that historical TimerFired event, CurrentTimeUtc + // is already at or beyond the deadline and no intermediate chunk may + // consume the next sequence number. + if isFinalChunk || + (!chunk.timerFireAt.IsZero() && !deadline.After(chunk.timerFireAt)) || + !deadline.After(ctx.CurrentTimeUtc) || + chunk.isCanceled || chunk.localErr != nil || chunk.failureDetails != nil { + logicalTimer.completeFrom(chunk, nil) + return + } + scheduleNextChunk() + }) + } + scheduleNextChunk() + return logicalTimer } -func (ctx *OrchestrationContext) createTimerInternal(delay time.Duration) *completableTask { - fireAt := ctx.CurrentTimeUtc.Add(delay) +func (ctx *OrchestrationContext) createTimerAction( + fireAt time.Time, + scope *cancellationScope, +) *completableTask { timerAction := helpers.NewCreateTimerAction(ctx.getNextSequenceNumber(), fireAt) ctx.pendingActions[timerAction.Id] = timerAction - task := newTask(ctx) + task := newTaskInScope(ctx, scope) ctx.pendingTasks[timerAction.Id] = task return task } @@ -386,97 +980,282 @@ func (ctx *OrchestrationContext) createTimerInternal(delay time.Duration) *compl // wait indefinitely for the event to be received. // // Orchestrators can wait for the same event name multiple times, so waiting for multiple events with the same name -// is allowed. Each event received by an orchestrator will complete just one task returned by this method. +// is allowed. Each event received by an orchestrator will complete just one task returned by this method. Live +// waiters use LIFO ordering, so the newest waiter receives the next event. Events buffered before any waiter exists +// retain FIFO arrival order. This ordering is part of the deterministic replay contract. // // Note that event names are case-insensitive. func (ctx *OrchestrationContext) WaitForSingleEvent(eventName string, timeout time.Duration) Task { - task := newTask(ctx) + engine := ctx.engineContext() + task := newTaskInScope(engine, ctx.scope) + if ctx.scope.isCanceled() { + return task + } key := strings.ToUpper(eventName) - if eventList, ok := ctx.bufferedExternalEvents[key]; ok { + if buffered, ok := engine.takeBufferedEvent(key); ok { // An event with this name arrived already and can be consumed immediately. - next := eventList.Front() - if eventList.Len() > 1 { - eventList.Remove(next) - } else { - delete(ctx.bufferedExternalEvents, key) - } - rawValue := []byte(next.Value.(*protos.HistoryEvent).GetEventRaised().GetInput().GetValue()) - task.complete(rawValue) + task.complete([]byte(buffered.event.GetEventRaised().GetInput().GetValue())) } else if timeout == 0 { // Zero-timeout means fail immediately if the event isn't already buffered. task.cancel() } else { // Keep a reference to this task so we can complete it when the event of this name arrives - var taskList *list.List - var ok bool - if taskList, ok = ctx.pendingExternalEventTasks[key]; !ok { + taskList, ok := engine.pendingExternalEventTasks[key] + if !ok { taskList = list.New() - ctx.pendingExternalEventTasks[key] = taskList + engine.pendingExternalEventTasks[key] = taskList } taskElement := taskList.PushBack(task) + task.onCompleted(func() { + engine.removePendingEventTask(key, taskElement) + }) if timeout > 0 { - ctx.createTimerInternal(timeout).onCompleted(func() { + engine.createTimerInternal(timeout, ctx.scope).onCompleted(func() { task.cancel() - if taskList.Len() > 1 { - taskList.Remove(taskElement) - } else { - delete(ctx.pendingExternalEventTasks, key) - } }) } } return task } -func (ctx *OrchestrationContext) ContinueAsNew(newInput any, options ...ContinueAsNewOption) { - ctx.continuedAsNew = true - ctx.continuedAsNewInput = newInput - for _, option := range options { - option(ctx) +// CallEntity sends an operation request to an entity and waits for its response. +func (ctx *OrchestrationContext) CallEntity(entityID api.EntityID, operationName string, opts ...callEntityOption) Task { + engine := ctx.engineContext() + if engine.isTerminated || ctx.scope.isCanceled() { + task := newTaskInScope(engine, ctx.scope) + task.cancel() + return task } + if !engine.entitiesSupported { + return ctx.newFailedTask(engine, errEntitiesUnsupported) + } + options := new(callEntityOptions) + for _, configure := range opts { + if err := configure(options, engine.converter); err != nil { + return ctx.newFailedTask(engine, api.WrapInvalidArgument(err)) + } + } + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return ctx.newFailedTask(engine, api.WrapInvalidArgument(err)) + } + if operationName == "" { + return ctx.newFailedTask(engine, api.WrapInvalidArgument(errors.New("entity operation name must not be empty"))) + } + entityKey := entityID.String() + if engine.criticalSectionID != "" { + if engine.criticalSectionAvailable == nil { + return ctx.newFailedTask(engine, fmt.Errorf("entity lock acquisition is still pending")) + } + available, locked := engine.criticalSectionAvailable[entityKey] + if !locked { + return ctx.newFailedTask(engine, fmt.Errorf("entity %s is not part of the current critical section", entityKey)) + } + if !available { + return ctx.newFailedTask(engine, fmt.Errorf("entity %s already has an outstanding call in the current critical section", entityKey)) + } + engine.criticalSectionAvailable[entityKey] = false + } + + requestID := ctx.NewGuid() + action := helpers.NewEntityOperationCalledAction( + engine.getNextSequenceNumber(), + requestID, + entityKey, + string(engine.ID), + engine.executionID, + operationName, + options.rawInput, + ) + engine.pendingActions[action.Id] = action + task := newTaskInScope(engine, ctx.scope) + task.entityID = entityID + task.entityOperation = operationName + engine.pendingEntityTasks[requestID] = task + if sectionID := engine.criticalSectionID; sectionID != "" { + task.onCompleted(func() { + if engine.criticalSectionID == sectionID && engine.criticalSectionAvailable != nil { + engine.criticalSectionAvailable[entityKey] = true + } + }) + } + return task } -func (ctx *OrchestrationContext) onExecutionStarted(es *protos.ExecutionStartedEvent) error { - orchestrator, ok := ctx.registry.orchestrators[es.Name] - if !ok { - // try looking for a "default" orchestrator - orchestrator, ok = ctx.registry.orchestrators["*"] - if !ok { - return fmt.Errorf("orchestrator named '%s' is not registered", es.Name) +// SignalEntity sends a fire-and-forget entity operation. +func (ctx *OrchestrationContext) SignalEntity(entityID api.EntityID, operationName string, opts ...signalEntityOption) error { + engine := ctx.engineContext() + if engine.isTerminated || ctx.scope.isCanceled() { + return ErrTaskCanceled + } + if !engine.entitiesSupported { + return errEntitiesUnsupported + } + options := new(signalEntityOptions) + for _, configure := range opts { + if err := configure(options, engine.converter); err != nil { + return api.WrapInvalidArgument(err) } } - ctx.Name = es.Name - if es.Input != nil { - ctx.rawInput = []byte(es.Input.Value) + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return api.WrapInvalidArgument(err) + } + if operationName == "" { + return api.WrapInvalidArgument(errors.New("entity operation name must not be empty")) + } + entityKey := entityID.String() + if engine.criticalSectionID != "" && engine.criticalSectionAvailable != nil { + if _, locked := engine.criticalSectionAvailable[entityKey]; locked { + return fmt.Errorf("signals to locked entity %s are not allowed inside a critical section", entityKey) + } } - output, appError := orchestrator(ctx) + action := helpers.NewEntityOperationSignaledAction( + engine.getNextSequenceNumber(), + ctx.NewGuid(), + entityKey, + operationName, + options.rawInput, + options.scheduledTime, + ) + engine.pendingActions[action.Id] = action + return nil +} - var err error - switch { - case appError != nil: - err = ctx.setFailed(appError) - case ctx.continuedAsNew: - err = ctx.setContinuedAsNew() - default: - err = ctx.setComplete(output) +// LockEntities acquires an ordered critical section over a set of entities. +func (ctx *OrchestrationContext) LockEntities(entityIDs ...api.EntityID) (func(), error) { + engine := ctx.engineContext() + if engine.isTerminated || ctx.scope.isCanceled() { + return nil, ErrTaskCanceled + } + if !engine.entitiesSupported { + return nil, errEntitiesUnsupported + } + if engine.criticalSectionID != "" { + return nil, fmt.Errorf("nested entity critical sections are not supported") + } + if len(entityIDs) == 0 { + return nil, fmt.Errorf("at least one entity is required for a critical section") + } + lockSet := make([]string, 0, len(entityIDs)) + seen := make(map[string]struct{}, len(entityIDs)) + for _, entityID := range entityIDs { + if err := helpers.ValidateEntityName(entityID.Name); err != nil { + return nil, err + } + key := entityID.String() + if _, ok := seen[key]; ok { + continue + } + seen[key] = struct{}{} + lockSet = append(lockSet, key) + } + sort.Strings(lockSet) + + criticalSectionID := ctx.NewGuid() + action := helpers.NewEntityLockRequestedAction( + engine.getNextSequenceNumber(), + criticalSectionID, + string(engine.ID), + lockSet, + ) + engine.pendingActions[action.Id] = action + engine.criticalSectionID = criticalSectionID + engine.criticalSectionLocks = append([]string(nil), lockSet...) + engine.criticalSectionRequestCommitted = engine.IsReplaying + lockTask := newTaskInScope(engine, ctx.scope) + engine.pendingEntityTasks[criticalSectionID] = lockTask + if err := lockTask.Await(nil); err != nil { + if !engine.criticalSectionRequestCommitted { + delete(engine.pendingActions, action.Id) + engine.clearCriticalSection() + } + return nil, err + } + engine.criticalSectionAvailable = make(map[string]bool, len(lockSet)) + for _, entity := range lockSet { + engine.criticalSectionAvailable[entity] = true } - if appError == nil && err != nil { - completionErr := fmt.Errorf("failed to complete the orchestration: %w", err) - if err2 := ctx.setFailed(completionErr); err2 != nil { - return completionErr + released := false + return func() { + if released { + return + } + if engine.scheduler != nil && engine.scheduler.isStopping() { + return } + released = true + engine.releaseCriticalSection(criticalSectionID) + }, nil +} + +// IsInCriticalSection reports whether the orchestration currently holds entity locks. +func (ctx *OrchestrationContext) IsInCriticalSection() bool { + return ctx.engineContext().criticalSectionID != "" +} + +func (ctx *OrchestrationContext) ContinueAsNew(newInput any, options ...ContinueAsNewOption) { + engine := ctx.engineContext() + engine.continuedAsNew = true + engine.continuedAsNewInput = newInput + for _, option := range options { + option(engine) + } +} + +// SendEvent sends an event to another orchestration instance as part of the +// current durable orchestration transaction. +func (ctx *OrchestrationContext) SendEvent(instanceID api.InstanceID, eventName string, payload any) error { + engine := ctx.engineContext() + raw, err := marshalData(engine.converter, payload) + if err != nil { + return fmt.Errorf("failed to marshal event payload: %w", err) + } + if engine.isTerminated || ctx.scope.isCanceled() { + return ErrTaskCanceled } + action := helpers.NewSendEventAction(string(instanceID), eventName, wrapperspb.String(string(raw))) + action.Id = engine.getNextSequenceNumber() + engine.pendingActions[action.Id] = action return nil } +func (ctx *OrchestrationContext) onExecutionStarted(es *protos.ExecutionStartedEvent) error { + orchestrator, ok := ctx.registry.getOrchestrator(es.Name, es.GetVersion().GetValue()) + if !ok { + return newTaskNotRegisteredError( + orchestratorTaskNotFoundErrorType, + es.Name, + es.GetVersion().GetValue(), + ) + } + ctx.Name = es.Name + ctx.Version = es.GetVersion().GetValue() + ctx.executionID = es.GetOrchestrationInstance().GetExecutionId().GetValue() + _, fields := contextprop.Decode(es.GetTags()) + ctx.contextFields = mergeStringMaps(ctx.contextFields, fields) + ctx.orchestrationTags = tagcodec.DecodeUserTagsOrPlain(es.GetTags()) + if parent := es.GetParentInstance(); parent != nil { + ctx.parentInstanceID = api.InstanceID(parent.GetOrchestrationInstance().GetInstanceId()) + } + if es.Input != nil { + ctx.rawInput = []byte(es.Input.Value) + } + + return ctx.scheduler.startRoot(orchestrator) +} + func (ctx *OrchestrationContext) onTaskScheduled(taskID int32, ts *protos.TaskScheduledEvent) error { - if a, ok := ctx.pendingActions[taskID]; !ok || a.GetScheduleTask() == nil { + action, ok := ctx.pendingActions[taskID] + scheduled := action.GetScheduleTask() + if !ok || scheduled == nil || + !strings.EqualFold(scheduled.GetName(), ts.GetName()) || + !versionsMatchReplayHistory(scheduled.GetVersion(), ts.GetVersion(), ctx.Version) { return fmt.Errorf( - "a previous execution called CallActivity for '%s' and sequence number %d at this point in the orchestration logic, but the current execution doesn't have this action with this sequence number", + "a previous execution called CallActivity for '%s' with version '%s' and sequence number %d at this point in the orchestration logic, but the current execution doesn't have a matching action", ts.Name, + ts.GetVersion().GetValue(), taskID, ) } @@ -520,10 +1299,15 @@ func (ctx *OrchestrationContext) onTaskFailed(tf *protos.TaskFailedEvent) error } func (ctx *OrchestrationContext) onSubOrchestrationScheduled(taskID int32, ts *protos.SubOrchestrationInstanceCreatedEvent) error { - if a, ok := ctx.pendingActions[taskID]; !ok || a.GetCreateSubOrchestration() == nil { + action, ok := ctx.pendingActions[taskID] + scheduled := action.GetCreateSubOrchestration() + if !ok || scheduled == nil || + !strings.EqualFold(scheduled.GetName(), ts.GetName()) || + !versionsMatchReplayHistory(scheduled.GetVersion(), ts.GetVersion(), ctx.defaultVersion) { return fmt.Errorf( - "a previous execution called CallSubOrchestrator for '%s' and sequence number %d at this point in the orchestration logic, but the current execution doesn't have this action with this sequence number", + "a previous execution called CallSubOrchestrator for '%s' with version '%s' and sequence number %d at this point in the orchestration logic, but the current execution doesn't have a matching action", ts.Name, + ts.GetVersion().GetValue(), taskID, ) } @@ -531,6 +1315,21 @@ func (ctx *OrchestrationContext) onSubOrchestrationScheduled(taskID int32, ts *p return nil } +func versionsMatchReplayHistory( + scheduled *wrapperspb.StringValue, + historical *wrapperspb.StringValue, + legacyDefault string, +) bool { + if strings.EqualFold(scheduled.GetValue(), historical.GetValue()) { + return true + } + // A nil historical version predates version inheritance/defaulting. A non-nil + // empty wrapper is an explicit unversioned selection and must still mismatch. + return historical == nil && + legacyDefault != "" && + strings.EqualFold(scheduled.GetValue(), legacyDefault) +} + func (ctx *OrchestrationContext) onSubOrchestrationCompleted(soc *protos.SubOrchestrationInstanceCompletedEvent) error { taskID := soc.TaskScheduledId task, ok := ctx.pendingTasks[taskID] @@ -590,6 +1389,9 @@ func (ctx *OrchestrationContext) onTimerFired(tf *protos.TimerFiredEvent) error delete(ctx.pendingTasks, timerID) // completing a task will resume the corresponding Await() call + if tf.GetFireAt() != nil { + task.timerFireAt = tf.GetFireAt().AsTime() + } task.complete(nil) return nil } @@ -597,30 +1399,101 @@ func (ctx *OrchestrationContext) onTimerFired(tf *protos.TimerFiredEvent) error func (ctx *OrchestrationContext) onExternalEventRaised(e *protos.HistoryEvent) error { er := e.GetEventRaised() key := strings.ToUpper(er.GetName()) - if pendingTasks, ok := ctx.pendingExternalEventTasks[key]; ok { - // Complete the previously allocated task associated with this event name. - elem := pendingTasks.Front() - task := elem.Value.(*completableTask) - if pendingTasks.Len() > 1 { - pendingTasks.Remove(elem) - } else { - delete(ctx.pendingExternalEventTasks, key) - } - rawValue := []byte(er.Input.GetValue()) - task.complete(rawValue) - } else { - // Add this event to the buffered list of events with this name. - var eventList *list.List - var ok bool - if eventList, ok = ctx.bufferedExternalEvents[key]; !ok { - eventList = list.New() - ctx.bufferedExternalEvents[key] = eventList - } - eventList.PushBack(e) + if pendingTasks, ok := ctx.pendingExternalEventTasks[key]; ok && pendingTasks.Len() > 0 { + task := pendingTasks.Back().Value.(*completableTask) + task.complete([]byte(er.Input.GetValue())) + return nil + } + + // No live waiter consumed the event, so keep it for a future receiver. + eventList, ok := ctx.bufferedExternalEvents[key] + if !ok { + eventList = list.New() + ctx.bufferedExternalEvents[key] = eventList + } + eventList.PushBack(&bufferedEvent{ + event: e, + order: ctx.scheduler.nextCompletionID(), + }) + for waiter := range ctx.eventWaiters[key] { + ctx.scheduler.makeRunnable(waiter) + } + return nil +} + +func (ctx *OrchestrationContext) onEventSent(eventID int32, event *protos.EventSentEvent) error { + action, ok := ctx.pendingActions[eventID] + sent := action.GetSendEvent() + if !ok || sent == nil || + sent.GetInstance().GetInstanceId() != event.GetInstanceId() || + sent.GetName() != event.GetName() || + sent.GetData().GetValue() != event.GetInput().GetValue() { + return fmt.Errorf( + "a previous execution sent event %q to %q with sequence number %d, but the current execution doesn't have this action", + event.GetName(), + event.GetInstanceId(), + eventID, + ) } + delete(ctx.pendingActions, eventID) return nil } +func (ctx *OrchestrationContext) peekBufferedEvent(key string) (*bufferedEvent, bool) { + eventList, ok := ctx.bufferedExternalEvents[key] + if !ok || eventList.Len() == 0 { + return nil, false + } + return eventList.Front().Value.(*bufferedEvent), true +} + +func (ctx *OrchestrationContext) takeBufferedEvent(key string) (*bufferedEvent, bool) { + eventList, ok := ctx.bufferedExternalEvents[key] + if !ok || eventList.Len() == 0 { + return nil, false + } + next := eventList.Front() + event := next.Value.(*bufferedEvent) + eventList.Remove(next) + if eventList.Len() == 0 { + delete(ctx.bufferedExternalEvents, key) + } + return event, true +} + +// removePendingEventTask removes one waiting task from the named list, dropping +// the list entirely once it becomes empty. +func (ctx *OrchestrationContext) removePendingEventTask(key string, element *list.Element) { + taskList, ok := ctx.pendingExternalEventTasks[key] + if !ok { + return + } + taskList.Remove(element) + if taskList.Len() == 0 { + delete(ctx.pendingExternalEventTasks, key) + } +} + +func (ctx *OrchestrationContext) addEventWaiter(key string, co *coroutine) { + waiters, ok := ctx.eventWaiters[key] + if !ok { + waiters = make(map[*coroutine]struct{}) + ctx.eventWaiters[key] = waiters + } + waiters[co] = struct{}{} +} + +func (ctx *OrchestrationContext) removeEventWaiter(key string, co *coroutine) { + waiters, ok := ctx.eventWaiters[key] + if !ok { + return + } + delete(waiters, co) + if len(waiters) == 0 { + delete(ctx.eventWaiters, key) + } +} + func (ctx *OrchestrationContext) onExecutionSuspended(er *protos.ExecutionSuspendedEvent) error { ctx.isSuspended = true return nil @@ -628,61 +1501,233 @@ func (ctx *OrchestrationContext) onExecutionSuspended(er *protos.ExecutionSuspen func (ctx *OrchestrationContext) onExecutionResumed(er *protos.ExecutionResumedEvent) error { ctx.isSuspended = false - for _, e := range ctx.suspendedEvents { - if err := ctx.processEvent(e); err != nil { - return err - } - } + ctx.resumedEvents = append(ctx.resumedEvents, ctx.suspendedEvents...) ctx.suspendedEvents = nil return nil } func (ctx *OrchestrationContext) onExecutionTerminated(et *protos.ExecutionTerminatedEvent) error { - if err := ctx.setCompleteInternal(et.Input, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, nil); err != nil { - return err + ctx.isTerminated = true + // Termination wins over suspension: clearing the flag lets the completion + // action be emitted and discards events that can no longer be processed. + ctx.isSuspended = false + ctx.suspendedEvents = nil + // Termination can arrive in the same work item that lets the root coroutine + // complete naturally. Keep exactly one terminal action, with termination + // taking precedence over the pending natural completion. + ctx.clearCompletionActions() + return ctx.setCompleteInternal(et.Input, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, nil) +} + +func (ctx *OrchestrationContext) onEntityOperationSent(eventID int32, requestID string) error { + action, ok := ctx.pendingActions[eventID] + if !ok { + return fmt.Errorf("entity operation %q does not match pending action %d", requestID, eventID) } + message := action.GetSendEntityMessage() + if message == nil { + return fmt.Errorf("entity operation %q does not match pending action %d", requestID, eventID) + } + sentRequestID := "" + if called := message.GetEntityOperationCalled(); called != nil { + sentRequestID = called.RequestId + } else if signaled := message.GetEntityOperationSignaled(); signaled != nil { + sentRequestID = signaled.RequestId + } + if sentRequestID != requestID { + return fmt.Errorf("entity operation request ID changed from %q to %q", sentRequestID, requestID) + } + delete(ctx.pendingActions, eventID) + return nil +} + +func (ctx *OrchestrationContext) onEntityOperationCompleted(event *protos.EntityOperationCompletedEvent) error { + if task := ctx.takePendingEntityTask(event.RequestId); task != nil { + task.complete([]byte(event.Output.GetValue())) + } + return nil +} + +func (ctx *OrchestrationContext) onEntityOperationFailed(event *protos.EntityOperationFailedEvent) error { + if task := ctx.takePendingEntityTask(event.RequestId); task != nil { + task.fail(event.FailureDetails) + } + return nil +} + +func (ctx *OrchestrationContext) onEntityLockRequested(eventID int32, event *protos.EntityLockRequestedEvent) error { + action, ok := ctx.pendingActions[eventID] + if !ok { + return fmt.Errorf("entity lock request %q does not match pending action %d", event.CriticalSectionId, eventID) + } + message := action.GetSendEntityMessage() + if message == nil || message.GetEntityLockRequested() == nil || + message.GetEntityLockRequested().CriticalSectionId != event.CriticalSectionId { + return fmt.Errorf("entity lock request %q does not match pending action %d", event.CriticalSectionId, eventID) + } + ctx.criticalSectionRequestCommitted = true + delete(ctx.pendingActions, eventID) + return nil +} + +func (ctx *OrchestrationContext) onEntityLockGranted(event *protos.EntityLockGrantedEvent) error { + if task := ctx.takePendingEntityTask(event.CriticalSectionId); task != nil { + task.complete(nil) + } + return nil +} + +func (ctx *OrchestrationContext) onEntityUnlockSent(eventID int32, event *protos.EntityUnlockSentEvent) error { + action, ok := ctx.pendingActions[eventID] + if !ok { + return fmt.Errorf("entity unlock %q does not match pending action %d", event.CriticalSectionId, eventID) + } + message := action.GetSendEntityMessage() + if message == nil || message.GetEntityUnlockSent() == nil || + message.GetEntityUnlockSent().CriticalSectionId != event.CriticalSectionId { + return fmt.Errorf("entity unlock %q does not match pending action %d", event.CriticalSectionId, eventID) + } + delete(ctx.pendingActions, eventID) return nil } +// takePendingEntityTask removes and returns the task awaiting the given entity request +// or critical section ID. Unknown IDs yield nil so that responses that no longer have a +// waiter, such as those replayed after a ContinueAsNew, are ignored. +func (ctx *OrchestrationContext) takePendingEntityTask(requestID string) *completableTask { + task, ok := ctx.pendingEntityTasks[requestID] + if !ok { + return nil + } + delete(ctx.pendingEntityTasks, requestID) + return task +} + func (ctx *OrchestrationContext) setComplete(output any) error { - status := protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED var rawOutput *wrapperspb.StringValue if output != nil { - bytes, err := json.Marshal(output) + bytes, err := marshalData(ctx.converter, output) if err != nil { - return fmt.Errorf("failed to marshal output to JSON: %w", err) + return fmt.Errorf("failed to serialize output: %w", err) } rawOutput = wrapperspb.String(string(bytes)) } - if err := ctx.setCompleteInternal(rawOutput, status, nil); err != nil { - return err - } - return nil + return ctx.setCompleteInternal(rawOutput, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, nil) } func (ctx *OrchestrationContext) setFailed(appError error) error { - fd := helpers.NewTaskFailureDetails(appError) - failedStatus := protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED - if err := ctx.setCompleteInternal(nil, failedStatus, fd); err != nil { - return err + ctx.clearCompletionActions() + provider := ctx.errorProperties + if errors.Is(appError, api.ErrTaskNotRegistered) { + provider = nil + } + return ctx.setCompleteInternal( + nil, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + failure.FromError(appError, provider), + ) +} + +func (ctx *OrchestrationContext) enforceHistoryLimit() { + ctx.clearCompletionActions() + unprocessedEventCount := len(ctx.unprocessedExternalEvents()) + handler := ctx.orchestrationOptions.OnHistoryLimitExceeded + if handler == nil { + _ = ctx.setHistoryLimitFailed(ctx.newHistoryLimitError(nil)) + return + } + + input, err := invokeHistoryLimitHandler(handler, HistoryLimitInfo{ + InstanceID: ctx.ID, + OrchestrationName: ctx.Name, + OrchestrationVersion: ctx.Version, + HistoryLength: ctx.HistoryLength(), + MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, + UnprocessedEventCount: unprocessedEventCount, + SerializedInput: string(ctx.rawInput), + Converter: ctx.converter, + }) + if err != nil { + _ = ctx.setHistoryLimitFailed(ctx.newHistoryLimitError(err)) + return + } + + ctx.continuedAsNew = true + ctx.continuedAsNewInput = input + ctx.saveBufferedExternalEvents = true + if maxHistoryEvents := ctx.orchestrationOptions.MaxHistoryEvents; maxHistoryEvents > 0 && + 2+unprocessedEventCount > maxHistoryEvents { + _ = ctx.setHistoryLimitFailed(ctx.newHistoryLimitError(fmt.Errorf( + "continue-as-new would retain %d external event(s) and exceed MaxHistoryEvents=%d", + unprocessedEventCount, + maxHistoryEvents, + ))) + return + } + if err := ctx.setContinuedAsNew(); err != nil { + _ = ctx.setHistoryLimitFailed(ctx.newHistoryLimitError(err)) } - return nil +} + +func invokeHistoryLimitHandler(handler HistoryLimitHandler, info HistoryLimitInfo) (input any, err error) { + defer func() { + if recovered := recover(); recovered != nil { + err = fmt.Errorf("panic: %v", recovered) + } + }() + return handler(info) +} + +func (ctx *OrchestrationContext) newHistoryLimitError(policyError error) *HistoryLimitError { + return &HistoryLimitError{ + InstanceID: ctx.ID, + HistoryLength: ctx.HistoryLength(), + MaxHistoryEvents: ctx.orchestrationOptions.MaxHistoryEvents, + PolicyError: policyError, + } +} + +func (ctx *OrchestrationContext) setHistoryLimitFailed(err error) error { + ctx.continuedAsNew = false + ctx.clearCompletionActions() + return ctx.setCompleteInternal( + nil, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, + failure.FromError(err, ctx.errorProperties), + ) +} + +func (ctx *OrchestrationContext) clearCompletionActions() { + for id, action := range ctx.pendingActions { + if action.GetCompleteOrchestration() != nil { + delete(ctx.pendingActions, id) + } + } +} + +func (ctx *OrchestrationContext) hasCompletionAction() bool { + for _, action := range ctx.pendingActions { + if action.GetCompleteOrchestration() != nil { + return true + } + } + return false } func (ctx *OrchestrationContext) setContinuedAsNew() error { - status := protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW var newRawInput *wrapperspb.StringValue if ctx.continuedAsNewInput != nil { - bytes, err := json.Marshal(ctx.continuedAsNewInput) + bytes, err := marshalData(ctx.converter, ctx.continuedAsNewInput) if err != nil { - return fmt.Errorf("failed to marshal continue-as-new payload to JSON: %w", err) + return fmt.Errorf("failed to serialize continue-as-new payload: %w", err) } newRawInput = wrapperspb.String(string(bytes)) } - if err := ctx.setCompleteInternal(newRawInput, status, nil); err != nil { - return err - } - return nil + return ctx.setCompleteInternal( + newRawInput, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + nil, + ) } func (ctx *OrchestrationContext) setCompleteInternal( @@ -690,6 +1735,7 @@ func (ctx *OrchestrationContext) setCompleteInternal( status protos.OrchestrationStatus, failureDetails *protos.TaskFailureDetails, ) error { + ctx.releaseCriticalSection(ctx.criticalSectionID) sequenceNumber := ctx.getNextSequenceNumber() completedAction := helpers.NewCompleteOrchestrationAction( sequenceNumber, @@ -698,10 +1744,49 @@ func (ctx *OrchestrationContext) setCompleteInternal( nil, // carryoverEvents is assigned later failureDetails, ) + completed := completedAction.GetCompleteOrchestration() + if status == protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW { + completed.NewVersion = ctx.continuedAsNewVersion + version := ctx.Version + if ctx.continuedAsNewVersion != nil { + version = ctx.continuedAsNewVersion.GetValue() + } + completed.Tags = contextprop.Encode(api.OrchestrationContextInfo{ + InstanceID: ctx.ID, + Name: ctx.Name, + Version: version, + ParentInstanceID: ctx.parentInstanceID, + }, ctx.contextFields, ctx.orchestrationTags) + } else { + completed.Tags = tagcodec.EncodeUserTags(ctx.orchestrationTags) + } ctx.pendingActions[sequenceNumber] = completedAction return nil } +func (ctx *OrchestrationContext) releaseCriticalSection(criticalSectionID string) { + if criticalSectionID == "" || ctx.criticalSectionID != criticalSectionID { + return + } + for _, entityID := range ctx.criticalSectionLocks { + action := helpers.NewEntityUnlockSentAction( + ctx.getNextSequenceNumber(), + criticalSectionID, + string(ctx.ID), + entityID, + ) + ctx.pendingActions[action.Id] = action + } + ctx.clearCriticalSection() +} + +func (ctx *OrchestrationContext) clearCriticalSection() { + ctx.criticalSectionID = "" + ctx.criticalSectionLocks = nil + ctx.criticalSectionAvailable = nil + ctx.criticalSectionRequestCommitted = false +} + func (ctx *OrchestrationContext) getNextSequenceNumber() int32 { current := ctx.sequenceNumber ctx.sequenceNumber++ @@ -713,19 +1798,53 @@ func (ctx *OrchestrationContext) actions() []*protos.OrchestratorAction { return nil } - var actions []*protos.OrchestratorAction - for _, a := range ctx.pendingActions { + actions := make([]*protos.OrchestratorAction, 0, len(ctx.pendingActions)) + for id := int32(0); id < ctx.sequenceNumber; id++ { + a, ok := ctx.pendingActions[id] + if !ok { + continue + } actions = append(actions, a) if ctx.continuedAsNew && ctx.saveBufferedExternalEvents { if co := a.GetCompleteOrchestration(); co != nil { - for _, eventList := range ctx.bufferedExternalEvents { - for item := eventList.Front(); item != nil; item = item.Next() { - e := item.Value.(*protos.HistoryEvent) - co.CarryoverEvents = append(co.CarryoverEvents, e) - } - } + co.CarryoverEvents = append(co.CarryoverEvents, ctx.unprocessedExternalEvents()...) } } } return actions } + +func (ctx *OrchestrationContext) unprocessedExternalEvents() []*protos.HistoryEvent { + buffered := make([]*bufferedEvent, 0) + for _, eventList := range ctx.bufferedExternalEvents { + for item := eventList.Front(); item != nil; item = item.Next() { + buffered = append(buffered, item.Value.(*bufferedEvent)) + } + } + sort.Slice(buffered, func(i, j int) bool { + return buffered[i].order < buffered[j].order + }) + + events := make([]*protos.HistoryEvent, 0, len(buffered)) + for _, event := range buffered { + events = append(events, event.event) + } + for _, event := range ctx.resumedEvents { + if event.event.GetEventRaised() != nil { + events = append(events, event.event) + } + } + for _, event := range ctx.suspendedEvents { + if event.event.GetEventRaised() != nil { + events = append(events, event.event) + } + } + + firstUnprocessedNewEvent := max(ctx.historyIndex-len(ctx.oldEvents), 0) + for _, event := range ctx.newEvents[firstUnprocessedNewEvent:] { + if event.GetEventRaised() != nil { + events = append(events, event) + } + } + return events +} diff --git a/task/orchestrator_test.go b/task/orchestrator_test.go index f4842b53..7462196d 100644 --- a/task/orchestrator_test.go +++ b/task/orchestrator_test.go @@ -1,10 +1,42 @@ package task import ( + "log/slog" "testing" "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/durationpb" ) +func supportedEntityParameters() *protos.OrchestratorEntityParameters { + return &protos.OrchestratorEntityParameters{ + EntityMessageReorderWindow: durationpb.New(0), + } +} + +func newTestOrchestrationContext( + registry *TaskRegistry, + id api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, +) *OrchestrationContext { + return newOrchestrationContext( + registry, + id, + oldEvents, + newEvents, + OrchestrationOptions{}, + slog.Default(), + MetricsHooks{}, + nil, + "", + api.DefaultDataConverter(), + true, + ) +} + func Test_computeNextDelay(t *testing.T) { time1 := time.Now() time2 := time.Now().Add(1 * time.Minute) @@ -13,7 +45,6 @@ func Test_computeNextDelay(t *testing.T) { policy RetryPolicy attempt int firstAttempt time.Time - err error } tests := []struct { name string @@ -29,7 +60,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 2, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 2 * time.Minute, }, attempt: 0, @@ -46,7 +77,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 2, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 2 * time.Minute, }, attempt: 1, @@ -63,7 +94,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 2, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 2 * time.Minute, }, attempt: 2, @@ -80,7 +111,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 2, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 2 * time.Minute, }, attempt: 3, @@ -97,7 +128,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 2, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 30 * time.Second, }, attempt: 3, @@ -114,7 +145,7 @@ func Test_computeNextDelay(t *testing.T) { InitialRetryInterval: 2 * time.Second, BackoffCoefficient: 1, MaxRetryInterval: 10 * time.Second, - Handle: func(err error) bool { return true }, + Handle: func(RetryContext) bool { return true }, RetryTimeout: 2 * time.Minute, }, attempt: 3, @@ -125,7 +156,11 @@ func Test_computeNextDelay(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := computeNextDelay(tt.args.currentTimeUtc, tt.args.policy, tt.args.attempt, tt.args.firstAttempt, tt.args.err); got != tt.want { + err := &TaskFailedError{ + TaskName: "activity", + FailureDetails: &api.FailureDetails{ErrorType: "TestError", ErrorMessage: "failed"}, + } + if got := computeNextDelay(tt.args.currentTimeUtc, tt.args.policy, tt.args.attempt, tt.args.firstAttempt, err); got != tt.want { t.Errorf("computeNextDelay() = %v, want %v", got, tt.want) } }) diff --git a/task/registry.go b/task/registry.go index f3469dc2..faa3501b 100644 --- a/task/registry.go +++ b/task/registry.go @@ -2,53 +2,335 @@ package task import ( "fmt" + "slices" + "strings" + "sync" "github.com/microsoft/durabletask-go/internal/helpers" ) -// TaskRegistry contains maps of names to corresponding orchestrator and activity functions. +type taskVersionKey struct { + name string + version string +} + +type taskRegistration[T any] struct { + Name string + Version string + Handler T +} + +// TaskRegistration identifies one registered task implementation. +type TaskRegistration struct { + Name string + Version string +} + +// TaskRegistrySnapshot is an immutable view of the tasks registered in a registry. +type TaskRegistrySnapshot struct { + Orchestrators []TaskRegistration + Activities []TaskRegistration + Entities []string +} + +// TaskRegistry contains registered orchestrator, activity, and entity functions. type TaskRegistry struct { - orchestrators map[string]Orchestrator - activities map[string]Activity + mu sync.RWMutex + orchestrators map[taskVersionKey]taskRegistration[Orchestrator] + activities map[taskVersionKey]taskRegistration[Activity] + entities map[string]EntityFactory } // NewTaskRegistry returns a new [TaskRegistry] struct. func NewTaskRegistry() *TaskRegistry { - r := &TaskRegistry{ - orchestrators: make(map[string]Orchestrator), - activities: make(map[string]Activity), + return &TaskRegistry{ + orchestrators: make(map[taskVersionKey]taskRegistration[Orchestrator]), + activities: make(map[taskVersionKey]taskRegistration[Activity]), + entities: make(map[string]EntityFactory), } - return r } // AddOrchestrator adds an orchestrator function to the registry. The name of the orchestrator // function is determined using reflection. func (r *TaskRegistry) AddOrchestrator(o Orchestrator) error { - name := helpers.GetTaskFunctionName(o) - return r.AddOrchestratorN(name, o) + if o == nil { + return fmt.Errorf("orchestrator function must not be nil") + } + return r.AddOrchestratorN(helpers.GetTaskFunctionName(o), o) } // AddOrchestratorN adds an orchestrator function to the registry with a specified name. func (r *TaskRegistry) AddOrchestratorN(name string, o Orchestrator) error { - if _, ok := r.orchestrators[name]; ok { - return fmt.Errorf("orchestrator named '%s' is already registered", name) + return r.AddOrchestratorNVersion(name, "", o) +} + +// AddOrchestratorVersion adds a versioned orchestrator whose name is determined using reflection. +func (r *TaskRegistry) AddOrchestratorVersion(version string, o Orchestrator) error { + if o == nil { + return fmt.Errorf("orchestrator function must not be nil") } - r.orchestrators[name] = o - return nil + return r.AddOrchestratorNVersion(helpers.GetTaskFunctionName(o), version, o) +} + +// AddOrchestratorNVersion adds an orchestrator with an explicit name and version. +func (r *TaskRegistry) AddOrchestratorNVersion(name, version string, o Orchestrator) error { + if o == nil { + return fmt.Errorf("orchestrator function must not be nil") + } + r.mu.Lock() + defer r.mu.Unlock() + return addTaskRegistration(r.orchestrators, "orchestrator", name, version, o) } // AddActivity adds an activity function to the registry. The name of the activity // function is determined using reflection. func (r *TaskRegistry) AddActivity(a Activity) error { - name := helpers.GetTaskFunctionName(a) - return r.AddActivityN(name, a) + if a == nil { + return fmt.Errorf("activity function must not be nil") + } + return r.AddActivityN(helpers.GetTaskFunctionName(a), a) } // AddActivityN adds an activity function to the registry with a specified name. func (r *TaskRegistry) AddActivityN(name string, a Activity) error { - if _, ok := r.activities[name]; ok { - return fmt.Errorf("activity named '%s' is already registered", name) + return r.AddActivityNVersion(name, "", a) +} + +// AddActivityVersion adds a versioned activity whose name is determined using reflection. +func (r *TaskRegistry) AddActivityVersion(version string, a Activity) error { + if a == nil { + return fmt.Errorf("activity function must not be nil") + } + return r.AddActivityNVersion(helpers.GetTaskFunctionName(a), version, a) +} + +// AddActivityNVersion adds an activity with an explicit name and version. +func (r *TaskRegistry) AddActivityNVersion(name, version string, a Activity) error { + if a == nil { + return fmt.Errorf("activity function must not be nil") + } + r.mu.Lock() + defer r.mu.Unlock() + return addTaskRegistration(r.activities, "activity", name, version, a) +} + +// AddEntity adds an entity function to the registry. The name of the entity +// function is determined using reflection. +func (r *TaskRegistry) AddEntity(e Entity) error { + if e == nil { + return fmt.Errorf("entity function must not be nil") + } + return r.AddEntityN(helpers.GetTaskFunctionName(e), e) +} + +// AddEntityN adds a shared entity function with a specified name. The function +// can be invoked concurrently by different entity batches and must be thread-safe. +func (r *TaskRegistry) AddEntityN(name string, e Entity) error { + if e == nil { + return fmt.Errorf("entity function must not be nil") + } + return r.AddEntityFactoryN(name, func(EntityFactoryContext) (EntityBatch, error) { + return EntityBatch{Entity: e}, nil + }) +} + +// AddEntityFactory adds an entity factory whose name is determined using reflection. +func (r *TaskRegistry) AddEntityFactory(factory EntityFactory) error { + if factory == nil { + return fmt.Errorf("entity factory must not be nil") + } + return r.AddEntityFactoryN(helpers.GetTaskFunctionName(factory), factory) +} + +// AddEntityFactoryN adds a named factory that creates one entity implementation per batch. +func (r *TaskRegistry) AddEntityFactoryN(name string, factory EntityFactory) error { + if factory == nil { + return fmt.Errorf("entity factory must not be nil") + } + if name != "*" { + if err := helpers.ValidateEntityName(name); err != nil { + return err + } + name = helpers.ToLowerInvariant(name) + } + r.mu.Lock() + defer r.mu.Unlock() + if _, ok := r.entities[name]; ok { + return fmt.Errorf("entity named '%s' is already registered", name) + } + r.entities[name] = factory + return nil +} + +func (r *TaskRegistry) getOrchestrator(name, version string) (Orchestrator, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + return resolveTaskRegistration(r.orchestrators, name, version) +} + +func (r *TaskRegistry) getActivity(name, version string) (Activity, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + return resolveTaskRegistration(r.activities, name, version) +} + +func (r *TaskRegistry) hasOrchestrator(name, version string) bool { + _, ok := r.getOrchestrator(name, version) + return ok +} + +func (r *TaskRegistry) getEntityFactory(name string) (EntityFactory, bool) { + if name != "*" { + name = helpers.ToLowerInvariant(name) + } + r.mu.RLock() + defer r.mu.RUnlock() + factory, ok := r.entities[name] + if !ok { + factory, ok = r.entities["*"] + } + return factory, ok +} + +// Snapshot returns a deterministic, immutable view of the registry. +func (r *TaskRegistry) Snapshot() TaskRegistrySnapshot { + r.mu.RLock() + defer r.mu.RUnlock() + snapshot := TaskRegistrySnapshot{ + Orchestrators: snapshotTaskRegistrations(r.orchestrators), + Activities: snapshotTaskRegistrations(r.activities), + Entities: make([]string, 0, len(r.entities)), + } + for name := range r.entities { + snapshot.Entities = append(snapshot.Entities, name) + } + sortTaskRegistrations(snapshot.Orchestrators) + sortTaskRegistrations(snapshot.Activities) + slices.Sort(snapshot.Entities) + return snapshot +} + +func resolveTaskRegistration[T any]( + registrations map[taskVersionKey]taskRegistration[T], + name string, + version string, +) (T, bool) { + key := taskVersionKey{ + name: normalizeTaskName(name), + version: normalizeTaskVersionForComparison(version), + } + if registration, ok := registrations[key]; ok { + return registration.Handler, true } - r.activities[name] = a + + nameHasVersions := hasVersionedTaskRegistration(registrations, key.name) + if key.version != "" && !nameHasVersions { + if registration, ok := registrations[taskVersionKey{name: key.name}]; ok { + return registration.Handler, true + } + } + if key.name == "*" || nameHasVersions { + var zero T + return zero, false + } + + key.name = "*" + if registration, ok := registrations[key]; ok { + return registration.Handler, true + } + if key.version != "" && !hasVersionedTaskRegistration(registrations, key.name) { + if registration, ok := registrations[taskVersionKey{name: key.name}]; ok { + return registration.Handler, true + } + } + var zero T + return zero, false +} + +func addTaskRegistration[T any]( + registrations map[taskVersionKey]taskRegistration[T], + kind string, + name string, + version string, + handler T, +) error { + key, err := newTaskVersionKey(name, version) + if err != nil { + return err + } + if _, ok := registrations[key]; ok { + return duplicateTaskRegistrationError(kind, name, version) + } + registrations[key] = taskRegistration[T]{Name: name, Version: version, Handler: handler} return nil } + +func hasVersionedTaskRegistration[T any]( + registrations map[taskVersionKey]taskRegistration[T], + name string, +) bool { + for key := range registrations { + if key.name == name && key.version != "" { + return true + } + } + return false +} + +func snapshotTaskRegistrations[T any]( + registrations map[taskVersionKey]taskRegistration[T], +) []TaskRegistration { + snapshot := make([]TaskRegistration, 0, len(registrations)) + for _, registration := range registrations { + snapshot = append(snapshot, TaskRegistration{ + Name: registration.Name, + Version: registration.Version, + }) + } + return snapshot +} + +func newTaskVersionKey(name, version string) (taskVersionKey, error) { + if name == "" { + return taskVersionKey{}, fmt.Errorf("task name must not be empty") + } + normalizedVersion, err := normalizeTaskVersionForRegistration(version) + if err != nil { + return taskVersionKey{}, err + } + return taskVersionKey{name: normalizeTaskName(name), version: normalizedVersion}, nil +} + +func normalizeTaskName(name string) string { + return strings.ToLower(name) +} + +func normalizeTaskVersionForRegistration(version string) (string, error) { + if version != "" && strings.TrimSpace(version) == "" { + return "", fmt.Errorf("task version cannot consist only of whitespace") + } + return strings.ToLower(version), nil +} + +func normalizeTaskVersionForComparison(version string) string { + if strings.TrimSpace(version) == "" { + return "" + } + return strings.ToLower(version) +} + +func duplicateTaskRegistrationError(kind, name, version string) error { + if version == "" { + return fmt.Errorf("%s named '%s' is already registered", kind, name) + } + return fmt.Errorf("%s named '%s' with version '%s' is already registered", kind, name, version) +} + +func sortTaskRegistrations(registrations []TaskRegistration) { + slices.SortFunc(registrations, func(left, right TaskRegistration) int { + if comparison := strings.Compare(strings.ToLower(left.Name), strings.ToLower(right.Name)); comparison != 0 { + return comparison + } + return strings.Compare(strings.ToLower(left.Version), strings.ToLower(right.Version)) + }) +} diff --git a/task/registry_bench_test.go b/task/registry_bench_test.go new file mode 100644 index 00000000..080a9f33 --- /dev/null +++ b/task/registry_bench_test.go @@ -0,0 +1,140 @@ +package task + +import ( + "fmt" + "testing" +) + +var taskRegistryBenchmarkSizes = [...]int{4, 64, 1024} + +// buildBenchmarkRegistry fills a registry with size orchestrator and activity +// names. Every other name also gets two explicit versions so the benchmark +// covers the versioned lookup and fallback paths, not just exact hits. +func buildBenchmarkRegistry(b *testing.B, size int) *TaskRegistry { + b.Helper() + registry := NewTaskRegistry() + orchestrator := func(*OrchestrationContext) (any, error) { return nil, nil } + activity := func(ActivityContext) (any, error) { return nil, nil } + for i := 0; i < size; i++ { + name := fmt.Sprintf("Orchestrator%d", i) + if err := registry.AddOrchestratorN(name, orchestrator); err != nil { + b.Fatalf("add orchestrator: %v", err) + } + if err := registry.AddActivityN(fmt.Sprintf("Activity%d", i), activity); err != nil { + b.Fatalf("add activity: %v", err) + } + if i%2 != 0 { + continue + } + versioned := fmt.Sprintf("Versioned%d", i) + for _, version := range []string{"1.0", "2.0"} { + if err := registry.AddOrchestratorNVersion(versioned, version, orchestrator); err != nil { + b.Fatalf("add versioned orchestrator: %v", err) + } + } + } + return registry +} + +// BenchmarkTaskRegistryGetOrchestrator measures the dispatch lookup that runs +// once per orchestration work item, across registry sizes. +func BenchmarkTaskRegistryGetOrchestrator(b *testing.B) { + for _, size := range taskRegistryBenchmarkSizes { + registry := buildBenchmarkRegistry(b, size) + name := fmt.Sprintf("orchestrator%d", size/2) + b.Run(fmt.Sprintf("hit/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, ok := registry.getOrchestrator(name, ""); !ok { + b.Fatalf("orchestrator %q not found", name) + } + } + }) + b.Run(fmt.Sprintf("miss/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, ok := registry.getOrchestrator("absent", ""); ok { + b.Fatal("absent orchestrator was found") + } + } + }) + } +} + +// BenchmarkTaskRegistryGetOrchestratorVersioned measures the version-matching +// paths: an exact version hit, and the unversioned fallback scan that has to +// prove a name has no versioned registrations. +func BenchmarkTaskRegistryGetOrchestratorVersioned(b *testing.B) { + for _, size := range taskRegistryBenchmarkSizes { + registry := buildBenchmarkRegistry(b, size) + versionedIndex := size / 2 + versionedIndex -= versionedIndex % 2 + versioned := fmt.Sprintf("versioned%d", versionedIndex) + unversioned := fmt.Sprintf("orchestrator%d", size/2) + b.Run(fmt.Sprintf("exact/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, ok := registry.getOrchestrator(versioned, "1.0"); !ok { + b.Fatalf("versioned orchestrator %q not found", versioned) + } + } + }) + b.Run(fmt.Sprintf("fallback/size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, ok := registry.getOrchestrator(unversioned, "9.9"); !ok { + b.Fatalf("unversioned fallback for %q not found", unversioned) + } + } + }) + } +} + +// BenchmarkTaskRegistryGetActivity measures the activity dispatch lookup. +func BenchmarkTaskRegistryGetActivity(b *testing.B) { + for _, size := range taskRegistryBenchmarkSizes { + registry := buildBenchmarkRegistry(b, size) + name := fmt.Sprintf("activity%d", size/2) + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if _, ok := registry.getActivity(name, ""); !ok { + b.Fatalf("activity %q not found", name) + } + } + }) + } +} + +// BenchmarkTaskRegistrySnapshot measures the sorted snapshot the worker builds +// once per start to derive its capabilities and work-item filters. +func BenchmarkTaskRegistrySnapshot(b *testing.B) { + for _, size := range taskRegistryBenchmarkSizes { + registry := buildBenchmarkRegistry(b, size) + b.Run(fmt.Sprintf("size=%d", size), func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + snapshot := registry.Snapshot() + if len(snapshot.Orchestrators) == 0 { + b.Fatal("snapshot is empty") + } + } + }) + } +} + +// BenchmarkTaskRegistryConcurrentLookup measures the read-locked lookup under +// the concurrency a worker actually applies to it. +func BenchmarkTaskRegistryConcurrentLookup(b *testing.B) { + registry := buildBenchmarkRegistry(b, 256) + name := "orchestrator128" + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + if _, ok := registry.getOrchestrator(name, ""); !ok { + b.Errorf("orchestrator %q not found", name) + return + } + } + }) +} diff --git a/task/registry_test.go b/task/registry_test.go new file mode 100644 index 00000000..1cf98bd4 --- /dev/null +++ b/task/registry_test.go @@ -0,0 +1,97 @@ +package task + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestTaskRegistryDispatchesByNameAndVersion(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("Process", func(*OrchestrationContext) (any, error) { + return "unversioned", nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("process", "V1", func(*OrchestrationContext) (any, error) { + return "v1", nil + })) + + handler, ok := registry.getOrchestrator("PROCESS", "v1") + require.True(t, ok) + result, err := handler(nil) + require.NoError(t, err) + require.Equal(t, "v1", result) + + handler, ok = registry.getOrchestrator("process", "") + require.True(t, ok) + result, err = handler(nil) + require.NoError(t, err) + require.Equal(t, "unversioned", result) + + _, ok = registry.getOrchestrator("process", "v2") + require.False(t, ok, "unknown versions must not fall back when versioned registrations exist") +} + +func TestTaskRegistryFallsBackToUnversionedOnlyRegistration(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddActivityN("legacy", func(ActivityContext) (any, error) { + return "legacy", nil + })) + + handler, ok := registry.getActivity("LEGACY", "v2") + require.True(t, ok) + result, err := handler(nil) + require.NoError(t, err) + require.Equal(t, "legacy", result) +} + +func TestTaskRegistryDoesNotUseWildcardForUnknownNamedVersion(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddActivityNVersion("activity", "v1", func(ActivityContext) (any, error) { + return "v1", nil + })) + require.NoError(t, registry.AddActivityN("*", func(ActivityContext) (any, error) { + return "wildcard", nil + })) + + _, ok := registry.getActivity("activity", "v2") + require.False(t, ok) + + _, ok = registry.getActivity("activity", "") + require.False(t, ok) +} + +func TestTaskRegistryRejectsDuplicateVersionKeys(t *testing.T) { + registry := NewTaskRegistry() + handler := func(*OrchestrationContext) (any, error) { return nil, nil } + require.NoError(t, registry.AddOrchestratorNVersion("Process", "V1", handler)) + require.Error(t, registry.AddOrchestratorNVersion("process", "v1", handler)) + require.NoError(t, registry.AddOrchestratorNVersion("process", "v2", handler)) + require.Error(t, registry.AddOrchestratorNVersion("process", " ", handler)) + require.Error(t, registry.AddOrchestrator(nil)) + require.Error(t, registry.AddActivity(nil)) + require.Error(t, registry.AddEntity(nil)) +} + +func TestTaskRegistrySnapshotIsSortedAndImmutable(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorNVersion("B", "v2", func(*OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("a", func(*OrchestrationContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddActivityNVersion("A", "v1", func(ActivityContext) (any, error) { + return nil, nil + })) + require.NoError(t, registry.AddEntityN("Counter", func(*EntityContext) (any, error) { + return nil, nil + })) + + snapshot := registry.Snapshot() + require.Equal(t, []TaskRegistration{{Name: "a"}, {Name: "B", Version: "v2"}}, snapshot.Orchestrators) + require.Equal(t, []TaskRegistration{{Name: "A", Version: "v1"}}, snapshot.Activities) + require.Equal(t, []string{"counter"}, snapshot.Entities) + + snapshot.Orchestrators[0].Name = "changed" + require.Equal(t, "a", registry.Snapshot().Orchestrators[0].Name) +} diff --git a/task/retry.go b/task/retry.go new file mode 100644 index 00000000..db277cf4 --- /dev/null +++ b/task/retry.go @@ -0,0 +1,45 @@ +package task + +import ( + "fmt" + "time" +) + +type retryTaskInfo struct { + kind WorkItemKind + name string + version string +} + +func (ctx *OrchestrationContext) reportRetry( + info retryTaskInfo, + failedAttempt int, + policy RetryPolicy, + delay time.Duration, + err error, +) { + engine := ctx.engineContext() + if engine.IsReplaying || engine.metrics.Retry == nil { + return + } + metric := RetryMetric{ + InstanceID: engine.ID, + OrchestrationName: engine.Name, + OrchestrationVersion: engine.Version, + TaskKind: info.kind, + TaskName: info.name, + TaskVersion: info.version, + FailedAttempt: failedAttempt, + NextAttempt: failedAttempt + 1, + MaxAttempts: policy.MaxAttempts, + Delay: delay, + ErrorType: fmt.Sprintf("%T", err), + ErrorMessage: err.Error(), + } + defer func() { + if recovered := recover(); recovered != nil { + engine.Logger().Error("retry metrics callback panicked", "error", recovered) + } + }() + engine.metrics.Retry(metric) +} diff --git a/task/retry_observability_test.go b/task/retry_observability_test.go new file mode 100644 index 00000000..5d4b02bc --- /dev/null +++ b/task/retry_observability_test.go @@ -0,0 +1,101 @@ +package task + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" +) + +func TestRetryObservabilityReportsNewRetryMetric(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("retry", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: time.Second, + })).Await(nil) + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("retry-observability") + metrics := make([]RetryMetric, 0, 1) + executor := NewTaskExecutor(registry, WithMetricsHooks(MetricsHooks{ + Retry: func(metric RetryMetric) { + metrics = append(metrics, metric) + }, + })) + + oldEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("retry", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "flaky", nil, nil, nil), + } + newEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ + ErrorType: "flaky", + ErrorMessage: "try again", + }), + } + result, err := executor.ExecuteOrchestrator(context.Background(), instanceID, oldEvents, newEvents, nil) + if err != nil { + t.Fatal(err) + } + if len(metrics) != 1 { + t.Fatalf("retry metrics = %d, want 1", len(metrics)) + } + metric := metrics[0] + if metric.TaskName != "flaky" || metric.FailedAttempt != 1 || metric.NextAttempt != 2 || metric.Delay != time.Second { + t.Fatalf("unexpected retry metric: %+v", metric) + } + if len(result.Response.Actions) != 1 || result.Response.Actions[0].GetCreateTimer() == nil { + t.Fatalf("expected retry timer action, got %v", result.Response.Actions) + } + +} + +func TestRetryMetricIsSuppressedDuringReplay(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("retry", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: time.Second, + })).Await(nil) + }); err != nil { + t.Fatal(err) + } + + metricCount := 0 + instanceID := api.InstanceID("retry-replay") + _, err := NewTaskExecutor(registry, WithMetricsHooks(MetricsHooks{ + Retry: func(RetryMetric) { + metricCount++ + }, + })).ExecuteOrchestrator( + + context.Background(), + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("retry", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "flaky", nil, nil, nil), + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ + ErrorType: "flaky", + ErrorMessage: "try again", + }), + helpers.NewTimerCreatedEvent(1, nil), + }, + nil, nil) + + if err != nil && !errors.Is(err, ErrTaskBlocked) { + t.Fatal(err) + } + if metricCount != 0 { + t.Fatalf("retry metric replay count = %d, want 0", metricCount) + } +} diff --git a/task/retry_policy_test.go b/task/retry_policy_test.go new file mode 100644 index 00000000..ec2c9bc5 --- /dev/null +++ b/task/retry_policy_test.go @@ -0,0 +1,388 @@ +package task + +import ( + "context" + "fmt" + "math" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestRetryOptionsCloneAndNormalizeCallerPolicy(t *testing.T) { + caller := &RetryPolicy{InitialRetryInterval: time.Second} + activityOption := WithActivityRetryPolicy(caller) + subOrchestrationOption := WithSubOrchestrationRetryPolicy(caller) + + // Mutation after option construction must not affect either captured policy. + caller.InitialRetryInterval = time.Hour + caller.MaxAttempts = 99 + activity := new(callActivityOptions) + if err := activityOption(activity, api.DefaultDataConverter()); err != nil { + t.Fatal(err) + } + subOrchestration := new(callSubOrchestratorOptions) + if err := subOrchestrationOption(subOrchestration, api.DefaultDataConverter()); err != nil { + t.Fatal(err) + } + for name, policy := range map[string]*RetryPolicy{ + "activity": activity.retryPolicy, + "sub-orchestration": subOrchestration.retryPolicy, + } { + if policy == caller { + t.Fatalf("%s option retained caller-owned policy", name) + } + if policy.InitialRetryInterval != time.Second || policy.MaxAttempts != 1 || + policy.BackoffCoefficient != 1 || policy.MaxRetryInterval != math.MaxInt64 || + policy.RetryTimeout != math.MaxInt64 || policy.Handle == nil { + t.Fatalf("%s policy was not independently normalized: %+v", name, policy) + } + } + if activity.retryPolicy == subOrchestration.retryPolicy { + t.Fatal("activity and sub-orchestration options share a mutable policy copy") + } +} + +func TestRetryPolicyValidateDoesNotMutateReceiver(t *testing.T) { + policy := &RetryPolicy{InitialRetryInterval: time.Second} + if err := policy.Validate(); err != nil { + t.Fatal(err) + } + if policy.MaxAttempts != 0 || policy.BackoffCoefficient != 0 || + policy.MaxRetryInterval != 0 || policy.RetryTimeout != 0 || policy.Handle != nil { + t.Fatalf("Validate mutated its receiver: %+v", policy) + } +} + +func TestComputeNextDelayDoesNotCrossRetryDeadline(t *testing.T) { + firstAttempt := time.Unix(0, 0).UTC() + failure := &TaskFailedError{ + TaskName: "activity", + FailureDetails: &api.FailureDetails{ErrorType: "TestError", ErrorMessage: "failed"}, + } + policy := RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 20 * time.Second, + BackoffCoefficient: 1, + MaxRetryInterval: time.Minute, + RetryTimeout: time.Minute, + Handle: func(RetryContext) bool { return true }, + } + + if delay := computeNextDelay(firstAttempt.Add(50*time.Second), policy, 0, firstAttempt, failure); delay != 0 { + t.Fatalf("delay %v crosses retry deadline", delay) + } + policy.InitialRetryInterval = 10 * time.Second + if delay := computeNextDelay(firstAttempt.Add(50*time.Second), policy, 0, firstAttempt, failure); delay != 10*time.Second { + t.Fatalf("delay at retry deadline = %v, want 10s", delay) + } + if delay := computeNextDelay(firstAttempt.Add(time.Minute), policy, 0, firstAttempt, failure); delay != 0 { + t.Fatalf("delay at expired retry deadline = %v, want 0", delay) + } +} + +func TestRetryOptionDoesNotReadCallerPolicyAfterConstruction(t *testing.T) { + caller := &RetryPolicy{InitialRetryInterval: time.Second, MaxAttempts: 3} + option := WithActivityRetryPolicy(caller) + + var writers sync.WaitGroup + writers.Add(1) + go func() { + defer writers.Done() + for i := 1; i <= 10_000; i++ { + caller.InitialRetryInterval = time.Duration(i) * time.Millisecond + caller.MaxAttempts = i + } + }() + for range 1_000 { + configured := new(callActivityOptions) + if err := option(configured, api.DefaultDataConverter()); err != nil { + t.Fatal(err) + } + if configured.retryPolicy.InitialRetryInterval != time.Second || configured.retryPolicy.MaxAttempts != 3 { + t.Fatalf("captured policy drifted: %+v", configured.retryPolicy) + } + } + writers.Wait() +} + +// TestRetryDoesNotScheduleTimerPastRetryTimeout proves the retry loop stops +// instead of creating a durable timer whose delay would carry the next attempt +// past RetryTimeout. Bounding the timer at creation keeps the decision on the +// failure event, where every input is replayed from history. +func TestRetryDoesNotScheduleTimerPastRetryTimeout(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("bounded-retry", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 10 * time.Second, + RetryTimeout: time.Minute, + })).Await(nil) + }); err != nil { + t.Fatal(err) + } + + firstAttempt := time.Unix(1_700_000_000, 0).UTC() + startedTurn := helpers.NewOrchestratorStartedEvent() + startedTurn.Timestamp = timestamppb.New(firstAttempt) + // Only five seconds of the retry budget remain, so the ten-second backoff + // would fire after the deadline. + failureTurn := helpers.NewOrchestratorStartedEvent() + failureTurn.Timestamp = timestamppb.New(firstAttempt.Add(55 * time.Second)) + instanceID := api.InstanceID("bounded-retry-instance") + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + instanceID, + []*protos.HistoryEvent{ + startedTurn, + helpers.NewExecutionStartedEvent("bounded-retry", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "flaky", nil, nil, nil), + }, + []*protos.HistoryEvent{ + failureTurn, + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ + ErrorType: "TransientFailure", + ErrorMessage: "retry me", + }), + }, nil) + + if err != nil { + t.Fatal(err) + } + for _, action := range result.Response.Actions { + if action.GetCreateTimer() != nil { + t.Fatal("retry timer scheduled past RetryTimeout") + } + if scheduled := action.GetScheduleTask(); scheduled != nil { + t.Fatalf("retry scheduled activity %q past RetryTimeout", scheduled.GetName()) + } + } + if completed := completionAction(t, result.Response); completed.GetOrchestrationStatus() != + protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { + t.Fatalf("orchestration status = %v, want FAILED", completed.GetOrchestrationStatus()) + } +} + +// TestRetryDecisionIsStableWhenCompletionIsRedelivered pins the replay contract +// the retry loop depends on: when a completion response is lost and DTS +// redelivers the work item, the same events must produce the same retry +// actions even though they arrive as replayed history. +func TestRetryDecisionIsStableWhenCompletionIsRedelivered(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("stable-retry", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("flaky", WithActivityRetryPolicy(&RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 10 * time.Second, + RetryTimeout: time.Minute, + })).Await(nil) + }); err != nil { + t.Fatal(err) + } + + firstAttempt := time.Unix(1_700_000_000, 0).UTC() + failureTurn := helpers.NewOrchestratorStartedEvent() + failureTurn.Timestamp = timestamppb.New(firstAttempt.Add(5 * time.Second)) + startedTurn := helpers.NewOrchestratorStartedEvent() + startedTurn.Timestamp = timestamppb.New(firstAttempt) + committed := []*protos.HistoryEvent{ + startedTurn, + helpers.NewExecutionStartedEvent("stable-retry", string(instanceIDStableRetry), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "flaky", nil, nil, nil), + } + delivered := []*protos.HistoryEvent{ + failureTurn, + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ + ErrorType: "TransientFailure", + ErrorMessage: "retry me", + }), + } + executor := NewTaskExecutor(registry) + + // First delivery: the failure arrives as a new event. + first, err := executor.ExecuteOrchestrator( + context.Background(), instanceIDStableRetry, committed, delivered, nil) + + if err != nil { + t.Fatal(err) + } + // Redelivery after a lost response: the identical failure is now replayed + // history, so IsReplaying flips while the decision must not. + second, err := executor.ExecuteOrchestrator( + context.Background(), instanceIDStableRetry, append(committed, delivered...), nil, nil) + + if err != nil { + t.Fatal(err) + } + + firstTimer := singleRetryTimer(t, first.Response) + secondTimer := singleRetryTimer(t, second.Response) + if !firstTimer.GetFireAt().AsTime().Equal(secondTimer.GetFireAt().AsTime()) { + t.Fatalf("retry timer moved across redelivery: %v then %v", + firstTimer.GetFireAt().AsTime(), secondTimer.GetFireAt().AsTime()) + } +} + +const instanceIDStableRetry = api.InstanceID("stable-retry-instance") + +func singleRetryTimer(t *testing.T, response *protos.OrchestratorResponse) *protos.CreateTimerAction { + t.Helper() + var timer *protos.CreateTimerAction + for _, action := range response.GetActions() { + if created := action.GetCreateTimer(); created != nil { + if timer != nil { + t.Fatal("more than one retry timer action") + } + timer = created + } + } + if timer == nil { + t.Fatalf("no retry timer action in %v", response.GetActions()) + } + return timer +} + +func TestComputeNextDelayPreservesPositiveDurations(t *testing.T) { + for _, test := range []struct { + name string + initial time.Duration + coefficient float64 + attempt int + cap time.Duration + timeout time.Duration + elapsed time.Duration + want time.Duration + }{ + {name: "submillisecond initial", initial: 500 * time.Microsecond, want: 500 * time.Microsecond}, + {name: "fractional backoff", initial: 1500 * time.Microsecond, coefficient: .5, attempt: 1, want: 750 * time.Microsecond}, + {name: "subnanosecond backoff", initial: time.Nanosecond, coefficient: .5, attempt: 1, want: time.Nanosecond}, + {name: "floating point underflow", initial: time.Second, coefficient: .01, attempt: 1000, want: time.Nanosecond}, + {name: "unscaled precision", initial: math.MaxInt64 - 31, coefficient: 1, attempt: 8, want: math.MaxInt64 - 31}, + {name: "duration overflow", initial: time.Second, coefficient: math.MaxFloat64, attempt: 2, want: math.MaxInt64}, + {name: "overflow capped", initial: time.Second, coefficient: math.MaxFloat64, attempt: 2, cap: 1234 * time.Microsecond, want: 1234 * time.Microsecond}, + {name: "submillisecond cap", initial: time.Millisecond, coefficient: 2, attempt: 3, cap: 500 * time.Microsecond, want: 500 * time.Microsecond}, + {name: "exact deadline", initial: 500 * time.Microsecond, timeout: time.Millisecond, elapsed: 500 * time.Microsecond, want: 500 * time.Microsecond}, + {name: "past deadline", initial: 500 * time.Microsecond, timeout: time.Millisecond, elapsed: 500*time.Microsecond + time.Nanosecond}, + } { + t.Run(test.name, func(t *testing.T) { + policy, err := (&RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: test.initial, + BackoffCoefficient: test.coefficient, + MaxRetryInterval: test.cap, + RetryTimeout: test.timeout, + }).Normalized() + require.NoError(t, err) + first := time.Unix(1_700_000_000, 0).UTC() + failure := &TaskFailedError{FailureDetails: &api.FailureDetails{ErrorType: "Transient", ErrorMessage: "retry"}} + require.Equal(t, test.want, computeNextDelay(first.Add(test.elapsed), policy, test.attempt, first, failure)) + }) + } +} + +func TestRetryPreservesSubmillisecondTimersAcrossReplay(t *testing.T) { + for _, kind := range []string{"activity", "sub-orchestration"} { + for _, test := range []struct { + name string + initial time.Duration + coefficient float64 + cap time.Duration + delays []time.Duration + }{ + {"decreasing", 500 * time.Microsecond, .5, 0, []time.Duration{500 * time.Microsecond, 250 * time.Microsecond}}, + {"increasing", 500 * time.Microsecond, 1.5, 0, []time.Duration{500 * time.Microsecond, 750 * time.Microsecond}}, + {"capped", 1500 * time.Microsecond, .5, 750 * time.Microsecond, []time.Duration{750 * time.Microsecond, 750 * time.Microsecond}}, + {"minimum", 500 * time.Microsecond, 1e-200, 0, []time.Duration{500 * time.Microsecond, time.Nanosecond}}, + } { + t.Run(kind+"/"+test.name, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("precise-retry", func(ctx *OrchestrationContext) (any, error) { + policy := &RetryPolicy{ + MaxAttempts: 3, InitialRetryInterval: test.initial, + BackoffCoefficient: test.coefficient, MaxRetryInterval: test.cap, + } + var pending Task + if kind == "activity" { + pending = ctx.CallActivity("flaky", WithActivityRetryPolicy(policy)) + } else { + pending = ctx.CallSubOrchestrator("flaky", WithSubOrchestrationRetryPolicy(policy)) + } + var value string + err := pending.Await(&value) + return value, err + })) + turnAt := func(at time.Time) *protos.HistoryEvent { + event := helpers.NewOrchestratorStartedEvent() + event.Timestamp = timestamppb.New(at) + return event + } + scheduled := func(id int32) *protos.HistoryEvent { + if kind == "activity" { + return helpers.NewTaskScheduledEvent(id, "flaky", nil, nil, nil) + } + return helpers.NewSubOrchestrationCreatedEvent(id, "flaky", nil, nil, fmt.Sprintf("instance:%04x", id), nil) + } + failed := func(id int32) *protos.HistoryEvent { + details := &protos.TaskFailureDetails{ErrorType: "Transient", ErrorMessage: "retry"} + if kind == "activity" { + return helpers.NewTaskFailedEvent(id, details) + } + return &protos.HistoryEvent{EventType: &protos.HistoryEvent_SubOrchestrationInstanceFailed{ + SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{ + TaskScheduledId: id, FailureDetails: details, + }, + }} + } + now := time.Unix(1_700_000_000, 0).UTC() + history := []*protos.HistoryEvent{ + turnAt(now), + helpers.NewExecutionStartedEvent("precise-retry", "instance", nil, nil, nil, nil), + scheduled(0), + } + for attempt, delay := range test.delays { + id := int32(2 * attempt) + delivered := []*protos.HistoryEvent{turnAt(now), failed(id)} + response := executeOrchestrationTurn(t, registry, "instance", history, delivered) + timer := singleRetryTimer(t, response) + history = append(history, delivered...) + replay := executeOrchestrationTurn(t, registry, "instance", history, nil) + require.True(t, proto.Equal(timer, singleRetryTimer(t, replay))) + require.Equal(t, now.Add(delay), timer.FireAt.AsTime()) + history = append(history, helpers.NewTimerCreatedEvent(id+1, timer.FireAt)) + now = timer.FireAt.AsTime() + fired := []*protos.HistoryEvent{turnAt(now), helpers.NewTimerFiredEvent(id+1, timer.FireAt, nil)} + retried := executeOrchestrationTurn(t, registry, "instance", history, fired) + require.Len(t, retried.Actions, 1) + require.Equal(t, id+2, retried.Actions[0].Id) + if kind == "activity" { + require.Equal(t, "flaky", retried.Actions[0].GetScheduleTask().GetName()) + } else { + require.Equal(t, "flaky", retried.Actions[0].GetCreateSubOrchestration().GetName()) + } + history = append(history, fired...) + history = append(history, scheduled(id+2)) + } + id := int32(2 * len(test.delays)) + completed := helpers.NewTaskCompletedEvent(id, wrapperspb.String(`"done"`)) + if kind == "sub-orchestration" { + completed = &protos.HistoryEvent{EventType: &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{ + TaskScheduledId: id, Result: wrapperspb.String(`"done"`), + }, + }} + } + delivered := []*protos.HistoryEvent{turnAt(now), completed} + require.Equal(t, `"done"`, completionResult(t, executeOrchestrationTurn(t, registry, "instance", history, delivered))) + history = append(history, delivered...) + require.Equal(t, `"done"`, completionResult(t, executeOrchestrationTurn(t, registry, "instance", history, nil))) + }) + } + } +} diff --git a/task/runservice_patterns_test.go b/task/runservice_patterns_test.go new file mode 100644 index 00000000..3ac814c6 --- /dev/null +++ b/task/runservice_patterns_test.go @@ -0,0 +1,178 @@ +package task + +import ( + "encoding/json" + "fmt" + "strconv" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestExecuteGatesPatternSignalWinsTimerAndCancelsScope(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("execute-gates", func(ctx *OrchestrationContext) (any, error) { + approval := NewEventChannel[string](ctx, "approval") + timerContext, cancelTimer := ctx.WithCancel() + timer := timerContext.CreateTimer(time.Hour) + result := "timeout" + ctx.Select( + OnEvent(approval, func(value string) { + result = value + cancelTimer() + }), + OnTask(timer, func(Task) { + result = "timeout" + }), + ) + return result, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("execute-gates") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("execute-gates", string(instanceID), nil, nil, nil, nil) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + if len(firstTurn.Actions) != 1 || firstTurn.Actions[0].GetCreateTimer() == nil { + t.Fatalf("expected one timer action, got %v", firstTurn.Actions) + } + + timerCreated := helpers.NewTimerCreatedEvent(0, firstTurn.Actions[0].GetCreateTimer().GetFireAt()) + secondTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{started, executionStarted, timerCreated}, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("approval", wrapperspb.String(`"approved"`)), + }, + ) + if got := completionResult(t, secondTurn); got != `"approved"` { + t.Fatalf("gate result = %s, want approved", got) + } +} + +func TestExecuteJobPatternManySignalsSharedStateAndWideFanOut(t *testing.T) { + const ( + signalCount = 15 + fanOutCount = 32 + ) + type output struct { + Signals []int + Sum int + FanOut int + } + + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("execute-job", func(ctx *OrchestrationContext) (any, error) { + signals := make([]int, signalCount) + waitGroup := ctx.NewWaitGroup() + waitGroup.Add(signalCount) + for i := 0; i < signalCount; i++ { + index := i + channel := NewEventChannel[int](ctx, fmt.Sprintf("signal-%02d", index)) + ctx.Go(func(ctx *OrchestrationContext) { + defer waitGroup.Done() + signals[index] = channel.Receive(ctx) + }) + } + waitGroup.Wait(ctx) + + tasks := make([]Task, fanOutCount) + for i := 0; i < fanOutCount; i++ { + tasks[i] = ctx.CallActivity("work", WithActivityInput(i)) + } + if err := ctx.WhenAll(tasks...); err != nil { + return nil, err + } + + sum := 0 + for _, value := range signals { + sum += value + } + return output{Signals: signals, Sum: sum, FanOut: len(tasks)}, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("execute-job") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("execute-job", string(instanceID), nil, nil, nil, nil) + firstTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + if len(firstTurn.Actions) != 0 { + t.Fatalf("signal wait produced unexpected actions: %v", firstTurn.Actions) + } + + signalEvents := make([]*protos.HistoryEvent, 0, signalCount+1) + signalEvents = append(signalEvents, helpers.NewOrchestratorStartedEvent()) + for i := 0; i < signalCount; i++ { + signalEvents = append(signalEvents, helpers.NewEventRaisedEvent( + fmt.Sprintf("signal-%02d", i), + wrapperspb.String(strconv.Itoa(i+1)), + )) + } + secondTurn := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{started, executionStarted}, + signalEvents, + ) + if len(secondTurn.Actions) != fanOutCount { + t.Fatalf("fan-out action count = %d, want %d", len(secondTurn.Actions), fanOutCount) + } + for i, action := range secondTurn.Actions { + scheduled := action.GetScheduleTask() + if scheduled == nil || scheduled.GetName() != "work" || action.GetId() != int32(i) { + t.Fatalf("unexpected fan-out action %d: %v", i, action) + } + } + + oldEvents := []*protos.HistoryEvent{started, executionStarted} + oldEvents = append(oldEvents, signalEvents...) + for i := 0; i < fanOutCount; i++ { + oldEvents = append(oldEvents, helpers.NewTaskScheduledEvent( + int32(i), + "work", + nil, + wrapperspb.String(strconv.Itoa(i)), + nil, + )) + } + newEvents := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent()} + for i := fanOutCount - 1; i >= 0; i-- { + newEvents = append(newEvents, helpers.NewTaskCompletedEvent(int32(i), nil)) + } + thirdTurn := executeOrchestrationTurn(t, registry, instanceID, oldEvents, newEvents) + + var result output + if err := json.Unmarshal([]byte(completionResult(t, thirdTurn)), &result); err != nil { + t.Fatal(err) + } + if result.FanOut != fanOutCount || result.Sum != signalCount*(signalCount+1)/2 { + t.Fatalf("unexpected execute-job result: %+v", result) + } + for i, value := range result.Signals { + if value != i+1 { + t.Fatalf("signal[%d] = %d, want %d", i, value, i+1) + } + } +} diff --git a/task/scheduler.go b/task/scheduler.go new file mode 100644 index 00000000..4018dc70 --- /dev/null +++ b/task/scheduler.go @@ -0,0 +1,188 @@ +package task + +import ( + "container/heap" + "fmt" +) + +type coroutineQueue []*coroutine + +func (q coroutineQueue) Len() int { return len(q) } +func (q coroutineQueue) Less(i, j int) bool { return q[i].id < q[j].id } +func (q coroutineQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] } + +func (q *coroutineQueue) Push(value any) { + *q = append(*q, value.(*coroutine)) +} + +func (q *coroutineQueue) Pop() any { + old := *q + last := len(old) - 1 + value := old[last] + old[last] = nil + *q = old[:last] + return value +} + +type coroutineScheduler struct { + ctx *OrchestrationContext + nextID uint64 + runQueue coroutineQueue + all []*coroutine + current *coroutine + root *coroutine + rootResult any + rootErr error + completionID uint64 + stopping bool + terminalErr error + rootFinalized bool + pendingCancel []*cancellationScope + pendingCancelSet map[*cancellationScope]struct{} +} + +func newCoroutineScheduler(ctx *OrchestrationContext) *coroutineScheduler { + s := &coroutineScheduler{ + ctx: ctx, + pendingCancelSet: make(map[*cancellationScope]struct{}), + } + heap.Init(&s.runQueue) + return s +} + +func (s *coroutineScheduler) startRoot(fn Orchestrator) error { + if s.root != nil { + return fmt.Errorf("orchestrator function was started more than once") + } + + s.root = s.spawn(s.ctx, func() { + s.rootResult, s.rootErr = fn(s.ctx) + }) + return nil +} + +func (s *coroutineScheduler) spawn(ctx *OrchestrationContext, fn func()) *coroutine { + if s.stopping { + return nil + } + c := &coroutine{ + id: s.nextID, + scheduler: s, + ctx: ctx, + fn: fn, + resume: make(chan struct{}), + signals: make(chan coroutineSignal, 1), + stop: make(chan struct{}), + exited: make(chan struct{}), + state: coroutineRunnable, + scope: ctx.scope, + } + s.nextID++ + if c.scope != nil { + c.scope.addCoroutine(c) + } + s.all = append(s.all, c) + go c.run() + heap.Push(&s.runQueue, c) + return c +} + +func (s *coroutineScheduler) hasRunnable() bool { + return s.runQueue.Len() > 0 +} + +func (s *coroutineScheduler) runNext() { + c := heap.Pop(&s.runQueue).(*coroutine) + if c.state != coroutineRunnable { + panic(fmt.Sprintf("coroutine %d is not runnable", c.id)) + } + + c.state = coroutineRunning + s.current = c + c.resume <- struct{}{} + signal := <-c.signals + s.current = nil + + switch signal.kind { + case coroutineYielded: + if c.state == coroutineRunning { + c.state = coroutineWaiting + } + case coroutineFinished, coroutineCanceled: + c.state = coroutineCompleted + case coroutinePanicked: + c.state = coroutineCompleted + s.terminalErr = coroutinePanicError(c.id, signal.panicValue, signal.panicStack) + default: + panic(fmt.Sprintf("unknown coroutine signal %d", signal.kind)) + } + s.applyPendingCancellations() +} + +func (s *coroutineScheduler) waitForTask(task *completableTask) { + c := s.mustCurrent() + task.addWaiter(c) + c.state = coroutineWaiting + c.yield() + task.removeWaiter(c) +} + +func (s *coroutineScheduler) requestCancellation(scope *cancellationScope) { + if scope == nil || scope.isCanceled() { + return + } + if _, ok := s.pendingCancelSet[scope]; ok { + return + } + s.pendingCancel = append(s.pendingCancel, scope) + s.pendingCancelSet[scope] = struct{}{} +} + +func (s *coroutineScheduler) applyPendingCancellations() { + for len(s.pendingCancel) > 0 { + scope := s.pendingCancel[0] + s.pendingCancel = s.pendingCancel[1:] + delete(s.pendingCancelSet, scope) + scope.cancel(s) + } + s.pendingCancel = nil +} + +func (s *coroutineScheduler) makeRunnable(c *coroutine) { + if s.stopping || c.state != coroutineWaiting { + return + } + c.state = coroutineRunnable + heap.Push(&s.runQueue, c) +} + +func (s *coroutineScheduler) nextCompletionID() uint64 { + id := s.completionID + s.completionID++ + return id +} + +func (s *coroutineScheduler) mustCurrent() *coroutine { + if s.current == nil { + panic("durable task operation called outside an orchestration coroutine") + } + return s.current +} + +func (s *coroutineScheduler) isRootCompleted() bool { + return s.root != nil && s.root.state == coroutineCompleted +} + +func (s *coroutineScheduler) isStopping() bool { + return s.stopping +} + +func (s *coroutineScheduler) shutdown() { + if s.stopping { + return + } + s.stopping = true + for _, c := range s.all { + c.exit() + } +} diff --git a/task/scheduler_test.go b/task/scheduler_test.go new file mode 100644 index 00000000..f1d263b5 --- /dev/null +++ b/task/scheduler_test.go @@ -0,0 +1,294 @@ +package task + +import ( + "context" + "runtime" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestCoroutineSchedulerRunsReadyCoroutinesInIDOrder(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("parallel", func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 0, 2) + wg := ctx.NewWaitGroup() + wg.Add(2) + + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + var value string + if err := ctx.CallActivity("first").Await(&value); err != nil { + panic(err) + } + results = append(results, value) + }) + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + var value string + if err := ctx.CallActivity("second").Await(&value); err != nil { + panic(err) + } + results = append(results, value) + }) + + wg.Wait(ctx) + return results, nil + }); err != nil { + t.Fatal(err) + } + + executor := NewTaskExecutor(registry) + instanceID := api.InstanceID("parallel-instance") + started := helpers.NewOrchestratorStartedEvent() + executionStarted := helpers.NewExecutionStartedEvent("parallel", string(instanceID), nil, nil, nil, nil) + + firstTurn, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, nil) + + if err != nil { + t.Fatal(err) + } + if len(firstTurn.Response.Actions) != 2 { + t.Fatalf("got %d first-turn actions, want 2", len(firstTurn.Response.Actions)) + } + if got := firstTurn.Response.Actions[0].GetScheduleTask().GetName(); got != "first" { + t.Fatalf("first action = %q, want first", got) + } + if got := firstTurn.Response.Actions[1].GetScheduleTask().GetName(); got != "second" { + t.Fatalf("second action = %q, want second", got) + } + + oldEvents := []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTaskScheduledEvent(0, "first", nil, nil, nil), + helpers.NewTaskScheduledEvent(1, "second", nil, nil, nil), + } + newEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"one"`)), + helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"two"`)), + } + secondTurn, err := executor.ExecuteOrchestrator(context.Background(), instanceID, oldEvents, newEvents, nil) + if err != nil { + t.Fatal(err) + } + if len(secondTurn.Response.Actions) != 1 { + t.Fatalf("got %d second-turn actions, want completion only", len(secondTurn.Response.Actions)) + } + completed := secondTurn.Response.Actions[0].GetCompleteOrchestration() + if completed == nil { + t.Fatal("second turn did not complete the orchestration") + } + if got, want := completed.GetResult().GetValue(), `["one","two"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCoroutineSchedulerUsesHistoryCompletionOrder(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("completion-order", func(ctx *OrchestrationContext) (any, error) { + results := make([]string, 0, 2) + wg := ctx.NewWaitGroup() + wg.Add(2) + for _, name := range []string{"first", "second"} { + name := name + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + var value string + if err := ctx.CallActivity(name).Await(&value); err != nil { + panic(err) + } + results = append(results, value) + }) + } + wg.Wait(ctx) + return results, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("completion-order-instance") + oldEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("completion-order", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "first", nil, nil, nil), + helpers.NewTaskScheduledEvent(1, "second", nil, nil, nil), + } + newEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"two"`)), + helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"one"`)), + } + + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + instanceID, + oldEvents, + newEvents, nil) + + if err != nil { + t.Fatal(err) + } + completed := result.Response.Actions[0].GetCompleteOrchestration() + if got, want := completed.GetResult().GetValue(), `["two","one"]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestCoroutinePanicFailsOrchestration(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("panic", func(ctx *OrchestrationContext) (any, error) { + ctx.Go(func(*OrchestrationContext) { + panic("boom") + }) + _ = ctx.WaitForSingleEvent("never", -1).Await(nil) + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("panic-instance") + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("panic", string(instanceID), nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + if len(result.Response.Actions) != 1 { + t.Fatalf("got %d actions, want one failure action", len(result.Response.Actions)) + } + completed := result.Response.Actions[0].GetCompleteOrchestration() + if completed == nil || completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { + t.Fatalf("unexpected completion action: %v", result.Response.Actions[0]) + } + if got := completed.GetFailureDetails().GetErrorMessage(); got != "coroutine 1 panicked: boom" { + t.Fatalf("failure = %q", got) + } +} + +func TestRootCoroutinePanicFailsOrchestration(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("root-panic", func(*OrchestrationContext) (any, error) { + panic("root boom") + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("root-panic-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("root-panic", string(instanceID), nil, nil, nil, nil), + }, + ) + completed := completionAction(t, result) + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { + t.Fatalf("status = %v, want failed", completed.GetOrchestrationStatus()) + } + if got := completed.GetFailureDetails().GetErrorMessage(); got != "coroutine 0 panicked: root boom" { + t.Fatalf("failure = %q", got) + } +} + +func TestCoroutineReplayIsByteDeterministic(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("deterministic", func(ctx *OrchestrationContext) (any, error) { + wg := ctx.NewWaitGroup() + wg.Add(4) + for i := 0; i < 4; i++ { + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + ctx.CallActivity("activity") + }) + } + wg.Wait(ctx) + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("deterministic-instance") + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("deterministic", string(instanceID), nil, nil, nil, nil), + } + executor := NewTaskExecutor(registry) + + var expected []byte + for i := 0; i < 100; i++ { + result, err := executor.ExecuteOrchestrator(context.Background(), instanceID, nil, events, nil) + if err != nil { + t.Fatal(err) + } + data, err := proto.Marshal(&protos.OrchestratorResponse{Actions: result.Response.Actions}) + if err != nil { + t.Fatal(err) + } + if i == 0 { + expected = data + } else if !proto.Equal( + &protos.OrchestratorResponse{Actions: result.Response.Actions}, + mustUnmarshalResponse(t, expected), + ) { + t.Fatalf("iteration %d produced different actions", i) + } + } +} + +func TestCoroutineSchedulerDoesNotLeakAcrossTurns(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("blocked", func(ctx *OrchestrationContext) (any, error) { + defer ctx.Go(func(*OrchestrationContext) {}) + _ = ctx.WaitForSingleEvent("never", -1).Await(nil) + return nil, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("blocked-instance") + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("blocked", string(instanceID), nil, nil, nil, nil), + } + executor := NewTaskExecutor(registry) + before := runtime.NumGoroutine() + + for i := 0; i < 10_000; i++ { + if _, err := executor.ExecuteOrchestrator(context.Background(), instanceID, nil, events, nil); err != nil { + t.Fatal(err) + } + } + + runtime.GC() + time.Sleep(50 * time.Millisecond) + if after := runtime.NumGoroutine(); after > before+2 { + t.Fatalf("goroutines grew from %d to %d", before, after) + } +} + +func mustUnmarshalResponse(t *testing.T, data []byte) *protos.OrchestratorResponse { + t.Helper() + response := new(protos.OrchestratorResponse) + if err := proto.Unmarshal(data, response); err != nil { + t.Fatal(err) + } + return response +} diff --git a/task/select.go b/task/select.go new file mode 100644 index 00000000..f01b67e7 --- /dev/null +++ b/task/select.go @@ -0,0 +1,134 @@ +package task + +import "fmt" + +// SelectCase is a durable selection case created by OnTask or OnEvent. +type SelectCase interface { + ready() (bool, uint64) + subscribe(*coroutine) + unsubscribe(*coroutine) + invoke() +} + +type taskSelectCase struct { + task Task + state *completableTask + handler func(Task) +} + +// OnTask creates a Select case that becomes ready when task completes. +func OnTask(task Task, handler func(Task)) SelectCase { + state, ok := taskState(task) + if !ok { + panic(fmt.Sprintf("task type %T cannot be used in a durable Select", task)) + } + return &taskSelectCase{task: task, state: state, handler: handler} +} + +func (c *taskSelectCase) ready() (bool, uint64) { + return c.state.isCompleted, c.state.completionID +} + +func (c *taskSelectCase) subscribe(coroutine *coroutine) { + c.state.addWaiter(coroutine) +} + +func (c *taskSelectCase) unsubscribe(coroutine *coroutine) { + c.state.removeWaiter(coroutine) +} + +func (c *taskSelectCase) invoke() { + if c.handler != nil { + c.handler(c.task) + } +} + +// WhenAny blocks until the first task completes and returns that task. If +// multiple tasks are ready, the task completed first in orchestration history wins. +func (ctx *OrchestrationContext) WhenAny(tasks ...Task) Task { + if len(tasks) == 0 { + panic("WhenAny requires at least one task") + } + cases := make([]SelectCase, len(tasks)) + for i, task := range tasks { + cases[i] = OnTask(task, nil) + } + return ctx.selectCase(cases).(*taskSelectCase).task +} + +// WhenAll blocks until every task completes and returns the earliest failure, +// ordered by orchestration history. +func (ctx *OrchestrationContext) WhenAll(tasks ...Task) error { + var firstErr error + var firstOrder uint64 + for _, task := range tasks { + err := task.Await(nil) + if err == nil { + continue + } + state, ok := taskState(task) + if !ok { + if firstErr == nil { + firstErr = err + firstOrder = ^uint64(0) + } + continue + } + if firstErr == nil || state.completionID < firstOrder { + firstErr = err + firstOrder = state.completionID + } + } + return firstErr +} + +// Select waits until one case is ready and invokes its handler. +func (ctx *OrchestrationContext) Select(cases ...SelectCase) { + ctx.selectCase(cases) +} + +func (ctx *OrchestrationContext) selectCase(cases []SelectCase) SelectCase { + if len(cases) == 0 { + panic("Select requires at least one case") + } + scheduler := ctx.engineContext().scheduler + if scheduler == nil { + panic("Select called outside orchestrator execution") + } + current := scheduler.mustCurrent() + + for { + if current.scope.isCanceled() || ctx.scope.isCanceled() { + panic(ErrTaskCanceled) + } + var selected SelectCase + var selectedOrder uint64 + for _, candidate := range cases { + ready, order := candidate.ready() + if ready && (selected == nil || order < selectedOrder) { + selected = candidate + selectedOrder = order + } + } + if selected != nil { + selected.invoke() + return selected + } + + ctx.scope.addWaiter(current) + for _, candidate := range cases { + candidate.subscribe(current) + } + current.state = coroutineWaiting + current.yield() + for _, candidate := range cases { + candidate.unsubscribe(current) + } + ctx.scope.removeWaiter(current) + } +} + +func taskState(task Task) (*completableTask, bool) { + state, ok := task.(*completableTask) + return state, ok +} diff --git a/task/select_test.go b/task/select_test.go new file mode 100644 index 00000000..ca2cce53 --- /dev/null +++ b/task/select_test.go @@ -0,0 +1,315 @@ +package task + +import ( + "context" + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestWhenAnyUsesHistoryOrder(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("when-any", func(ctx *OrchestrationContext) (any, error) { + first := ctx.CallActivity("first") + second := ctx.CallActivity("second") + switch ctx.WhenAny(first, second) { + case first: + return "first", nil + case second: + return "second", nil + default: + panic("WhenAny returned an unknown task") + } + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("when-any-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("when-any", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "first", nil, nil, nil), + helpers.NewTaskScheduledEvent(1, "second", nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskCompletedEvent(1, nil), + helpers.NewTaskCompletedEvent(0, nil), + }, + ) + if got, want := completionResult(t, result), `"second"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestWhenAllReturnsFirstFailureByHistory(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("when-all", func(ctx *OrchestrationContext) (any, error) { + first := ctx.CallActivity("first") + second := ctx.CallActivity("second") + return nil, ctx.WhenAll(first, second) + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("when-all-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("when-all", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "first", nil, nil, nil), + helpers.NewTaskScheduledEvent(1, "second", nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewTaskFailedEvent(1, &protos.TaskFailureDetails{ErrorType: "second", ErrorMessage: "second failed"}), + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ErrorType: "first", ErrorMessage: "first failed"}), + }, + ) + completed := completionAction(t, result) + if got := completed.GetFailureDetails().GetErrorMessage(); got != "Task 'second' (#1) failed with an unhandled exception: second failed" { + t.Fatalf("failure = %q", got) + } +} + +func TestSelectChoosesEventBeforeLaterTaskCompletion(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("select", func(ctx *OrchestrationContext) (any, error) { + activity := ctx.CallActivity("activity") + channel := NewEventChannel[string](ctx, "signal") + selected := "" + ctx.Select( + OnTask(activity, func(Task) { selected = "activity" }), + OnEvent(channel, func(value string) { selected = value }), + ) + return selected, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("select-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("select", string(instanceID), nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "activity", nil, nil, nil), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("signal", wrapperspb.String(`"event"`)), + helpers.NewTaskCompletedEvent(0, nil), + }, + ) + if got, want := completionResult(t, result), `"event"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestEventChannelReceivesRepeatedValues(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("channel", func(ctx *OrchestrationContext) (any, error) { + channel := NewEventChannel[int](ctx, "value") + if channel != NewEventChannel[int](ctx, "VALUE") { + return nil, errors.New("same event name returned a different channel") + } + return []int{channel.Receive(ctx), channel.Receive(ctx)}, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("channel-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("channel", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("value", wrapperspb.String("1")), + helpers.NewEventRaisedEvent("VALUE", wrapperspb.String("2")), + }, + ) + if got, want := completionResult(t, result), `[1,2]`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestContinueAsNewCarriesEventsInArrivalOrderAcrossNames(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("carryover", func(ctx *OrchestrationContext) (any, error) { + NewEventChannel[int](ctx, "first") + NewEventChannel[int](ctx, "second") + ctx.ContinueAsNew(nil, WithKeepUnprocessedEvents()) + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("carryover-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("carryover", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("second", wrapperspb.String("2")), + helpers.NewEventRaisedEvent("first", wrapperspb.String("1")), + }, + ) + carryover := completionAction(t, result).GetCarryoverEvents() + if len(carryover) != 2 { + t.Fatalf("carryover count = %d, want 2", len(carryover)) + } + if got := carryover[0].GetEventRaised().GetName(); got != "second" { + t.Fatalf("first carryover event = %q, want second", got) + } + if got := carryover[1].GetEventRaised().GetName(); got != "first" { + t.Fatalf("second carryover event = %q, want first", got) + } +} + +func TestCoroutinesSendEventsInDeterministicOrder(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("send-events", func(ctx *OrchestrationContext) (any, error) { + wg := ctx.NewWaitGroup() + wg.Add(2) + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + if err := ctx.SendEvent("target", "first", 1); err != nil { + panic(err) + } + }) + ctx.Go(func(ctx *OrchestrationContext) { + defer wg.Done() + if err := ctx.SendEvent("target", "second", 2); err != nil { + panic(err) + } + }) + wg.Wait(ctx) + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("send-events-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("send-events", string(instanceID), nil, nil, nil, nil), + }, + ) + if len(result.Actions) != 3 { + t.Fatalf("action count = %d, want two events and completion", len(result.Actions)) + } + for i, name := range []string{"first", "second"} { + action := result.Actions[i] + if action.GetId() != int32(i) || action.GetSendEvent().GetName() != name { + t.Fatalf("action %d = %v", i, action) + } + } +} + +func TestSendEventReplaysFromEventSentHistory(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("send-and-wait", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.SendEvent("target", "ping", 1); err != nil { + return nil, err + } + return nil, ctx.WaitForSingleEvent("done", -1).Await(nil) + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("send-and-wait-instance") + oldEvents := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("send-and-wait", string(instanceID), nil, nil, nil, nil), + helpers.NewSendEventEvent(0, "target", "ping", wrapperspb.String("1")), + } + result := executeOrchestrationTurn(t, registry, instanceID, oldEvents, nil) + if len(result.Actions) != 0 { + t.Fatalf("replay produced unexpected actions: %v", result.Actions) + } +} + +func executeOrchestrationTurn( + t *testing.T, + registry *TaskRegistry, + instanceID api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, +) *protos.OrchestratorResponse { + t.Helper() + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + instanceID, + oldEvents, + newEvents, + supportedEntityParameters(), + ) + if err != nil { + t.Fatal(err) + } + return result.Response +} + +func completionAction(t *testing.T, response *protos.OrchestratorResponse) *protos.CompleteOrchestrationAction { + t.Helper() + for _, action := range response.Actions { + if completed := action.GetCompleteOrchestration(); completed != nil { + return completed + } + } + t.Fatal("orchestration did not produce a completion action") + return nil +} + +func completionResult(t *testing.T, response *protos.OrchestratorResponse) string { + t.Helper() + return completionAction(t, response).GetResult().GetValue() +} + +func TestEventChannelReceiveErrReturnsPayloadError(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("event-error", func(ctx *OrchestrationContext) (any, error) { + _, err := NewEventChannel[int](ctx, "value").ReceiveErr(ctx) + return err != nil, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("event-error-instance") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("event-error", string(instanceID), nil, nil, nil, nil), + helpers.NewEventRaisedEvent("value", wrapperspb.String(`"not-an-int"`)), + }, + ) + if got, want := completionResult(t, result), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} diff --git a/task/suspension_test.go b/task/suspension_test.go new file mode 100644 index 00000000..480194a1 --- /dev/null +++ b/task/suspension_test.go @@ -0,0 +1,196 @@ +package task + +import ( + "fmt" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func suspensionRegistry(t *testing.T) *TaskRegistry { + t.Helper() + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("suspendable", func(ctx *OrchestrationContext) (any, error) { + var payload string + if err := ctx.WaitForSingleEvent("release", -1).Await(&payload); err != nil { + return nil, err + } + return payload, nil + })) + return registry +} + +func suspensionHistory(events ...*protos.HistoryEvent) []*protos.HistoryEvent { + return append([]*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("suspendable", "instance", nil, nil, nil, nil), + }, events...) +} + +// TestRedundantSuspendIsDroppedNotBuffered asserts a suspend received while +// already suspended is dropped instead of buffered. Buffering it would +// re-suspend the orchestration as soon as the matching resume drained the +// buffer, so N suspends would require N resumes to make progress again. +func TestRedundantSuspendIsDroppedNotBuffered(t *testing.T) { + for _, suspendCount := range []int{1, 2, 3} { + t.Run(fmt.Sprintf("suspends=%d", suspendCount), func(t *testing.T) { + events := make([]*protos.HistoryEvent, 0, suspendCount+2) + for range suspendCount { + events = append(events, helpers.NewSuspendOrchestrationEvent("hold")) + } + + events = append( + events, + helpers.NewEventRaisedEvent("release", wrapperspb.String(`"payload"`)), + helpers.NewResumeOrchestrationEvent("go"), + ) + + response := executeOrchestrationTurn( + t, + suspensionRegistry(t), + "instance", + nil, + suspensionHistory(events...), + ) + completed := completionAction(t, response) + require.Equal( + t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + completed.GetOrchestrationStatus(), + ) + require.Equal(t, `"payload"`, completed.GetResult().GetValue()) + }) + } +} + +func TestTerminationReplacesNaturalCompletionInTheSameTurn(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("complete-on-event", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + return "natural", nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("terminate-completion-race") + result := executeOrchestrationTurn( + t, + registry, + instanceID, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent( + "complete-on-event", + string(instanceID), + nil, + nil, + nil, + nil, + ), + }, + []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewEventRaisedEvent("finish", nil), + helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"terminated"`), true), + }, + ) + var completions []*protos.CompleteOrchestrationAction + for _, action := range result.Actions { + if completion := action.GetCompleteOrchestration(); completion != nil { + completions = append(completions, completion) + } + } + if len(completions) != 1 { + t.Fatalf("completion actions = %d, want 1: %v", len(completions), result.Actions) + } + if got, want := completions[0].GetOrchestrationStatus(), + protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED; got != want { + t.Fatalf("completion status = %v, want %v", got, want) + } + if got, want := completions[0].GetResult().GetValue(), `"terminated"`; got != want { + t.Fatalf("completion result = %q, want %q", got, want) + } +} + +// TestSuspendedOrchestrationEmitsNoActions asserts a still-suspended +// orchestration withholds its actions, which is what keeps buffered events from +// being acted upon while suspended. +func TestSuspendedOrchestrationEmitsNoActions(t *testing.T) { + response := executeOrchestrationTurn(t, suspensionRegistry(t), "instance", nil, suspensionHistory( + helpers.NewSuspendOrchestrationEvent("hold"), + helpers.NewEventRaisedEvent("release", wrapperspb.String(`"payload"`)), + )) + require.Empty(t, response.Actions) +} + +// TestTerminationOverridesSuspension asserts termination is processed even +// while suspended and still emits the terminal completion action. +func TestTerminationOverridesSuspension(t *testing.T) { + for _, test := range []struct { + name string + events []*protos.HistoryEvent + }{ + { + name: "terminate-while-suspended", + events: []*protos.HistoryEvent{ + helpers.NewSuspendOrchestrationEvent("hold"), + helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"stopped"`), false), + }, + }, + { + name: "terminate-after-repeated-suspends", + events: []*protos.HistoryEvent{ + helpers.NewSuspendOrchestrationEvent("hold"), + helpers.NewSuspendOrchestrationEvent("hold again"), + helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"stopped"`), false), + }, + }, + { + name: "terminate-with-buffered-events", + events: []*protos.HistoryEvent{ + helpers.NewSuspendOrchestrationEvent("hold"), + helpers.NewEventRaisedEvent("release", wrapperspb.String(`"payload"`)), + helpers.NewExecutionTerminatedEvent(wrapperspb.String(`"stopped"`), false), + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + response := executeOrchestrationTurn( + t, + suspensionRegistry(t), + "instance", + nil, + suspensionHistory(test.events...), + ) + completed := completionAction(t, response) + require.Equal( + t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, + completed.GetOrchestrationStatus(), + ) + require.Equal(t, `"stopped"`, completed.GetResult().GetValue()) + }) + } +} + +// TestResumeWithoutSuspendIsANoOp asserts resuming an orchestration that was +// never suspended does not disturb normal event processing. +func TestResumeWithoutSuspendIsANoOp(t *testing.T) { + response := executeOrchestrationTurn(t, suspensionRegistry(t), "instance", nil, suspensionHistory( + helpers.NewResumeOrchestrationEvent("no-op"), + helpers.NewResumeOrchestrationEvent("still a no-op"), + helpers.NewEventRaisedEvent("release", wrapperspb.String(`"payload"`)), + )) + completed := completionAction(t, response) + require.Equal( + t, + protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, + completed.GetOrchestrationStatus(), + ) + require.Equal(t, `"payload"`, completed.GetResult().GetValue()) +} diff --git a/task/task.go b/task/task.go index 62d6c131..498fc6aa 100644 --- a/task/task.go +++ b/task/task.go @@ -3,7 +3,10 @@ package task import ( "errors" "fmt" + "time" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/failure" "github.com/microsoft/durabletask-go/internal/protos" ) @@ -22,18 +25,36 @@ type Task interface { } type completableTask struct { - orchestrationCtx *OrchestrationContext - isCompleted bool - isCanceled bool - rawResult []byte - failureDetails *protos.TaskFailureDetails - completedCallback func() + orchestrationCtx *OrchestrationContext + isCompleted bool + isCanceled bool + rawResult []byte + failureDetails *protos.TaskFailureDetails + localErr error + taskName string + taskVersion string + taskID int32 + timerFireAt time.Time + entityID api.EntityID + entityOperation string + completionID uint64 + completedCallbacks []func() + waiters map[*coroutine]struct{} + scope *cancellationScope + scopeIndex int } -func newTask(ctx *OrchestrationContext) *completableTask { - return &completableTask{ +func newTaskInScope(ctx *OrchestrationContext, scope *cancellationScope) *completableTask { + task := &completableTask{ orchestrationCtx: ctx, + waiters: make(map[*coroutine]struct{}), + scope: scope, + scopeIndex: -1, } + if scope != nil { + scope.addTask(task) + } + return task } // Await blocks the current orchestrator until the task is complete and then saves the unmarshalled @@ -48,19 +69,44 @@ func newTask(ctx *OrchestrationContext) *completableTask { func (t *completableTask) Await(v any) error { for { if t.isCompleted { - if t.failureDetails != nil { - return fmt.Errorf("task failed with an error: %v", t.failureDetails.ErrorMessage) - } else if t.isCanceled { + if t.localErr != nil { + return t.localErr + } + if t.isCanceled { return ErrTaskCanceled } + if t.failureDetails != nil { + details := failure.FromProto(t.failureDetails) + if t.entityOperation != "" { + return &EntityOperationFailedError{ + EntityID: t.entityID, + OperationName: t.entityOperation, + FailureDetails: details, + } + } + return &TaskFailedError{ + TaskName: t.taskName, + TaskVersion: t.taskVersion, + TaskID: t.taskID, + FailureDetails: details, + } + } if v != nil && len(t.rawResult) > 0 { - if err := unmarshalData(t.rawResult, v); err != nil { + if err := unmarshalData(t.orchestrationCtx.converter, t.rawResult, v); err != nil { return fmt.Errorf("failed to decode task result: %w", err) } } return nil } + if scheduler := t.orchestrationCtx.scheduler; scheduler != nil { + if current := scheduler.mustCurrent(); current.scope.isCanceled() { + return ErrTaskCanceled + } + scheduler.waitForTask(t) + continue + } + ok, err := t.orchestrationCtx.processNextEvent() if err != nil { // TODO: If there is an error here, we need some kind of well-known panic to kill the orchestration @@ -75,38 +121,70 @@ func (t *completableTask) Await(v any) error { } func (t *completableTask) onCompleted(callback func()) { - t.completedCallback = callback + if t.isCompleted { + callback() + return + } + t.completedCallbacks = append(t.completedCallbacks, callback) } func (t *completableTask) complete(rawResult []byte) { + if t.isCompleted { + return + } t.rawResult = rawResult t.completeInternal() } func (t *completableTask) fail(fd *protos.TaskFailureDetails) { + if t.isCompleted { + return + } t.failureDetails = fd t.completeInternal() } +func (t *completableTask) failLocal(err error) { + if t.isCompleted { + return + } + t.localErr = err + t.completeInternal() +} + func (t *completableTask) cancel() { + if t.isCompleted { + return + } t.isCanceled = true t.completeInternal() } func (t *completableTask) completeInternal() { + if t.isCompleted { + return + } t.isCompleted = true - if t.completedCallback != nil { - t.completedCallback() + t.scope.removeTask(t) + if scheduler := t.orchestrationCtx.scheduler; scheduler != nil { + t.completionID = scheduler.nextCompletionID() + for waiter := range t.waiters { + scheduler.makeRunnable(waiter) + } + clear(t.waiters) } + for _, callback := range t.completedCallbacks { + callback() + } + t.completedCallbacks = nil } -type taskWrapper struct { - delegate Task - onAwaitResult func(any, error) error +func (t *completableTask) addWaiter(c *coroutine) { + if !t.isCompleted { + t.waiters[c] = struct{}{} + } } -var _ Task = &taskWrapper{} - -func (t *taskWrapper) Await(v any) error { - return t.onAwaitResult(v, t.delegate.Await(v)) +func (t *completableTask) removeWaiter(c *coroutine) { + delete(t.waiters, c) } diff --git a/task/task_completion_test.go b/task/task_completion_test.go new file mode 100644 index 00000000..0f159fe3 --- /dev/null +++ b/task/task_completion_test.go @@ -0,0 +1,26 @@ +package task + +import ( + "testing" + + "github.com/microsoft/durabletask-go/internal/protos" +) + +func TestCompletedTaskIgnoresLaterCancellationFailureAndCompletion(t *testing.T) { + ctx := newTestOrchestrationContext(NewTaskRegistry(), "completed-task", nil, nil) + scope := newCancellationScope(nil) + task := newTaskInScope(ctx, scope) + task.complete([]byte(`"first"`)) + + scope.cancel(nil) + task.fail(&protos.TaskFailureDetails{ErrorMessage: "late failure"}) + task.complete([]byte(`"second"`)) + + var result string + if err := task.Await(&result); err != nil { + t.Fatal(err) + } + if result != "first" { + t.Fatalf("result = %q, want first", result) + } +} diff --git a/task/timer_test.go b/task/timer_test.go new file mode 100644 index 00000000..e0ecd821 --- /dev/null +++ b/task/timer_test.go @@ -0,0 +1,375 @@ +package task + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestLongTimerSplitsIntoDeterministicSequentialChunks(t *testing.T) { + const ( + maximumInterval = 3 * time.Hour + delay = 7 * time.Hour + ) + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("long-timer", func(ctx *OrchestrationContext) (any, error) { + return "done", ctx.CreateTimer(delay).Await(nil) + }); err != nil { + t.Fatal(err) + } + executor := NewTaskExecutor(registry, WithMaximumTimerInterval(maximumInterval)) + instanceID := api.InstanceID("long-timer-instance") + startTime := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + started := timerOrchestratorStartedAt(startTime) + executionStarted := helpers.NewExecutionStartedEvent( + "long-timer", + string(instanceID), + nil, + nil, + nil, + nil, + ) + + var oldEvents []*protos.HistoryEvent + newEvents := []*protos.HistoryEvent{started, executionStarted} + response := executeTimerTurn(t, executor, instanceID, oldEvents, newEvents) + expectedFireTimes := []time.Time{ + startTime.Add(maximumInterval), + startTime.Add(2 * maximumInterval), + startTime.Add(delay), + } + for timerID, expectedFireTime := range expectedFireTimes { + timer := onlyTimerAction(t, response) + requireTimerFireAt(t, timer, expectedFireTime) + + oldEvents = append(oldEvents, newEvents...) + oldEvents = append( + oldEvents, + helpers.NewTimerCreatedEvent(int32(timerID), timer.GetFireAt()), + ) + newEvents = []*protos.HistoryEvent{ + timerOrchestratorStartedAt(timer.GetFireAt().AsTime()), + helpers.NewTimerFiredEvent(int32(timerID), timer.GetFireAt(), nil), + } + response = executeTimerTurn(t, executor, instanceID, oldEvents, newEvents) + } + if got, want := completionResult(t, response), `"done"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } +} + +func TestLongTimerReplaysHistoryCreatedBeforeTimerSplitting(t *testing.T) { + const delay = 7 * time.Hour + registry := NewTaskRegistry() + if err := registry.AddActivityN("after-timer", func(ActivityContext) (any, error) { + return "done", nil + }); err != nil { + t.Fatal(err) + } + if err := registry.AddOrchestratorN("legacy-long-timer", func(ctx *OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(delay).Await(nil); err != nil { + return nil, err + } + var result string + if err := ctx.CallActivity("after-timer").Await(&result); err != nil { + return nil, err + } + return result, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("legacy-long-timer-instance") + startTime := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + deadline := startTime.Add(delay) + history := []*protos.HistoryEvent{ + timerOrchestratorStartedAt(startTime), + helpers.NewExecutionStartedEvent( + "legacy-long-timer", + string(instanceID), + nil, + nil, + nil, + nil, + ), + helpers.NewTimerCreatedEvent(0, timestamppb.New(deadline)), + timerOrchestratorStartedAt(deadline.Add(-time.Second)), + helpers.NewTimerFiredEvent(0, timestamppb.New(deadline), nil), + helpers.NewTaskScheduledEvent(1, "after-timer", nil, nil, nil), + timerOrchestratorStartedAt(deadline), + helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"done"`)), + } + response := executeTimerTurn( + t, + NewTaskExecutor(registry, WithMaximumTimerInterval(3*time.Hour)), + instanceID, + history, + nil, + ) + if got, want := completionResult(t, response), `"done"`; got != want { + t.Fatalf("result = %s, want %s", got, want) + } + for _, action := range response.Actions { + if action.GetCreateTimer() != nil || action.GetScheduleTask() != nil { + t.Fatalf("legacy timer replay produced unexpected action: %v", action) + } + } +} + +func TestCreateTimerPreservesZeroAndNegativeDeadlines(t *testing.T) { + tests := []struct { + name string + delay time.Duration + }{ + {name: "zero", delay: 0}, + {name: "negative", delay: -time.Hour}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("timer-edge", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CreateTimer(test.delay).Await(nil) + }); err != nil { + t.Fatal(err) + } + startTime := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + instanceID := api.InstanceID("timer-edge-" + test.name) + response := executeTimerTurn( + t, + NewTaskExecutor(registry), + instanceID, + nil, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(startTime), + helpers.NewExecutionStartedEvent("timer-edge", string(instanceID), nil, nil, nil, nil), + }, + ) + requireTimerFireAt(t, onlyTimerAction(t, response), startTime.Add(test.delay)) + }) + } +} + +func TestCreateTimerHandlesExtremeDurationsAndOverflow(t *testing.T) { + const maximumDuration = time.Duration(1<<63 - 1) + startTime := time.Date(2000, time.January, 1, 0, 0, 0, 0, time.UTC) + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("extreme-timer", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CreateTimer(maximumDuration).Await(nil) + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("extreme-timer-instance") + response := executeTimerTurn( + t, + NewTaskExecutor(registry, WithMaximumTimerInterval(maximumDuration)), + instanceID, + nil, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(startTime), + helpers.NewExecutionStartedEvent("extreme-timer", string(instanceID), nil, nil, nil, nil), + }, + ) + requireTimerFireAt(t, onlyTimerAction(t, response), startTime.Add(maximumDuration)) + + overflowRegistry := NewTaskRegistry() + if err := overflowRegistry.AddOrchestratorN("overflow-timer", func(ctx *OrchestrationContext) (any, error) { + err := ctx.CreateTimer(time.Hour).Await(nil) + return errors.Is(err, api.ErrInvalidArgument), nil + }); err != nil { + t.Fatal(err) + } + overflowStart := time.Date(9999, time.December, 31, 23, 30, 0, 0, time.UTC) + overflowID := api.InstanceID("overflow-timer-instance") + overflow := executeTimerTurn( + t, + NewTaskExecutor(overflowRegistry), + overflowID, + nil, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(overflowStart), + helpers.NewExecutionStartedEvent("overflow-timer", string(overflowID), nil, nil, nil, nil), + }, + ) + if got, want := completionResult(t, overflow), "true"; got != want { + t.Fatalf("overflow result = %s, want %s", got, want) + } + for _, action := range overflow.Actions { + if action.GetCreateTimer() != nil { + t.Fatalf("overflow scheduled timer action %v", action) + } + } +} + +func TestMaximumTimerIntervalConfiguration(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("default-timer", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CreateTimer(7 * 24 * time.Hour).Await(nil) + }); err != nil { + t.Fatal(err) + } + startTime := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + instanceID := api.InstanceID("default-timer-instance") + response := executeTimerTurn( + t, + NewTaskExecutor(registry, WithMaximumTimerInterval(0)), + instanceID, + nil, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(startTime), + helpers.NewExecutionStartedEvent("default-timer", string(instanceID), nil, nil, nil, nil), + }, + ) + requireTimerFireAt(t, onlyTimerAction(t, response), startTime.Add(DefaultMaximumTimerInterval)) + + defer func() { + if recovered := recover(); recovered == nil { + t.Fatal("negative maximum timer interval did not panic") + } + }() + NewTaskExecutor(registry, WithMaximumTimerInterval(-time.Second)) +} + +func TestLongTimerCancellationStopsFutureChunks(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("cancel-long-timer", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + timer := child.CreateTimer(7 * time.Hour) + cancel() + return errors.Is(timer.Await(nil), ErrTaskCanceled), nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("cancel-long-timer-instance") + response := executeTimerTurn( + t, + NewTaskExecutor(registry, WithMaximumTimerInterval(3*time.Hour)), + instanceID, + nil, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC)), + helpers.NewExecutionStartedEvent("cancel-long-timer", string(instanceID), nil, nil, nil, nil), + }, + ) + if got, want := completionResult(t, response), "true"; got != want { + t.Fatalf("result = %s, want %s", got, want) + } + timerCount := 0 + for _, action := range response.Actions { + if action.GetCreateTimer() != nil { + timerCount++ + } + } + if timerCount != 1 { + t.Fatalf("timer actions = %d, want 1", timerCount) + } +} + +func TestLongTimerHonorsHistoryLimits(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("limited-long-timer", func(ctx *OrchestrationContext) (any, error) { + return nil, ctx.CreateTimer(7 * time.Hour).Await(nil) + }); err != nil { + t.Fatal(err) + } + executor := NewTaskExecutor( + registry, + WithOrchestrationOptions(OrchestrationOptions{ + MaximumTimerInterval: 3 * time.Hour, + MaxHistoryEvents: 4, + }), + ) + instanceID := api.InstanceID("limited-long-timer-instance") + startTime := time.Date(2026, time.August, 27, 12, 0, 0, 0, time.UTC) + started := timerOrchestratorStartedAt(startTime) + executionStarted := helpers.NewExecutionStartedEvent( + "limited-long-timer", + string(instanceID), + nil, + nil, + nil, + nil, + ) + first := executeTimerTurn( + t, + executor, + instanceID, + nil, + []*protos.HistoryEvent{started, executionStarted}, + ) + timer := onlyTimerAction(t, first) + second := executeTimerTurn( + t, + executor, + instanceID, + []*protos.HistoryEvent{ + started, + executionStarted, + helpers.NewTimerCreatedEvent(0, timer.GetFireAt()), + }, + []*protos.HistoryEvent{ + timerOrchestratorStartedAt(timer.GetFireAt().AsTime()), + helpers.NewTimerFiredEvent(0, timer.GetFireAt(), nil), + }, + ) + failure := completionAction(t, second).GetFailureDetails() + if got, want := failure.GetErrorType(), string(api.ErrorTypeHistoryLimitExceeded); got != want { + t.Fatalf("failure type = %q, want %q", got, want) + } +} + +func executeTimerTurn( + t *testing.T, + executor Executor, + instanceID api.InstanceID, + oldEvents []*protos.HistoryEvent, + newEvents []*protos.HistoryEvent, +) *protos.OrchestratorResponse { + t.Helper() + result, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + oldEvents, + newEvents, nil) + + if err != nil { + t.Fatal(err) + } + return result.Response +} + +func timerOrchestratorStartedAt(timestamp time.Time) *protos.HistoryEvent { + event := helpers.NewOrchestratorStartedEvent() + event.Timestamp = timestamppb.New(timestamp) + return event +} + +func onlyTimerAction(t *testing.T, response *protos.OrchestratorResponse) *protos.CreateTimerAction { + t.Helper() + var timer *protos.CreateTimerAction + for _, action := range response.Actions { + if candidate := action.GetCreateTimer(); candidate != nil { + if timer != nil { + t.Fatalf("multiple timer actions in one turn: %v", response.Actions) + } + timer = candidate + } + } + if timer == nil { + t.Fatalf("no timer action in response: %v", response.Actions) + } + return timer +} + +func requireTimerFireAt(t *testing.T, timer *protos.CreateTimerAction, expected time.Time) { + t.Helper() + if actual := timer.GetFireAt().AsTime(); !actual.Equal(expected) { + t.Fatalf("timer fire time = %v, want %v", actual, expected) + } +} diff --git a/task/versioning.go b/task/versioning.go new file mode 100644 index 00000000..083bec51 --- /dev/null +++ b/task/versioning.go @@ -0,0 +1,204 @@ +package task + +import ( + "fmt" + "strconv" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" +) + +// VersionMatchStrategy determines which task versions a worker accepts. +type VersionMatchStrategy int + +const ( + VersionMatchNone VersionMatchStrategy = iota + VersionMatchStrict + VersionMatchCurrentOrOlder +) + +// VersionFailureStrategy determines how a worker handles a version mismatch. +type VersionFailureStrategy int + +const ( + VersionFailureReject VersionFailureStrategy = iota + VersionFailureFail +) + +// UnversionedTaskVersion explicitly selects an unversioned task. +const UnversionedTaskVersion = "" + +// VersioningOptions configures version-aware orchestration and activity dispatch. +type VersioningOptions struct { + Version string + DefaultVersion string + MatchStrategy VersionMatchStrategy + FailureStrategy VersionFailureStrategy +} + +// Validate checks that the versioning configuration is supported. +func (o VersioningOptions) Validate() error { + switch o.MatchStrategy { + case VersionMatchNone, VersionMatchStrict, VersionMatchCurrentOrOlder: + default: + return &versionConfigurationError{strategy: o.MatchStrategy} + } + switch o.FailureStrategy { + case VersionFailureReject, VersionFailureFail: + default: + return fmt.Errorf("unknown version failure strategy %d", o.FailureStrategy) + } + if _, err := normalizeTaskVersionForRegistration(o.Version); err != nil { + return fmt.Errorf("invalid worker version: %w", err) + } + if _, err := normalizeTaskVersionForRegistration(o.DefaultVersion); err != nil { + return fmt.Errorf("invalid default version: %w", err) + } + return nil +} + +// VersionMismatchError indicates that a work item is incompatible with this worker. +type VersionMismatchError struct { + TaskVersion string + WorkerVersion string + Strategy VersionMatchStrategy +} + +func (e *VersionMismatchError) Error() string { + return fmt.Sprintf( + "task version %q does not match worker version %q using strategy %d", + e.TaskVersion, + e.WorkerVersion, + e.Strategy, + ) +} + +func (*VersionMismatchError) DurableTaskErrorType() api.ErrorType { + return versionMismatchErrorType +} + +func (*VersionMismatchError) NonRetriable() bool { + return true +} + +func (*VersionMismatchError) Is(target error) bool { + return target == api.ErrVersionMismatch +} + +// WorkItemAbandonDelay asks the worker to delay redelivery of a +// version-incompatible activity work item instead of retrying it immediately. +func (*VersionMismatchError) WorkItemAbandonDelay() time.Duration { + return time.Second +} + +func (o *VersioningOptions) check(taskVersion string) error { + if o == nil { + return nil + } + if err := o.Validate(); err != nil { + return err + } + if o.MatchStrategy == VersionMatchNone { + return nil + } + comparison := compareVersions(taskVersion, o.Version) + switch o.MatchStrategy { + case VersionMatchStrict: + if comparison == 0 { + return nil + } + case VersionMatchCurrentOrOlder: + if comparison <= 0 { + return nil + } + default: + return &versionConfigurationError{strategy: o.MatchStrategy} + } + + return &VersionMismatchError{ + TaskVersion: taskVersion, + WorkerVersion: o.Version, + Strategy: o.MatchStrategy, + } +} + +func (o *VersioningOptions) defaultVersion() string { + if o == nil { + return "" + } + return o.DefaultVersion +} + +type versionConfigurationError struct { + strategy VersionMatchStrategy +} + +func (e *versionConfigurationError) Error() string { + return fmt.Sprintf("unknown version match strategy %d", e.strategy) +} + +func (*versionConfigurationError) DurableTaskErrorType() api.ErrorType { + return api.ErrorTypeVersionError +} + +func (*versionConfigurationError) NonRetriable() bool { + return true +} + +func (*versionConfigurationError) Is(target error) bool { + return target == api.ErrVersionMismatch +} + +func compareVersions(left, right string) int { + left = strings.TrimSpace(left) + right = strings.TrimSpace(right) + switch { + case left == "" && right == "": + return 0 + case left == "": + return -1 + case right == "": + return 1 + } + + leftParts, leftOK := numericVersion(left) + rightParts, rightOK := numericVersion(right) + if leftOK && rightOK { + maxParts := max(len(leftParts), len(rightParts)) + for i := 0; i < maxParts; i++ { + leftValue := -1 + if i < len(leftParts) { + leftValue = leftParts[i] + } + rightValue := -1 + if i < len(rightParts) { + rightValue = rightParts[i] + } + if leftValue < rightValue { + return -1 + } + if leftValue > rightValue { + return 1 + } + } + return 0 + } + return strings.Compare(strings.ToLower(left), strings.ToLower(right)) +} + +func numericVersion(version string) ([]int, bool) { + parts := strings.Split(version, ".") + if len(parts) < 2 || len(parts) > 4 { + return nil, false + } + values := make([]int, len(parts)) + for i, part := range parts { + value, err := strconv.Atoi(part) + if err != nil || value < 0 { + return nil, false + } + values[i] = value + } + return values, true +} diff --git a/task/versioning_test.go b/task/versioning_test.go new file mode 100644 index 00000000..5c0aebe4 --- /dev/null +++ b/task/versioning_test.go @@ -0,0 +1,494 @@ +package task + +import ( + "context" + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestCompareVersionsMatchesDurableTaskRules(t *testing.T) { + tests := []struct { + left string + right string + want int + }{ + {"", "", 0}, + {"", "1.0", -1}, + {"1.0", "", 1}, + {"1.2", "1.10", -1}, + {"2.0.0", "2.0", 1}, + {"preview-A", "preview-a", 0}, + {"preview-b", "preview-a", 1}, + } + for _, test := range tests { + got := compareVersions(test.left, test.right) + if got < 0 { + got = -1 + } else if got > 0 { + got = 1 + } + if got != test.want { + t.Fatalf("compareVersions(%q, %q) = %d, want %d", test.left, test.right, got, test.want) + } + } +} + +func TestVersionMismatchRejectsOrchestration(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("versioned", func(*OrchestrationContext) (any, error) { + return nil, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("version-reject") + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + Version: "1.0", + MatchStrategy: VersionMatchCurrentOrOlder, + FailureStrategy: VersionFailureReject, + })) + _, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent( + "versioned", + string(instanceID), + nil, + nil, + nil, + nil, + wrapperspb.String("2.0"), + ), + }, nil) + + var mismatch *VersionMismatchError + if !errors.As(err, &mismatch) { + t.Fatalf("error = %v, want VersionMismatchError", err) + } +} + +func TestVersionMismatchFailsOrchestration(t *testing.T) { + registry := NewTaskRegistry() + instanceID := api.InstanceID("version-fail") + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + Version: "1.0", + MatchStrategy: VersionMatchStrict, + FailureStrategy: VersionFailureFail, + })) + result, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent( + "versioned", + string(instanceID), + nil, + nil, + nil, + nil, + wrapperspb.String("2.0"), + ), + }, nil) + + if err != nil { + t.Fatal(err) + } + completed := result.Response.Actions[0].GetCompleteOrchestration() + if completed.GetOrchestrationStatus() != protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { + t.Fatalf("status = %v", completed.GetOrchestrationStatus()) + } + if !completed.GetFailureDetails().GetIsNonRetriable() { + t.Fatal("version mismatch failure must be non-retriable") + } +} + +func TestVersionMismatchFailsActivity(t *testing.T) { + registry := NewTaskRegistry() + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + Version: "1.0", + MatchStrategy: VersionMatchStrict, + FailureStrategy: VersionFailureFail, + })) + event := helpers.NewTaskScheduledEvent( + 1, + "activity", + wrapperspb.String("2.0"), + nil, + nil, + ) + result, err := executor.ExecuteActivity(context.Background(), "instance", event) + if err != nil { + t.Fatal(err) + } + failed := result.GetTaskFailed() + if failed == nil || !failed.GetFailureDetails().GetIsNonRetriable() { + t.Fatalf("activity result = %v", result) + } +} + +func TestVersionedRegistryDispatchAndSchedulingDefaults(t *testing.T) { + registry := NewTaskRegistry() + requireNoError := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + requireNoError(registry.AddOrchestratorNVersion("parent", "v1", func(ctx *OrchestrationContext) (any, error) { + ctx.CallActivity("activity") + ctx.CallSubOrchestrator("child") + ctx.ContinueAsNew("next", WithContinueAsNewVersion("v3")) + return nil, nil + })) + + instanceID := api.InstanceID("versioned-defaults") + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + DefaultVersion: "v2", + MatchStrategy: VersionMatchNone, + })) + result, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent( + "parent", + string(instanceID), + nil, + nil, + nil, + nil, + wrapperspb.String("v1"), + ), + }, nil) + + if err != nil { + t.Fatal(err) + } + + var activityVersion, subVersion, newVersion string + for _, action := range result.Response.Actions { + switch { + case action.GetScheduleTask() != nil: + activityVersion = action.GetScheduleTask().GetVersion().GetValue() + case action.GetCreateSubOrchestration() != nil: + subVersion = action.GetCreateSubOrchestration().GetVersion().GetValue() + case action.GetCompleteOrchestration() != nil: + newVersion = action.GetCompleteOrchestration().GetNewVersion().GetValue() + } + } + if activityVersion != "v1" { + t.Fatalf("activity version = %q, want inherited v1", activityVersion) + } + if subVersion != "v2" { + t.Fatalf("sub-orchestration version = %q, want default v2", subVersion) + } + if newVersion != "v3" { + t.Fatalf("continue-as-new version = %q, want v3", newVersion) + } +} + +func TestAllowedUnversionedSystemOrchestratorBypassesStrictVersionMatch(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("system", func(*OrchestrationContext) (any, error) { + return "ok", nil + }); err != nil { + t.Fatal(err) + } + executor := NewTaskExecutor( + registry, + WithVersioning(VersioningOptions{ + Version: "1.0", + MatchStrategy: VersionMatchStrict, + }), + WithUnversionedOrchestratorNames("system"), + ) + result, err := executor.ExecuteOrchestrator( + context.Background(), + "system-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("system", "system-instance", nil, nil, nil, nil, wrapperspb.String("")), + }, nil) + + if err != nil { + t.Fatal(err) + } + if result.Response.Actions[len(result.Response.Actions)-1].GetCompleteOrchestration() == nil { + t.Fatalf("system orchestrator did not complete: %#v", result.Response.Actions) + } +} + +// TestAllowedUnversionedSystemActivityBypassesStrictVersionMatch covers the +// activity half of the same contract: a system orchestration runs unversioned, +// and an activity inherits its caller's version, so its work item arrives +// unversioned and would otherwise be rejected by a strict worker. +func TestAllowedUnversionedSystemActivityBypassesStrictVersionMatch(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddActivityN("SystemActivity", func(ActivityContext) (any, error) { + return "ok", nil + }); err != nil { + t.Fatal(err) + } + scheduled := helpers.NewTaskScheduledEvent(1, "SystemActivity", wrapperspb.String(""), nil, nil) + versioning := VersioningOptions{Version: "1.0", MatchStrategy: VersionMatchStrict} + + rejected := NewTaskExecutor(registry, WithVersioning(versioning)) + if _, err := rejected.ExecuteActivity(context.Background(), "system-instance", scheduled); err == nil { + t.Fatal("expected a strict version mismatch without the allow-list") + } + + allowed := NewTaskExecutor( + registry, + WithVersioning(versioning), + WithUnversionedActivityNames("systemactivity"), + ) + response, err := allowed.ExecuteActivity(context.Background(), "system-instance", scheduled) + if err != nil { + t.Fatal(err) + } + if response.GetTaskCompleted() == nil { + t.Fatalf("system activity did not complete: %#v", response) + } + + // A versioned work item for the same name is still version-checked. + versionedScheduled := helpers.NewTaskScheduledEvent(2, "SystemActivity", wrapperspb.String("9.9"), nil, nil) + if _, err := allowed.ExecuteActivity(context.Background(), "system-instance", versionedScheduled); err == nil { + t.Fatal("expected a strict version mismatch for a versioned work item") + } +} + +func TestSubOrchestrationOptionsAddTagsAndContextFields(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("parent", func(ctx *OrchestrationContext) (any, error) { + ctx.CallSubOrchestrator( + "child", + WithSubOrchestrationTags(map[string]string{"schedule": "daily"}), + WithSubOrchestrationContextFields(api.ContextFields{"tenant": "north"}), + ) + return nil, nil + }); err != nil { + t.Fatal(err) + } + executor := NewTaskExecutor(registry) + result, err := executor.ExecuteOrchestrator( + context.Background(), + "parent-instance", + nil, + []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("parent", "parent-instance", nil, nil, nil, nil), + }, nil) + + if err != nil { + t.Fatal(err) + } + for _, action := range result.Response.Actions { + if created := action.GetCreateSubOrchestration(); created != nil { + if created.Tags["schedule"] != "daily" { + t.Fatalf("tags = %#v", created.Tags) + } + if created.Tags["__durabletask.context.field.tenant"] != "north" { + t.Fatalf("context tags = %#v", created.Tags) + } + return + } + } + t.Fatal("missing sub-orchestration action") +} + +func TestActivityTagsOverrideOrchestrationTagsAndAreCloned(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("parent", func(ctx *OrchestrationContext) (any, error) { + tags := map[string]string{"scope": "activity", "child": "yes"} + ctx.CallActivity("activity", WithActivityTags(tags)) + tags["child"] = "mutated" + return nil, nil + }); err != nil { + t.Fatal(err) + } + started := helpers.NewExecutionStartedEvent("parent", "parent-instance", nil, nil, nil, nil) + started.GetExecutionStarted().Tags = map[string]string{"scope": "orchestration", "parent": "yes"} + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "parent-instance", + nil, + []*protos.HistoryEvent{started}, nil) + + if err != nil { + t.Fatal(err) + } + var foundActivity, foundCompletion bool + for _, action := range result.Response.Actions { + if scheduled := action.GetScheduleTask(); scheduled != nil { + if scheduled.Tags["scope"] != "activity" || + scheduled.Tags["parent"] != "yes" || + scheduled.Tags["child"] != "yes" { + t.Fatalf("activity tags = %#v", scheduled.Tags) + } + foundActivity = true + } + if completed := action.GetCompleteOrchestration(); completed != nil { + if completed.Tags["scope"] != "orchestration" || completed.Tags["parent"] != "yes" { + t.Fatalf("completion tags = %#v", completed.Tags) + } + foundCompletion = true + } + } + if !foundActivity || !foundCompletion { + t.Fatalf("found activity=%t completion=%t", foundActivity, foundCompletion) + } +} + +func TestActivityTagsRejectReservedAndEmptyKeys(t *testing.T) { + for _, tags := range []map[string]string{ + {"": "value"}, + {api.ReservedContextFieldPrefix + "tenant": "value"}, + } { + if err := WithActivityTags(tags)(new(callActivityOptions), api.DefaultDataConverter()); err == nil { + t.Fatalf("WithActivityTags(%#v) succeeded", tags) + } + } +} + +func TestActivityTagValidationIsDeterministic(t *testing.T) { + tags := map[string]string{ + api.ReservedContextFieldPrefix + "z": "value", + api.ReservedContextFieldPrefix + "a": "value", + } + for range 20 { + err := WithActivityTags(tags)(new(callActivityOptions), api.DefaultDataConverter()) + if err == nil || err.Error() != `activity tag "__durabletask.context.a" uses a reserved prefix` { + t.Fatalf("unexpected validation error: %v", err) + } + } +} + +func TestContinueAsNewCarriesTagsAndNextVersion(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorNVersion("eternal", "v1", func(ctx *OrchestrationContext) (any, error) { + ctx.ContinueAsNew("next", WithContinueAsNewVersion("v2")) + return nil, nil + }); err != nil { + t.Fatal(err) + } + started := helpers.NewExecutionStartedEvent( + "eternal", + "eternal-instance", + nil, + nil, + nil, + nil, + wrapperspb.String("v1"), + ) + started.GetExecutionStarted().Tags = map[string]string{ + "tenant": "north", + "__durabletask.context.field.correlation_id": "42", + } + result, err := NewTaskExecutor(registry).ExecuteOrchestrator( + context.Background(), + "eternal-instance", + nil, + []*protos.HistoryEvent{started}, nil) + + if err != nil { + t.Fatal(err) + } + completed := completionAction(t, result.Response) + if completed.GetTags()["tenant"] != "north" { + t.Fatalf("continue-as-new tags = %#v", completed.GetTags()) + } + if completed.GetTags()["__durabletask.context.field.correlation_id"] != "42" { + t.Fatalf("continue-as-new context tags = %#v", completed.GetTags()) + } + if completed.GetTags()["__durabletask.context.orchestration_version"] != "v2" { + t.Fatalf("continue-as-new version tag = %#v", completed.GetTags()) + } +} + +func TestExplicitUnversionedSchedulingOverridesDefaults(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorNVersion("parent", "v1", func(ctx *OrchestrationContext) (any, error) { + ctx.CallActivity("activity", WithActivityVersion("")) + ctx.CallSubOrchestrator("child", WithSubOrchestrationVersion("")) + ctx.ContinueAsNew(nil, WithContinueAsNewVersion("")) + return nil, nil + }); err != nil { + t.Fatal(err) + } + instanceID := api.InstanceID("explicit-unversioned") + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + DefaultVersion: "v2", + MatchStrategy: VersionMatchNone, + })) + result, err := executor.ExecuteOrchestrator( + context.Background(), + instanceID, + nil, + []*protos.HistoryEvent{helpers.NewExecutionStartedEvent( + "parent", + string(instanceID), + nil, + nil, + nil, + nil, + wrapperspb.String("v1"), + )}, nil) + + if err != nil { + t.Fatal(err) + } + for _, action := range result.Response.Actions { + if scheduled := action.GetScheduleTask(); scheduled != nil && scheduled.Version == nil { + t.Fatal("explicit unversioned activity must retain wrapper presence") + } + if scheduled := action.GetCreateSubOrchestration(); scheduled != nil && scheduled.Version == nil { + t.Fatal("explicit unversioned sub-orchestration must retain wrapper presence") + } + if completed := action.GetCompleteOrchestration(); completed != nil { + if completed.NewVersion == nil || completed.NewVersion.GetValue() != "" { + t.Fatal("explicit unversioned ContinueAsNew must retain wrapper presence") + } + } + } +} + +func TestVersionedReplayAcceptsLegacyUnversionedChildHistory(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorNVersion("parent", "v1", func(ctx *OrchestrationContext) (any, error) { + ctx.CallActivity("activity") + ctx.CallSubOrchestrator("child") + return nil, nil + }); err != nil { + t.Fatal(err) + } + + instanceID := api.InstanceID("legacy-unversioned-children") + executor := NewTaskExecutor(registry, WithVersioning(VersioningOptions{ + DefaultVersion: "v2", + MatchStrategy: VersionMatchNone, + })) + oldEvents := []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent( + "parent", + string(instanceID), + nil, + nil, + nil, + nil, + wrapperspb.String("v1"), + ), + helpers.NewTaskScheduledEvent(0, "activity", nil, nil, nil), + helpers.NewSubOrchestrationCreatedEvent(1, "child", nil, nil, "child-instance", nil), + } + + if _, err := executor.ExecuteOrchestrator(context.Background(), instanceID, oldEvents, nil, nil); err != nil { + t.Fatalf("legacy unversioned child history failed replay: %v", err) + } +} diff --git a/task/waitgroup.go b/task/waitgroup.go new file mode 100644 index 00000000..6eb39b44 --- /dev/null +++ b/task/waitgroup.go @@ -0,0 +1,60 @@ +package task + +// WaitGroup coordinates orchestration coroutines. It behaves like sync.WaitGroup, +// but Wait yields to the deterministic orchestration scheduler. +type WaitGroup interface { + Add(delta int) + Done() + Wait(ctx *OrchestrationContext) +} + +type orchestrationWaitGroup struct { + scheduler *coroutineScheduler + count int + waiters map[*coroutine]struct{} +} + +func newOrchestrationWaitGroup(scheduler *coroutineScheduler) *orchestrationWaitGroup { + return &orchestrationWaitGroup{ + scheduler: scheduler, + waiters: make(map[*coroutine]struct{}), + } +} + +func (wg *orchestrationWaitGroup) Add(delta int) { + next := wg.count + delta + if next < 0 { + panic("task.WaitGroup counter became negative") + } + wg.count = next + if next != 0 { + return + } + for c := range wg.waiters { + wg.scheduler.makeRunnable(c) + } + clear(wg.waiters) +} + +func (wg *orchestrationWaitGroup) Done() { + wg.Add(-1) +} + +func (wg *orchestrationWaitGroup) Wait(ctx *OrchestrationContext) { + if ctx.engineContext().scheduler != wg.scheduler { + panic("task.WaitGroup belongs to a different orchestration") + } + for wg.count > 0 { + c := wg.scheduler.mustCurrent() + ctx.scope.addWaiter(c) + if c.scope.isCanceled() || ctx.scope.isCanceled() { + ctx.scope.removeWaiter(c) + panic(ErrTaskCanceled) + } + wg.waiters[c] = struct{}{} + c.state = coroutineWaiting + c.yield() + delete(wg.waiters, c) + ctx.scope.removeWaiter(c) + } +} diff --git a/tests/backend_test.go b/tests/backend_test.go deleted file mode 100644 index 1d72729a..00000000 --- a/tests/backend_test.go +++ /dev/null @@ -1,537 +0,0 @@ -package tests - -import ( - "context" - "fmt" - "github.com/microsoft/durabletask-go/backend/postgres" - "os" - "reflect" - "runtime" - "testing" - "time" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/types/known/timestamppb" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -var ( - ctx = context.Background() - logger = backend.DefaultLogger() - sqliteInMemoryOptions = sqlite.NewSqliteOptions("") - sqliteFileOptions = sqlite.NewSqliteOptions("test.sqlite3") -) - -func getRunnableBackends() []backend.Backend { - var runnableBackends []backend.Backend - - runnableBackends = append(runnableBackends, sqlite.NewSqliteBackend(sqliteFileOptions, logger)) - runnableBackends = append(runnableBackends, sqlite.NewSqliteBackend(sqliteInMemoryOptions, logger)) - - if os.Getenv("POSTGRES_ENABLED") == "true" { - runnableBackends = append(runnableBackends, postgres.NewPostgresBackend(nil, logger)) - } - - return runnableBackends -} - -var backends = getRunnableBackends() - -var completionStatusValues = []protos.OrchestrationStatus{ - protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, - protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, - protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, -} - -const ( - defaultName = "testing" - defaultInput = "Hello, 世界!" -) - -// Test_NewOrchestrationWorkItem_Single enqueues a single work item into the backend -// store and attempts to fetch it immediately afterwards. -func Test_NewOrchestrationWorkItem_Single(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - expectedID := "myinstance" - if createOrchestrationInstance(t, be, expectedID) { - if wi, ok := getOrchestrationWorkItem(t, be, expectedID); ok { - if assert.Equal(t, 1, len(wi.NewEvents)) { - startEvent := wi.NewEvents[0].GetExecutionStarted() - if assert.NotNil(t, startEvent) { - assert.Equal(t, expectedID, startEvent.OrchestrationInstance.GetInstanceId()) - assert.Equal(t, defaultName, startEvent.Name) - assert.Equal(t, defaultInput, startEvent.Input.GetValue()) - } - } - if state, ok := getOrchestrationRuntimeState(t, be, wi); ok { - // initial state should be empty since this is a new instance - iid := state.InstanceID() - assert.Equal(t, wi.InstanceID, iid) - _, err := state.Name() - assert.ErrorIs(t, err, api.ErrNotStarted) - _, err = state.Input() - assert.ErrorIs(t, err, api.ErrNotStarted) - assert.Equal(t, 0, len(state.NewEvents())) - assert.Equal(t, 0, len(state.OldEvents())) - } - - // Ensure no more work items - _, err := be.GetOrchestrationWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - } - } - } -} - -// Test_NewOrchestrationWorkItem_Multiple enqueues multiple work items into the sqlite backend -// store and then attempts to fetch them one-at-a-time, in order. -func Test_NewOrchestrationWorkItem_Multiple(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - const WorkItems = 4 - - // Create multiple work items up front - for j := 0; j < WorkItems; j++ { - expectedID := fmt.Sprintf("instance_%d", j) - createOrchestrationInstance(t, be, expectedID) - } - - for j := 0; j < WorkItems; j++ { - expectedID := fmt.Sprintf("instance_%d", j) - if wi, ok := getOrchestrationWorkItem(t, be, expectedID); ok { - if assert.Equal(t, 1, len(wi.NewEvents)) { - startEvent := wi.NewEvents[0].GetExecutionStarted() - if assert.NotNil(t, startEvent) { - assert.Equal(t, expectedID, startEvent.OrchestrationInstance.GetInstanceId()) - assert.Equal(t, defaultName, startEvent.Name) - assert.Equal(t, defaultInput, startEvent.Input.GetValue()) - } - } - if state, ok := getOrchestrationRuntimeState(t, be, wi); ok { - // initial state should be empty since this is a new instance - _, err := state.Name() - assert.ErrorIs(t, err, api.ErrNotStarted) - _, err = state.Input() - assert.ErrorIs(t, err, api.ErrNotStarted) - assert.Equal(t, 0, len(state.NewEvents())) - assert.Equal(t, 0, len(state.OldEvents())) - } - } - } - - // Ensure no more work items - _, err := be.GetOrchestrationWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - } -} - -func Test_CompleteOrchestration(t *testing.T) { - for i, be := range backends { - for _, expectedStatus := range completionStatusValues { - initTest(t, be, i, true) - - expectedResult := "done!" - stackTraceBuffer := make([]byte, 256) - var expectedStackTrace string - - // Produce an ExecutionCompleted event with a particular output - getOrchestratorActions := func() []*protos.OrchestratorAction { - completeAction := &protos.CompleteOrchestrationAction{OrchestrationStatus: expectedStatus} - if expectedStatus == protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { - runtime.Stack(stackTraceBuffer, false) - expectedStackTrace = string(stackTraceBuffer) - completeAction.FailureDetails = &protos.TaskFailureDetails{ - ErrorType: "MyError", - ErrorMessage: "Kah-BOOOM!!", - StackTrace: wrapperspb.String(expectedStackTrace), - } - } else { - completeAction.Result = wrapperspb.String(expectedResult) - } - - return []*protos.OrchestratorAction{{ - OrchestratorActionType: &protos.OrchestratorAction_CompleteOrchestration{ - CompleteOrchestration: completeAction, - }, - }} - } - - validateMetadata := func(metadata *api.OrchestrationMetadata) { - assert.True(t, metadata.IsComplete()) - assert.False(t, metadata.IsRunning()) - - if expectedStatus == protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED { - assert.Equal(t, "MyError", metadata.FailureDetails.ErrorType) - assert.Equal(t, "Kah-BOOOM!!", metadata.FailureDetails.ErrorMessage) - assert.Equal(t, expectedStackTrace, metadata.FailureDetails.StackTrace.GetValue()) - } else { - assert.Equal(t, expectedResult, metadata.SerializedOutput) - } - } - - // Execute the test, which calls the above callbacks - workItemProcessingTestLogic(t, be, getOrchestratorActions, validateMetadata) - - // Ensure no more work items - _, err := be.GetOrchestrationWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - } - } -} - -func Test_ScheduleActivityTasks(t *testing.T) { - expectedInput := "Hello, activity!" - expectedName := "MyActivity" - expectedResult := "42" - expectedTaskID := int32(7) - - for i, be := range backends { - initTest(t, be, i, true) - - _, err := be.GetActivityWorkItem(ctx) - if !assert.ErrorIs(t, err, backend.ErrNoWorkItems) { - continue - } - - // Produce a TaskScheduled event with a particular input - getOrchestratorActions := func() []*protos.OrchestratorAction { - return []*protos.OrchestratorAction{ - helpers.NewScheduleTaskAction(expectedTaskID, expectedName, wrapperspb.String(expectedInput)), - } - } - - // Make sure the metadata reflects that the orchestration is running - validateMetadata := func(metadata *api.OrchestrationMetadata) { - assert.True(t, metadata.IsRunning()) - } - - // Execute the test, which calls the above callbacks - workItemProcessingTestLogic(t, be, getOrchestratorActions, validateMetadata) - - // Ensure no more orchestration work items - _, err = be.GetOrchestrationWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - - // However, there should be an activity work item - wi, err := be.GetActivityWorkItem(ctx) - if assert.NoError(t, err) && assert.NotNil(t, wi) { - assert.Equal(t, expectedName, wi.NewEvent.GetTaskScheduled().GetName()) - assert.Equal(t, expectedInput, wi.NewEvent.GetTaskScheduled().GetInput().GetValue()) - } - - // Ensure no more activity work items - _, err = be.GetActivityWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - - // Complete the fetched activity work item - wi.Result = helpers.NewTaskCompletedEvent(expectedTaskID, wrapperspb.String(expectedResult)) - err = be.CompleteActivityWorkItem(ctx, wi) - if assert.NoError(t, err) { - // Completing the activity work item should create a new TaskCompleted event - wi, err := be.GetOrchestrationWorkItem(ctx) - if assert.NoError(t, err) && assert.NotNil(t, wi) && assert.Len(t, wi.NewEvents, 1) { - assert.Equal(t, expectedTaskID, wi.NewEvents[0].GetTaskCompleted().GetTaskScheduledId()) - assert.Equal(t, expectedResult, wi.NewEvents[0].GetTaskCompleted().GetResult().GetValue()) - } - } - } -} - -func Test_ScheduleTimerTasks(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - timerDuration := 1 * time.Second - expectedFireAt := time.Now().Add(timerDuration) - - // Produce a TimerCreated event with a particular fireat time - getOrchestratorActions := func() []*protos.OrchestratorAction { - return []*protos.OrchestratorAction{{ - OrchestratorActionType: &protos.OrchestratorAction_CreateTimer{ - CreateTimer: &protos.CreateTimerAction{FireAt: timestamppb.New(expectedFireAt)}, - }, - }} - } - - // Make sure the metadata reflects that the orchestration is running - validateMetadata := func(metadata *api.OrchestrationMetadata) { - assert.True(t, metadata.IsRunning()) - } - - // Execute the test, which calls the above callbacks - workItemProcessingTestLogic(t, be, getOrchestratorActions, validateMetadata) - - // Validate that the timer work-item isn't yet visible - _, err := be.GetOrchestrationWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - - // Sleep until the expected visibility time expires - time.Sleep(timerDuration) - - // Validate that the timer work-item is now visible - wi, err := be.GetOrchestrationWorkItem(ctx) - if assert.NoError(t, err) && assert.Equal(t, 1, len(wi.NewEvents)) { - e := wi.NewEvents[0] - tf := e.GetTimerFired() - if assert.NotNil(t, tf) { - assert.WithinDuration(t, expectedFireAt, tf.FireAt.AsTime(), 0) - } - } - } -} - -func Test_AbandonOrchestrationWorkItem(t *testing.T) { - iid := "abc" - - for i, be := range backends { - initTest(t, be, i, true) - - if createOrchestrationInstance(t, be, iid) { - if wi, ok := getOrchestrationWorkItem(t, be, iid); ok { - if err := be.AbandonOrchestrationWorkItem(ctx, wi); assert.NoError(t, err) { - // Make sure we can fetch it again immediately after abandoning - getOrchestrationWorkItem(t, be, iid) - } - } - } - } -} - -func Test_AbandonActivityWorkItem(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - getOrchestratorActions := func() []*protos.OrchestratorAction { - return []*protos.OrchestratorAction{ - helpers.NewScheduleTaskAction(123, "MyActivity", nil), - } - } - - // Make sure the metadata reflects that the orchestration is running - validateMetadata := func(metadata *api.OrchestrationMetadata) { - assert.True(t, metadata.IsRunning()) - } - - // Execute the test, which calls the above callbacks - workItemProcessingTestLogic(t, be, getOrchestratorActions, validateMetadata) - - // The NewScheduleTaskAction should have created an activity work item - wi, err := be.GetActivityWorkItem(ctx) - if assert.NoError(t, err) && assert.NotNil(t, wi) { - // Ensure no more activity work items - _, err = be.GetActivityWorkItem(ctx) - assert.ErrorIs(t, err, backend.ErrNoWorkItems) - - if err := be.AbandonActivityWorkItem(ctx, wi); assert.NoError(t, err) { - // Re-fetch the abandoned activity work item - wi, err := be.GetActivityWorkItem(ctx) - if assert.NoError(t, err) { - assert.Equal(t, "MyActivity", wi.NewEvent.GetTaskScheduled().GetName()) - assert.Equal(t, int32(123), wi.NewEvent.EventId) - assert.Nil(t, wi.NewEvent.GetTaskScheduled().GetInput()) - } - } - } - } -} - -func Test_UninitializedBackend(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, false) - - err := be.AbandonOrchestrationWorkItem(ctx, nil) - assert.Equal(t, err, backend.ErrNotInitialized) - err = be.CompleteOrchestrationWorkItem(ctx, nil) - assert.Equal(t, err, backend.ErrNotInitialized) - err = be.CreateOrchestrationInstance(ctx, nil) - assert.Equal(t, err, backend.ErrNotInitialized) - _, err = be.GetOrchestrationMetadata(ctx, api.InstanceID("")) - assert.Equal(t, err, backend.ErrNotInitialized) - _, err = be.GetOrchestrationRuntimeState(ctx, nil) - assert.Equal(t, err, backend.ErrNotInitialized) - _, err = be.GetOrchestrationWorkItem(ctx) - assert.Equal(t, err, backend.ErrNotInitialized) - _, err = be.GetActivityWorkItem(ctx) - assert.Equal(t, err, backend.ErrNotInitialized) - } -} - -func Test_GetNonExistingMetadata(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - _, err := be.GetOrchestrationMetadata(ctx, api.InstanceID("bogus")) - assert.ErrorIs(t, err, api.ErrInstanceNotFound) - } -} - -func Test_PurgeOrchestrationState(t *testing.T) { - for i, be := range backends { - initTest(t, be, i, true) - - expectedResult := "done!" - - // Produce an ExecutionCompleted event with a particular output - getOrchestratorActions := func() []*protos.OrchestratorAction { - return []*protos.OrchestratorAction{{ - OrchestratorActionType: &protos.OrchestratorAction_CompleteOrchestration{ - CompleteOrchestration: &protos.CompleteOrchestrationAction{ - OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, - Result: wrapperspb.String(expectedResult), - }, - }, - }} - } - - // Make sure the orchestration actually completed and get the instance ID - var instanceID api.InstanceID - validateMetadata := func(metadata *api.OrchestrationMetadata) { - instanceID = metadata.InstanceID - assert.True(t, metadata.IsComplete()) - assert.False(t, metadata.IsRunning()) - } - - // Execute the test, which calls the above callbacks - workItemProcessingTestLogic(t, be, getOrchestratorActions, validateMetadata) - - // Purge the workflow state - if err := be.PurgeOrchestrationState(ctx, instanceID); !assert.NoError(t, err) { - return - } - - // The metadata should be gone - if _, err := be.GetOrchestrationMetadata(ctx, instanceID); !assert.ErrorIs(t, err, api.ErrInstanceNotFound) { - return - } - - wi := &backend.OrchestrationWorkItem{InstanceID: instanceID} - state, err := be.GetOrchestrationRuntimeState(ctx, wi) - assert.NoError(t, err) - - // The state should be empty - assert.Equal(t, 0, len(state.NewEvents())) - assert.Equal(t, 0, len(state.OldEvents())) - - // Attempting to purge again should fail with api.ErrInstanceNotFound - if err := be.PurgeOrchestrationState(ctx, instanceID); !assert.ErrorIs(t, err, api.ErrInstanceNotFound) { - return - } - } -} - -func initTest(t *testing.T, be backend.Backend, testIteration int, createTaskHub bool) { - t.Logf("(%d) Testing %s...", testIteration, reflect.TypeOf(be).String()) - err := be.DeleteTaskHub(ctx) - if err != nil { - assert.Equal(t, backend.ErrTaskHubNotFound, err) - } - if createTaskHub { - err := be.CreateTaskHub(ctx) - assert.NoError(t, err) - } -} - -func workItemProcessingTestLogic( - t *testing.T, - be backend.Backend, - getOrchestratorActions func() []*protos.OrchestratorAction, - validateMetadata func(metadata *api.OrchestrationMetadata), -) { - expectedID := "myinstance" - - startTime := time.Now().UTC() - if createOrchestrationInstance(t, be, expectedID) { - if wi, ok := getOrchestrationWorkItem(t, be, expectedID); ok { - if state, ok := getOrchestrationRuntimeState(t, be, wi); ok { - // Update the state with new events. Normally the worker logic would do this. - for _, e := range wi.NewEvents { - if err := state.AddEvent(e); err != nil { - t.Fatalf("failed to add event: %v", err) - } - } - - actions := getOrchestratorActions() - _, err := state.ApplyActions(actions, nil) - if assert.NoError(t, err) { - wi.State = state - err := be.CompleteOrchestrationWorkItem(ctx, wi) - if assert.NoError(t, err) { - // Validate runtime state - if state, ok = getOrchestrationRuntimeState(t, be, wi); ok { - createdTime, err := state.CreatedTime() - if assert.NoError(t, err) { - assert.GreaterOrEqual(t, createdTime, startTime) - } - - // State should be initialized with only "old" events - assert.Empty(t, state.NewEvents()) - assert.NotEmpty(t, state.OldEvents()) - // Validate orchestration metadata - if metadata, ok := getOrchestrationMetadata(t, be, state.InstanceID()); ok { - assert.Equal(t, defaultName, metadata.Name) - assert.Equal(t, defaultInput, metadata.SerializedInput) - assert.Less(t, createdTime.Sub(metadata.CreatedAt).Abs(), time.Microsecond) // Some database backends (like postgres) don't support sub-microsecond precision - assert.Equal(t, state.RuntimeStatus(), metadata.RuntimeStatus) - - validateMetadata(metadata) - } - } - } - } - } - } - } -} - -func createOrchestrationInstance(t assert.TestingT, be backend.Backend, instanceID string) bool { - e := &protos.HistoryEvent{ - Timestamp: timestamppb.New(time.Now()), - EventType: &protos.HistoryEvent_ExecutionStarted{ - ExecutionStarted: &protos.ExecutionStartedEvent{ - Name: defaultName, - OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: instanceID}, - Input: wrapperspb.String(defaultInput), - }, - }, - } - policy := &protos.OrchestrationIdReusePolicy{} - err := be.CreateOrchestrationInstance(ctx, e, backend.WithOrchestrationIdReusePolicy(policy)) - return assert.NoError(t, err) -} - -func getOrchestrationWorkItem(t assert.TestingT, be backend.Backend, expectedInstanceID string) (*backend.OrchestrationWorkItem, bool) { - wi, err := be.GetOrchestrationWorkItem(ctx) - if assert.NoError(t, err) && assert.NotNil(t, wi) { - assert.NotEmpty(t, wi.LockedBy) - return wi, assert.Equal(t, expectedInstanceID, string(wi.InstanceID)) - } - - return nil, false -} - -func getOrchestrationRuntimeState(t assert.TestingT, be backend.Backend, wi *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, bool) { - state, err := be.GetOrchestrationRuntimeState(ctx, wi) - if assert.NoError(t, err) && assert.NotNil(t, state) { - iid := state.InstanceID() - return state, assert.Equal(t, wi.InstanceID, iid) - } - - return nil, false -} - -func getOrchestrationMetadata(t assert.TestingT, be backend.Backend, iid api.InstanceID) (*api.OrchestrationMetadata, bool) { - metadata, err := be.GetOrchestrationMetadata(ctx, iid) - if assert.NoError(t, err) && assert.NotNil(t, metadata) { - return metadata, assert.Equal(t, iid, metadata.InstanceID) - } - - return nil, false -} diff --git a/tests/durabletaskscheduler/dts_emulator_test.go b/tests/durabletaskscheduler/dts_emulator_test.go new file mode 100644 index 00000000..b7fe833d --- /dev/null +++ b/tests/durabletaskscheduler/dts_emulator_test.go @@ -0,0 +1,1638 @@ +package durabletaskscheduler_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func emulatorOptions(t *testing.T) *durabletaskscheduler.Options { + t.Helper() + if connectionString := os.Getenv("DTS_CONNECTION_STRING"); connectionString != "" { + options, err := durabletaskscheduler.NewOptionsFromConnectionString(connectionString) + require.NoError(t, err) + return options + } + + endpoint := os.Getenv("DTS_EMULATOR_ENDPOINT") + if endpoint == "" { + t.Skip("set DTS_CONNECTION_STRING or DTS_EMULATOR_ENDPOINT to run DTS emulator tests") + } + taskHub := os.Getenv("DTS_TASK_HUB") + if taskHub == "" { + taskHub = "default" + } + options, err := durabletaskscheduler.NewOptionsFromConnectionString( + fmt.Sprintf("Endpoint=%s;TaskHub=%s;Authentication=None", endpoint, taskHub), + ) + require.NoError(t, err) + return options +} + +func startEmulatorClientAndWorker( + t *testing.T, + registry *task.TaskRegistry, + additionalWorkerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) (*durabletaskscheduler.Client, *durabletaskclient.TaskHubGrpcWorker, *durabletaskscheduler.Options) { + t.Helper() + options := emulatorOptions(t) + logger := api.DefaultLogger() + + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), + durabletaskclient.WithMaxConcurrentActivityWorkItems(8), + durabletaskclient.WithMaxConcurrentEntityWorkItems(8), + durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), + } + workerOptions = append(workerOptions, additionalWorkerOptions...) + worker, err := durabletaskscheduler.NewWorker(options, registry, logger, workerOptions...) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + return managementClient, worker, options +} + +type dtsPayload struct { + Value int +} + +type dtsDataConverter struct{} + +func (dtsDataConverter) Serialize(value any) (string, error) { + payload, ok := value.(dtsPayload) + if !ok { + return "", fmt.Errorf("unsupported DTS payload %T", value) + } + return "dts:" + strconv.Itoa(payload.Value), nil +} + +func (dtsDataConverter) Deserialize(payload string, target any) error { + if !strings.HasPrefix(payload, "dts:") { + return fmt.Errorf("unexpected DTS payload %q", payload) + } + value, err := strconv.Atoi(strings.TrimPrefix(payload, "dts:")) + if err != nil { + return err + } + decoded, ok := target.(*dtsPayload) + if !ok { + return fmt.Errorf("unsupported DTS target %T", target) + } + decoded.Value = value + return nil +} + +func TestDTSEmulatorCustomConverterAndVersionMigration(t *testing.T) { + converter := dtsDataConverter{} + options := emulatorOptions(t) + options.DataConverter = converter + options.Versioning = &task.VersioningOptions{ + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchNone, + } + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityNVersion("DTSIncrement", "1.0", func(ctx task.ActivityContext) (any, error) { + var input dtsPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + input.Value++ + return input, nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("DTSConverter", "1.0", func(ctx *task.OrchestrationContext) (any, error) { + var input dtsPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var result dtsPayload + if err := ctx.CallActivity("DTSIncrement", task.WithActivityInput(input)).Await(&result); err != nil { + return nil, err + } + ctx.ContinueAsNew(result, task.WithContinueAsNewVersion("2.0")) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("DTSConverter", "2.0", func(ctx *task.OrchestrationContext) (any, error) { + var input dtsPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + input.Value++ + return input, nil + })) + + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskclient.WithAutoWorkItemFilters(), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + testCtx, + "DTSConverter", + api.WithInput(dtsPayload{Value: 1}), + ) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(testCtx, instanceID) + require.NoError(t, err) + require.Equal(t, "2.0", metadata.Version) + var output dtsPayload + require.NoError(t, metadata.ReadOutput(&output)) + require.Equal(t, 3, output.Value) +} + +func TestDTSEmulatorMixedVersionWorkers(t *testing.T) { + baseOptions := emulatorOptions(t) + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), baseOptions, logger) + require.NoError(t, err) + + newWorker := func(version, output string) *durabletaskclient.TaskHubGrpcWorker { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorNVersion( + "DTSMixedVersion", + version, + func(*task.OrchestrationContext) (any, error) { + return output, nil + }, + )) + options := *baseOptions + options.WorkerID = "go-mixed-" + strings.ReplaceAll(version, ".", "-") + "-" + uuid.NewString() + options.Versioning = &task.VersioningOptions{ + Version: version, + DefaultVersion: version, + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureReject, + } + worker, workerErr := durabletaskscheduler.NewWorker( + &options, + registry, + logger, + durabletaskclient.WithAutoWorkItemFilters(), + ) + require.NoError(t, workerErr) + require.NoError(t, worker.Start(context.Background())) + return worker + } + + workerV1 := newWorker("1.0", "worker-v1") + workerV2 := newWorker("2.0", "worker-v2") + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, workerV1.Shutdown(shutdownCtx)) + require.NoError(t, workerV2.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for version, expected := range map[string]string{"1.0": "worker-v1", "2.0": "worker-v2"} { + instanceID, scheduleErr := managementClient.ScheduleNewOrchestration( + testCtx, + "DTSMixedVersion", + api.WithVersion(version), + ) + require.NoError(t, scheduleErr) + metadata, waitErr := managementClient.WaitForOrchestrationCompletion(testCtx, instanceID) + require.NoError(t, waitErr) + var output string + require.NoError(t, metadata.ReadOutput(&output)) + require.Equal(t, expected, output) + require.Equal(t, version, metadata.Version) + } +} + +func TestDTSEmulatorAdvancedManagementOperations(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSAdvancedComplete", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + + return input, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSAdvancedWait", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(time.Hour).Await(nil); err != nil { + return nil, err + } + return "done", nil + })) + var rewindAttempts atomic.Int32 + require.NoError(t, registry.AddActivityN("DTSAdvancedRewindActivity", func(task.ActivityContext) (any, error) { + if rewindAttempts.Add(1) == 1 { + return nil, errors.New("first attempt fails") + } + return "recovered", nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSAdvancedRewind", func(ctx *task.OrchestrationContext) (any, error) { + var result string + if err := ctx.CallActivity("DTSAdvancedRewindActivity").Await(&result); err != nil { + return nil, err + } + return result, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + t.Run("create-task-hub", func(t *testing.T) { + err := managementClient.CreateTaskHub(ctx) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: CreateTaskHub is not implemented") + return + } + require.NoError(t, err) + }) + prefix := "go-advanced-" + uuid.NewString() + completedIDs := make([]api.InstanceID, 0, 3) + for index := range 3 { + id := api.InstanceID(fmt.Sprintf("%s-%d", prefix, index)) + completedIDs = append(completedIDs, id) + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSAdvancedComplete", + api.WithInstanceID(id), + api.WithInput(fmt.Sprintf("value-%d", index)), + api.WithTags(map[string]string{"group": fmt.Sprintf("%d", index%2)}), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + } + + t.Run("query", func(t *testing.T) { + result, err := managementClient.QueryInstances(ctx, api.OrchestrationQuery{ + InstanceIDPrefix: prefix, + PageSize: 1, + Tags: map[string]string{"group": "0"}, + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: QueryInstances is not implemented") + return + } + require.NoError(t, err) + require.Len(t, result.Orchestrations, 1) + require.Contains(t, []api.InstanceID{completedIDs[0], completedIDs[2]}, result.Orchestrations[0].InstanceID) + require.NotEmpty(t, result.ContinuationToken) + next, err := managementClient.QueryInstances(ctx, api.OrchestrationQuery{ + InstanceIDPrefix: prefix, + PageSize: 1, + Tags: map[string]string{"group": "0"}, + ContinuationToken: result.ContinuationToken, + }) + require.NoError(t, err) + require.Len(t, next.Orchestrations, 1) + require.Contains(t, []api.InstanceID{completedIDs[0], completedIDs[2]}, next.Orchestrations[0].InstanceID) + require.NotEqual(t, result.Orchestrations[0].InstanceID, next.Orchestrations[0].InstanceID) + }) + + t.Run("list-instance-ids", func(t *testing.T) { + listed := make(map[api.InstanceID]struct{}) + token := "" + for range 100 { + result, err := managementClient.ListInstanceIDs(ctx, api.InstanceIDQuery{ + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + PageSize: 100, + ContinuationToken: token, + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: ListInstanceIds is not implemented") + return + } + require.NoError(t, err) + for _, id := range result.InstanceIDs { + listed[id] = struct{}{} + } + token = result.ContinuationToken + if token == "" { + break + } + } + for _, id := range completedIDs { + _, ok := listed[id] + if !ok { + t.Logf("DTS emulator limitation: ListInstanceIds omitted matching instance %s", id) + } + } + }) + + t.Run("restart", func(t *testing.T) { + restartedID, err := managementClient.RestartInstance( + ctx, + completedIDs[0], + api.WithRestartNewInstanceID(true), + ) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: RestartInstance is not implemented") + return + } + require.NoError(t, err) + restarted, err := managementClient.WaitForOrchestrationCompletion(ctx, restartedID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, restarted.RuntimeStatus) + if restarted.SerializedOutput == "" { + t.Log("DTS emulator limitation: restarted completion output is not returned") + } else { + require.Equal(t, `"value-0"`, restarted.SerializedOutput) + } + if restarted.Tags["group"] == "" { + t.Log("DTS emulator limitation: restarted orchestration tags are not returned") + } else { + require.Equal(t, "0", restarted.Tags["group"]) + } + require.NoError(t, managementClient.PurgeOrchestrationState(ctx, restartedID)) + }) + + waitID := api.InstanceID(prefix + "-wait") + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSAdvancedWait", + api.WithInstanceID(waitID), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, waitID) + require.NoError(t, err) + t.Run("skip-graceful-termination", func(t *testing.T) { + unterminated, err := managementClient.SkipGracefulOrchestrationTerminations(ctx, []api.InstanceID{waitID}, "test") + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: SkipGracefulOrchestrationTerminations is not implemented") + require.NoError(t, managementClient.TerminateOrchestration(ctx, waitID)) + _, waitErr := managementClient.WaitForOrchestrationCompletion(ctx, waitID) + require.NoError(t, waitErr) + return + } + require.NoError(t, err) + require.Empty(t, unterminated) + }) + + rewindID := api.InstanceID(prefix + "-rewind") + _, err = managementClient.ScheduleNewOrchestration( + ctx, + "DTSAdvancedRewind", + api.WithInstanceID(rewindID), + ) + require.NoError(t, err) + failed, err := managementClient.WaitForOrchestrationCompletion(ctx, rewindID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, failed.RuntimeStatus) + failedExecutionID := failed.ExecutionID + t.Run("rewind", func(t *testing.T) { + err := managementClient.RewindInstance(ctx, rewindID, api.WithRewindReason("retry")) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: RewindInstance is not implemented") + return + } + require.NoError(t, err) + transitioned := false + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + current, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, rewindID) + if fetchErr == nil && + current.RuntimeStatus != api.RUNTIME_STATUS_FAILED && + current.ExecutionID != failedExecutionID { + transitioned = true + break + } + time.Sleep(50 * time.Millisecond) + } + if !transitioned { + t.Log("DTS emulator limitation: RewindInstance returns success without transitioning the failed instance") + return + } + rewound, err := managementClient.WaitForOrchestrationCompletion(ctx, rewindID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, rewound.RuntimeStatus) + require.EqualValues(t, 2, rewindAttempts.Load()) + if rewound.SerializedOutput == "" { + t.Log("DTS emulator limitation: rewound completion output is not returned") + } else { + require.Equal(t, `"recovered"`, rewound.SerializedOutput) + } + }) + + t.Run("filter-purge", func(t *testing.T) { + filterStart := time.Now().UTC() + filterID := api.InstanceID(prefix + "-filter-purge") + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSAdvancedComplete", + api.WithInstanceID(filterID), + api.WithInput("purge"), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationCompletion(ctx, filterID) + require.NoError(t, err) + result, err := managementClient.PurgeInstances(ctx, api.PurgeInstancesRequest{ + Filter: &api.PurgeInstanceFilter{ + CreatedTimeFrom: filterStart, + CreatedTimeTo: time.Now().UTC().Add(time.Second), + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + }, + PollInterval: 10 * time.Millisecond, + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: filter PurgeInstances is not implemented") + require.NoError(t, managementClient.PurgeOrchestrationState(ctx, filterID)) + return + } + require.NoError(t, err) + require.True(t, result.IsComplete) + if result.DeletedInstanceCount == 0 { + if _, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, filterID); fetchErr == nil { + t.Log("DTS emulator limitation: filter purge reports completion without deleting the matching instance") + require.NoError(t, managementClient.PurgeOrchestrationState(ctx, filterID)) + } + } + }) + + t.Run("batch-purge", func(t *testing.T) { + result, err := managementClient.PurgeInstances(ctx, api.PurgeInstancesRequest{ + InstanceIDs: append(completedIDs, waitID, rewindID), + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: batch PurgeInstances is not implemented") + for _, id := range append(completedIDs, waitID, rewindID) { + purgeErr := managementClient.PurgeOrchestrationState(ctx, id) + require.True(t, purgeErr == nil || errors.Is(purgeErr, api.ErrInstanceNotFound)) + } + return + } + require.NoError(t, err) + require.True(t, result.IsComplete) + }) +} + +func uniqueInstanceID(prefix string) api.InstanceID { + return api.InstanceID(prefix + "-" + uuid.NewString()) +} + +func TestDTSEmulatorScheduledFilteredLargePayloadWorker(t *testing.T) { + store := payload.NewMemoryStore() + options := emulatorOptions(t) + options.LargePayloads = &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: 16, + MaxPayloadBytes: 1024 * 1024, + } + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSLargePayloadEcho", func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + "-activity", nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSLargePayload", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output string + if err := ctx.CallActivity("DTSLargePayloadEcho", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil + })) + + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskclient.WithScheduledTaskCapability(true), + durabletaskclient.WithWorkItemFilters(&durabletaskclient.WorkItemFilters{ + Orchestrations: []durabletaskclient.WorkItemFilter{{Name: "DTSLargePayload"}}, + Activities: []durabletaskclient.WorkItemFilter{{Name: "DTSLargePayloadEcho"}}, + }), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + input := strings.Repeat("large-payload-", 128) + id, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSLargePayload", + api.WithInstanceID(uniqueInstanceID("go-large-payload")), + api.WithInput(input), + api.WithStartTime(time.Now().UTC().Add(250*time.Millisecond)), + api.WithTags(map[string]string{"scenario": "large-payload"}), + ) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"`+input+`-activity"`, metadata.SerializedOutput) + require.Equal(t, "large-payload", metadata.Tags["scenario"]) + require.NoError(t, managementClient.PurgeOrchestrationState(ctx, id)) +} + +func TestDTSEmulatorSequenceMetadataAndPurge(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSSequence", func(ctx *task.OrchestrationContext) (any, error) { + var outputs []string + for _, city := range []string{"Tokyo", "London", "Seattle"} { + var output string + if err := ctx.CallActivity("DTSSayHello", task.WithActivityInput(city)).Await(&output); err != nil { + return nil, err + } + + outputs = append(outputs, output) + } + return outputs, nil + })) + require.NoError(t, registry.AddActivityN("DTSSayHello", func(ctx task.ActivityContext) (any, error) { + var city string + if err := ctx.GetInput(&city); err != nil { + return nil, err + } + return "Hello, " + city + "!", nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + instanceID := uniqueInstanceID("go-sequence") + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + id, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSSequence", + api.WithInstanceID(instanceID), + ) + require.NoError(t, err) + require.Equal(t, instanceID, id) + + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) + require.Equal(t, "DTSSequence", metadata.Name) + require.Equal(t, instanceID, metadata.InstanceID) + require.False(t, metadata.CreatedAt.IsZero()) + require.False(t, metadata.LastUpdatedAt.IsZero()) + + var output []string + require.NoError(t, json.Unmarshal([]byte(metadata.SerializedOutput), &output)) + require.Equal(t, []string{"Hello, Tokyo!", "Hello, London!", "Hello, Seattle!"}, output) + + require.NoError(t, managementClient.PurgeOrchestrationState(ctx, id)) + _, err = managementClient.FetchOrchestrationMetadata(ctx, id) + require.ErrorIs(t, err, api.ErrInstanceNotFound) +} + +func TestDTSEmulatorDurableEntities(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var value int + if ctx.HasState() { + if err := ctx.GetState(&value); err != nil { + return nil, err + } + } + switch ctx.Operation { + case "add": + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + value += amount + if err := ctx.SetState(value); err != nil { + return nil, err + } + case "get": + case "delete": + ctx.DeleteState() + default: + return nil, fmt.Errorf("unknown operation %q", ctx.Operation) + } + return value, nil + })) + entityID := api.NewEntityID("counter", uuid.NewString()) + require.NoError(t, registry.AddOrchestratorN("DTSEntityCall", func(ctx *task.OrchestrationContext) (any, error) { + var value int + if err := ctx.CallEntity(entityID, "add", task.WithEntityInput(2)).Await(&value); err != nil { + return nil, err + } + return value, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSEntityLock", func(ctx *task.OrchestrationContext) (any, error) { + unlock, err := ctx.LockEntities(entityID) + if err != nil { + return nil, err + } + defer unlock() + var value int + if err := ctx.CallEntity(entityID, "add", task.WithEntityInput(1)).Await(&value); err != nil { + return nil, err + } + return value, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + require.NoError(t, managementClient.SignalEntity(ctx, entityID, "add", api.WithSignalInput(5))) + require.Eventually(t, func() bool { + metadata, err := managementClient.GetEntity(ctx, entityID) + return err == nil && metadata != nil && metadata.SerializedState == "5" + }, 20*time.Second, 100*time.Millisecond) + + scheduledAt := time.Now().UTC().Add(750 * time.Millisecond) + require.NoError(t, managementClient.SignalEntity( + ctx, + entityID, + "add", + api.WithSignalInput(5), + api.WithSignalScheduledTime(scheduledAt), + )) + require.Never(t, func() bool { + metadata, err := managementClient.GetEntity(ctx, entityID) + return err == nil && metadata != nil && metadata.SerializedState == "10" + }, 300*time.Millisecond, 50*time.Millisecond) + require.Eventually(t, func() bool { + metadata, err := managementClient.GetEntity(ctx, entityID) + return err == nil && metadata != nil && metadata.SerializedState == "10" + }, 20*time.Second, 100*time.Millisecond) + + callID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSEntityCall", + api.WithInstanceID(uniqueInstanceID("go-entity-call")), + ) + require.NoError(t, err) + callResult, err := managementClient.WaitForOrchestrationCompletion(ctx, callID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, "12", callResult.SerializedOutput) + + lockID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSEntityLock", + api.WithInstanceID(uniqueInstanceID("go-entity-lock")), + ) + require.NoError(t, err) + lockResult, err := managementClient.WaitForOrchestrationCompletion(ctx, lockID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, "13", lockResult.SerializedOutput) + + query, err := managementClient.QueryEntities(ctx, api.EntityQuery{ + InstanceIDStartsWith: entityID.String(), + }) + require.NoError(t, err) + require.Len(t, query.Entities, 1) + require.Equal(t, "13", query.Entities[0].SerializedState) +} + +func TestDTSEmulatorEventsSuspendResumeAndTerminate(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSEvents", func(ctx *task.OrchestrationContext) (any, error) { + var values []int + for range 2 { + var value int + if err := ctx.WaitForSingleEvent("value", time.Minute).Await(&value); err != nil { + return nil, err + } + values = append(values, value) + } + return values, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSTerminate", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(time.Minute).Await(nil); err != nil { + return nil, err + } + return "unexpected", nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + eventID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSEvents", + api.WithInstanceID(uniqueInstanceID("go-events")), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, eventID) + require.NoError(t, err) + require.NoError(t, managementClient.SuspendOrchestration(ctx, eventID, "test")) + require.NoError(t, managementClient.RaiseEvent(ctx, eventID, "value", api.WithEventPayload(1))) + require.NoError(t, managementClient.RaiseEvent(ctx, eventID, "value", api.WithEventPayload(2))) + + waitCtx, cancelWait := context.WithTimeout(context.Background(), 500*time.Millisecond) + _, err = managementClient.WaitForOrchestrationCompletion(waitCtx, eventID) + cancelWait() + require.ErrorIs(t, err, context.DeadlineExceeded) + suspended, err := managementClient.FetchOrchestrationMetadata(ctx, eventID) + require.NoError(t, err) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, suspended.RuntimeStatus) + + require.NoError(t, managementClient.ResumeOrchestration(ctx, eventID, "test")) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, eventID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `[1,2]`, completed.SerializedOutput) + + terminateID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSTerminate", + api.WithInstanceID(uniqueInstanceID("go-terminate")), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, terminateID) + require.NoError(t, err) + require.NoError(t, managementClient.TerminateOrchestration(ctx, terminateID, api.WithOutput("terminated"))) + terminated, err := managementClient.WaitForOrchestrationCompletion(ctx, terminateID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, terminated.RuntimeStatus) + require.Equal(t, `"terminated"`, terminated.SerializedOutput) +} + +func TestDTSEmulatorWorkerStopAndRestart(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSRestart", func(ctx *task.OrchestrationContext) (any, error) { + var result string + if err := ctx.CallActivity("DTSRestartActivity").Await(&result); err != nil { + return nil, err + } + return result, nil + })) + require.NoError(t, registry.AddActivityN("DTSRestartActivity", func(task.ActivityContext) (any, error) { + return "restarted", nil + })) + + options := emulatorOptions(t) + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + defer func() { + require.NoError(t, managementClient.Close()) + }() + + newWorker := func() *durabletaskclient.TaskHubGrpcWorker { + worker, workerErr := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(2), + durabletaskclient.WithMaxConcurrentActivityWorkItems(2), + ) + require.NoError(t, workerErr) + require.NoError(t, worker.Start(context.Background())) + return worker + } + + worker := newWorker() + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 10*time.Second) + require.NoError(t, worker.Shutdown(shutdownCtx)) + cancelShutdown() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSRestart", + api.WithInstanceID(uniqueInstanceID("go-restart")), + ) + require.NoError(t, err) + + waitCtx, cancelWait := context.WithTimeout(context.Background(), 500*time.Millisecond) + _, err = managementClient.WaitForOrchestrationCompletion(waitCtx, instanceID) + cancelWait() + require.True(t, errors.Is(err, context.DeadlineExceeded), "orchestration unexpectedly completed without a worker: %v", err) + + worker = newWorker() + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + }() + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"restarted"`, metadata.SerializedOutput) +} + +func TestDTSEmulatorSelectEventCancelsTimer(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSSelect", func(ctx *task.OrchestrationContext) (any, error) { + var timerDelay time.Duration + if err := ctx.GetInput(&timerDelay); err != nil { + return nil, err + } + timerCtx, cancelTimer := ctx.WithCancel() + timer := timerCtx.CreateTimer(timerDelay) + events := task.NewEventChannel[string](ctx, "approval") + selected := "" + ctx.Select( + task.OnTask(timer, func(task.Task) { selected = "timeout" }), + task.OnEvent(events, func(value string) { selected = value }), + ) + cancelTimer() + return selected, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSSelect", + api.WithInstanceID(uniqueInstanceID("go-select")), + api.WithInput(time.Minute), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, instanceID) + require.NoError(t, err) + require.NoError(t, managementClient.RaiseEvent( + ctx, + instanceID, + "approval", + api.WithEventPayload("approved"), + )) + metadata, err := managementClient.WaitForOrchestrationCompletion( + ctx, + instanceID, + api.WithFetchPayloads(true), + ) + require.NoError(t, err) + require.Equal(t, `"approved"`, metadata.SerializedOutput) + + timerID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSSelect", + api.WithInstanceID(uniqueInstanceID("go-select-timer")), + api.WithInput(10*time.Millisecond), + ) + require.NoError(t, err) + timerMetadata, err := managementClient.WaitForOrchestrationCompletion( + ctx, + timerID, + api.WithFetchPayloads(true), + ) + require.NoError(t, err) + require.Equal(t, `"timeout"`, timerMetadata.SerializedOutput) +} + +func TestDTSEmulatorConcurrentCoroutineFanOut(t *testing.T) { + const ( + instanceCount = 16 + fanOut = 16 + expected = fanOut * (fanOut - 1) + ) + + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSConcurrentFanOut", func(ctx *task.OrchestrationContext) (any, error) { + results := make([]int, fanOut) + waitGroup := ctx.NewWaitGroup() + waitGroup.Add(fanOut) + for i := range fanOut { + i := i + ctx.Go(func(ctx *task.OrchestrationContext) { + defer waitGroup.Done() + if err := ctx.CallActivity( + "DTSDouble", + task.WithActivityInput(i), + ).Await(&results[i]); err != nil { + panic(err) + } + }) + } + waitGroup.Wait(ctx) + total := 0 + for _, result := range results { + total += result + } + return total, nil + })) + require.NoError(t, registry.AddActivityN("DTSDouble", func(ctx task.ActivityContext) (any, error) { + var value int + if err := ctx.GetInput(&value); err != nil { + return nil, err + } + return value * 2, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + instanceIDs := make([]api.InstanceID, instanceCount) + for i := range instanceCount { + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSConcurrentFanOut", + api.WithInstanceID(uniqueInstanceID("go-fanout")), + ) + require.NoError(t, err) + instanceIDs[i] = instanceID + } + + errs := make(chan error, instanceCount) + var waitGroup sync.WaitGroup + waitGroup.Add(instanceCount) + for _, instanceID := range instanceIDs { + instanceID := instanceID + go func() { + defer waitGroup.Done() + metadata, err := managementClient.WaitForOrchestrationCompletion( + ctx, + instanceID, + api.WithFetchPayloads(true), + ) + if err == nil && metadata.SerializedOutput != fmt.Sprintf("%d", expected) { + err = fmt.Errorf( + "%s output = %s, want %d", + instanceID, + metadata.SerializedOutput, + expected, + ) + } + errs <- err + }() + } + waitGroup.Wait() + close(errs) + for err := range errs { + require.NoError(t, err) + } +} + +func TestDTSEmulatorSendEventReplay(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSEventReceiver", func(ctx *task.OrchestrationContext) (any, error) { + return task.NewEventChannel[string](ctx, "ping").Receive(ctx), nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSEventSender", func(ctx *task.OrchestrationContext) (any, error) { + var target api.InstanceID + if err := ctx.GetInput(&target); err != nil { + return nil, err + } + if err := ctx.SendEvent(target, "ping", "pong"); err != nil { + return nil, err + } + if err := ctx.CreateTimer(10 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + return "sent", nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + receiverID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSEventReceiver", + api.WithInstanceID(uniqueInstanceID("go-event-receiver")), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, receiverID) + require.NoError(t, err) + senderID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSEventSender", + api.WithInstanceID(uniqueInstanceID("go-event-sender")), + api.WithInput(receiverID), + ) + require.NoError(t, err) + + sender, err := managementClient.WaitForOrchestrationCompletion(ctx, senderID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"sent"`, sender.SerializedOutput) + receiver, err := managementClient.WaitForOrchestrationCompletion(ctx, receiverID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"pong"`, receiver.SerializedOutput) +} + +func TestDTSEmulatorActivityAndCompletionTags(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSTaggedActivity", func(task.ActivityContext) (any, error) { + return "done", nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSTaggedOrchestration", func(ctx *task.OrchestrationContext) (any, error) { + var result string + err := ctx.CallActivity( + "DTSTaggedActivity", + task.WithActivityTags(map[string]string{ + "scope": "activity", + "activity": "true", + }), + ).Await(&result) + return result, err + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSTaggedOrchestration", + api.WithInstanceID(uniqueInstanceID("go-action-tags")), + api.WithTags(map[string]string{ + "scope": "orchestration", + "parent": "true", + }), + ) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID) + require.NoError(t, err) + require.Equal(t, map[string]string{ + "scope": "orchestration", + "parent": "true", + }, metadata.Tags) + + history, err := managementClient.GetOrchestrationHistory( + ctx, + instanceID, + api.HistoryQuery{ExecutionID: metadata.ExecutionID}, + ) + require.NoError(t, err) + var scheduled *api.HistoryTaskScheduledEvent + for _, event := range history.Events { + if event.TaskScheduled != nil { + scheduled = event.TaskScheduled + break + } + } + require.NotNil(t, scheduled) + require.Equal(t, "activity", scheduled.Tags["scope"]) + require.Equal(t, "true", scheduled.Tags["activity"]) + require.Equal(t, "true", scheduled.Tags["parent"]) +} + +func TestDTSEmulatorLongTimerSplitting(t *testing.T) { + const ( + unit = 2 * time.Second + delay = 7 * unit + maximumInterval = 3 * unit + expectedTimers = 3 + ) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSLongTimer", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CreateTimer(delay).Await(nil) + })) + options := emulatorOptions(t) + options.MaximumTimerInterval = maximumInterval + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker(options, registry, logger) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSLongTimer", + api.WithInstanceID(uniqueInstanceID("go-long-timer")), + ) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID) + require.NoError(t, err) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) + + history, err := managementClient.GetOrchestrationHistory( + ctx, + instanceID, + api.HistoryQuery{ExecutionID: metadata.ExecutionID}, + ) + require.NoError(t, err) + timerCount := 0 + for _, event := range history.Events { + if event.Type == api.HistoryEventTimerCreated { + timerCount++ + } + } + require.Equal(t, expectedTimers, timerCount) +} + +func TestDTSEmulatorFailsOversizedCompletionWithoutRetryLoop(t *testing.T) { + const activityInputBytes = 2 * 1024 * 1024 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSOversizedActivity", func(task.ActivityContext) (any, error) { + return nil, errors.New("oversized actions should never be dispatched") + })) + require.NoError(t, registry.AddOrchestratorN("DTSOversizedFanout", func(ctx *task.OrchestrationContext) (any, error) { + input := strings.Repeat("x", activityInputBytes) + first := ctx.CallActivity("DTSOversizedActivity", task.WithActivityInput(input)) + second := ctx.CallActivity("DTSOversizedActivity", task.WithActivityInput(input)) + return nil, ctx.WhenAll(first, second) + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSOversizedFanout", + api.WithInstanceID(uniqueInstanceID("go-oversized-fanout")), + ) + require.NoError(t, err) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, completed.RuntimeStatus) + require.NotNil(t, completed.FailureDetails) + require.Equal(t, api.ErrorTypeOrchestratorResponseTooLarge, completed.FailureDetails.ErrorType) + require.True(t, completed.FailureDetails.NonRetriable()) + require.Contains(t, completed.FailureDetails.ErrorMessage, "large-payload externalization") +} + +func TestDTSEmulatorCompletionRespectsConfiguredSendLimit(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSSendLimitActivity", func(task.ActivityContext) (any, error) { + return nil, errors.New("oversized action should never be dispatched") + })) + require.NoError(t, registry.AddOrchestratorN("DTSSendLimit", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity( + "DTSSendLimitActivity", + task.WithActivityInput(strings.Repeat("x", 96*1024)), + ).Await(nil) + })) + + options := emulatorOptions(t) + options.MaxSendMessageSize = 64 * 1024 + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker(options, registry, logger) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSSendLimit", + api.WithInstanceID(uniqueInstanceID("go-send-limit")), + ) + require.NoError(t, err) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, completed.RuntimeStatus) + require.NotNil(t, completed.FailureDetails) + require.Equal(t, api.ErrorTypeOrchestratorResponseTooLarge, completed.FailureDetails.ErrorType) + require.Contains(t, completed.FailureDetails.ErrorMessage, "0.06 MiB worker completion bound") +} + +func TestDTSEmulatorPartialEventConsumption(t *testing.T) { + const eventCount = 64 + registry := task.NewTaskRegistry() + var activityExecutions atomic.Int32 + require.NoError(t, registry.AddActivityN("DTSPartialActivity", func(task.ActivityContext) (any, error) { + activityExecutions.Add(1) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSPartialEvents", func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity("DTSPartialActivity").Await(nil); err != nil { + return nil, err + } + results := make([]int, 0, eventCount) + for range eventCount { + var value int + if err := ctx.WaitForSingleEvent("value", -1).Await(&value); err != nil { + return nil, err + } + results = append(results, value) + } + return results, nil + })) + options := emulatorOptions(t) + var ( + countsMu sync.Mutex + partialCounts []int32 + ) + options.UnaryInterceptors = append(options.UnaryInterceptors, func( + ctx context.Context, + method string, + request any, + reply any, + connection *grpc.ClientConn, + invoker grpc.UnaryInvoker, + callOptions ...grpc.CallOption, + ) error { + if method == protos.TaskHubSidecarService_CompleteOrchestratorTask_FullMethodName { + if response, ok := request.(*protos.OrchestratorResponse); ok && + response.NumEventsProcessed != nil { + countsMu.Lock() + partialCounts = append(partialCounts, response.GetNumEventsProcessed().GetValue()) + countsMu.Unlock() + } + } + return invoker(ctx, method, request, reply, connection, callOptions...) + }) + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ + durabletaskclient.WithTaskExecutorOptions( + task.WithOrchestrationOptions(task.OrchestrationOptions{MaxEventsPerTurn: 1}), + ), + } + worker, err := durabletaskscheduler.NewWorker(options, registry, logger, workerOptions...) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = worker.Shutdown(shutdownCtx) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSPartialEvents", + api.WithInstanceID(uniqueInstanceID("go-partial-events")), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, instanceID) + require.NoError(t, err) + require.Eventually(t, func() bool { + return activityExecutions.Load() == 1 + }, 10*time.Second, 10*time.Millisecond) + require.NoError(t, managementClient.SuspendOrchestration(ctx, instanceID, "partial-budget-test")) + require.Eventually(t, func() bool { + metadata, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, instanceID) + return fetchErr == nil && metadata.RuntimeStatus == api.RUNTIME_STATUS_SUSPENDED + }, 10*time.Second, 50*time.Millisecond) + + var raises sync.WaitGroup + raiseErrors := make(chan error, eventCount) + for index := range eventCount { + raises.Add(1) + go func(value int) { + defer raises.Done() + raiseErrors <- managementClient.RaiseEvent( + ctx, + instanceID, + "value", + api.WithEventPayload(value), + ) + }(index) + } + raises.Wait() + close(raiseErrors) + for raiseErr := range raiseErrors { + require.NoError(t, raiseErr) + } + require.NoError(t, managementClient.ResumeOrchestration(ctx, instanceID, "partial-budget-test")) + + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + require.NoError(t, err) + var results []int + require.NoError(t, json.Unmarshal([]byte(completed.SerializedOutput), &results)) + slices.Sort(results) + require.Equal(t, eventCount, len(results)) + for index, value := range results { + require.Equal(t, index, value) + } + require.EqualValues(t, 1, activityExecutions.Load()) + countsMu.Lock() + defer countsMu.Unlock() + require.NotEmpty(t, partialCounts, "DTS did not deliver a batch large enough for partial consumption") + for _, count := range partialCounts { + require.EqualValues(t, 1, count) + } +} + +func TestDTSEmulatorSubOrchestrationRetryAndContinueAsNew(t *testing.T) { + var attempts atomic.Int32 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSFlaky", func(task.ActivityContext) (any, error) { + if attempts.Add(1) == 1 { + return nil, errors.New("retry me") + } + return "recovered", nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSChild", func(ctx *task.OrchestrationContext) (any, error) { + var result string + err := ctx.CallActivity( + "DTSFlaky", + task.WithActivityRetryPolicy(&task.RetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: 10 * time.Millisecond, + }), + ).Await(&result) + return result, err + })) + require.NoError(t, registry.AddOrchestratorN("DTSParent", func(ctx *task.OrchestrationContext) (any, error) { + var result string + err := ctx.CallSubOrchestrator("DTSChild").Await(&result) + return result, err + })) + require.NoError(t, registry.AddOrchestratorN("DTSContinueAsNew", func(ctx *task.OrchestrationContext) (any, error) { + var generation int + if err := ctx.GetInput(&generation); err != nil { + return nil, err + } + if generation < 2 { + ctx.ContinueAsNew(generation + 1) + return nil, nil + } + return generation, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + parentID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSParent", + api.WithInstanceID(uniqueInstanceID("go-parent")), + ) + require.NoError(t, err) + parent, err := managementClient.WaitForOrchestrationCompletion(ctx, parentID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"recovered"`, parent.SerializedOutput) + require.EqualValues(t, 2, attempts.Load()) + + continueID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSContinueAsNew", + api.WithInstanceID(uniqueInstanceID("go-continue")), + api.WithInput(0), + ) + require.NoError(t, err) + continued, err := managementClient.WaitForOrchestrationCompletion(ctx, continueID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, "2", continued.SerializedOutput) +} + +func TestDTSEmulatorScheduledTasksAndHistory(t *testing.T) { + options := emulatorOptions(t) + options.Versioning = &task.VersioningOptions{ + Version: "1.0", + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureFail, + } + registry := task.NewTaskRegistry() + var scheduledAttempts atomic.Int32 + targetName := "DTSScheduledTarget" + strings.ReplaceAll(uuid.NewString(), "-", "")[:8] + require.NoError(t, registry.AddOrchestratorNVersion(targetName, "1.0", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if scheduledAttempts.Add(1) == 1 { + return nil, errors.New("retry scheduled target") + } + return input, nil + })) + require.NoError(t, durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion( + registry, + options.Versioning.DefaultVersion, + )) + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskscheduler.WithScheduledTasks(), + durabletaskclient.WithAutoWorkItemFilters(), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + createdFrom := time.Now().UTC().Add(-time.Second) + scheduleID := "go-schedule-" + uuid.NewString() + scheduleTag := "scheduled-task-" + uuid.NewString() + handle, err := managementClient.ScheduledTasks().Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ + ScheduleID: scheduleID, + OrchestrationName: targetName, + TypedOrchestrationInput: "scheduled", + Interval: time.Hour, + StartImmediatelyIfLate: true, + Tags: map[string]string{"source": scheduleTag}, + ContextFields: api.ContextFields{"tenant": "north"}, + RetryPolicy: &durabletaskscheduler.ScheduleRetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: time.Second, + BackoffCoefficient: 1, + MaxRetryInterval: time.Second, + RetryTimeout: 10 * time.Second, + }, + }) + require.NoError(t, err) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + _ = handle.Delete(cleanupCtx) + }) + description, err := handle.Describe(ctx) + require.NoError(t, err) + require.Equal(t, durabletaskscheduler.ScheduleStatusActive, description.Status) + + var target *api.OrchestrationMetadata + var lastObserved []*api.OrchestrationMetadata + deadline := time.Now().Add(30 * time.Second) + for target == nil && time.Now().Before(deadline) { + result, queryErr := managementClient.QueryInstances(ctx, api.OrchestrationQuery{ + CreatedTimeFrom: createdFrom, + PageSize: 100, + FetchInputsAndOutputs: true, + }) + if queryErr == nil { + lastObserved = result.Orchestrations + for _, orchestration := range result.Orchestrations { + if orchestration.Name == targetName && + orchestration.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED && + orchestration.Tags["source"] == scheduleTag { + target = orchestration + break + } + } + } + if target == nil { + time.Sleep(200 * time.Millisecond) + } + } + if target == nil { + t.Fatalf("scheduled target did not complete; observed instances: %+v", lastObserved) + } + require.Equal(t, map[string]string{"source": scheduleTag}, target.Tags) + require.EqualValues(t, 2, scheduledAttempts.Load()) + + history, err := managementClient.GetOrchestrationHistory( + ctx, + target.InstanceID, + api.HistoryQuery{ExecutionID: target.ExecutionID}, + ) + require.NoError(t, err) + var foundContext bool + for _, event := range history.Events { + if event.ExecutionStarted != nil { + require.Equal(t, api.ContextFields{"tenant": "north"}, event.ExecutionStarted.ContextFields) + foundContext = true + } + } + require.True(t, foundContext) + + page, err := managementClient.ScheduledTasks().List(ctx, durabletaskscheduler.ScheduleQuery{ + ScheduleIDPrefix: scheduleID, + PageSize: 10, + }) + require.NoError(t, err) + require.Len(t, page.Schedules, 1) + + require.NoError(t, handle.Pause(ctx)) + description, err = handle.Describe(ctx) + require.NoError(t, err) + require.Equal(t, durabletaskscheduler.ScheduleStatusPaused, description.Status) + require.NoError(t, handle.Resume(ctx)) + require.NoError(t, handle.Delete(ctx)) + require.Eventually(t, func() bool { + schedule, getErr := managementClient.ScheduledTasks().Get(ctx, scheduleID) + return getErr == nil && schedule == nil + }, 10*time.Second, 200*time.Millisecond) +} + +func TestDTSEmulatorAzuriteBlobV2RoundTrip(t *testing.T) { + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run DTS and Azurite interop") + } + store, err := payload.NewAzureBlobStore(payload.AzureBlobStoreOptions{ + ConnectionString: connectionString, + Container: "dtgop1" + strings.ReplaceAll(uuid.NewString(), "-", "")[:16], + AllowInsecureHTTP: true, + }) + require.NoError(t, err) + options := emulatorOptions(t) + options.LargePayloads = &api.LargePayloadOptions{Store: store, Resolver: store} + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSBlobV2Echo", func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSBlobV2", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + first := ctx.CallActivity("DTSBlobV2Echo", task.WithActivityInput(input)) + second := ctx.CallActivity("DTSBlobV2Echo", task.WithActivityInput(input)) + if err := ctx.WhenAll(first, second); err != nil { + return nil, err + } + var firstResult, secondResult string + if err := first.Await(&firstResult); err != nil { + return nil, err + } + if err := second.Await(&secondResult); err != nil { + return nil, err + } + if firstResult != secondResult { + return nil, errors.New("large-payload activity results did not match") + } + return firstResult, nil + })) + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskclient.WithAutoWorkItemFilters(), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + // Two inline actions of this size would exceed the DTS completion limit. + // Blob externalization must reduce the fan-out response before it is sent. + input := strings.Repeat("blob-v2-large-payload-", 100_000) + instanceID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSBlobV2", + api.WithInstanceID(uniqueInstanceID("go-blob-v2")), + api.WithInput(input), + ) + require.NoError(t, err) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID) + require.NoError(t, err) + var output string + require.NoError(t, completed.ReadOutput(&output)) + require.Equal(t, input, output) + history, err := managementClient.GetOrchestrationHistory(ctx, instanceID, api.HistoryQuery{}) + require.NoError(t, err) + require.NotEmpty(t, history.Events) +} diff --git a/tests/durabletaskscheduler/dts_export_history_test.go b/tests/durabletaskscheduler/dts_export_history_test.go new file mode 100644 index 00000000..83fec467 --- /dev/null +++ b/tests/durabletaskscheduler/dts_export_history_test.go @@ -0,0 +1,430 @@ +package durabletaskscheduler_test + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/exporthistory" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// exportHistoryAzuriteStore builds an export store against a throwaway Azurite +// container. It skips when Azurite is not configured. +func exportHistoryAzuriteStore(t *testing.T) (*exporthistory.AzureBlobHistoryStore, string, *azblob.Client) { + t.Helper() + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run the export history end-to-end test") + } + var suffix [8]byte + _, err := rand.Read(suffix[:]) + require.NoError(t, err) + container := "dtsexport" + hex.EncodeToString(suffix[:]) + + store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ + ConnectionString: connectionString, + ContainerName: container, + AllowInsecureHTTP: true, + }) + require.NoError(t, err) + + // A second client reads the exported objects back without reaching into the + // store's internals. + reader, err := azblob.NewClientFromConnectionString(connectionString, nil) + require.NoError(t, err) + t.Cleanup(func() { + _, _ = reader.DeleteContainer(context.Background(), container, nil) + }) + return store, container, reader +} + +// TestDTSExportHistoryEndToEnd exports real orchestration histories from a live +// Durable Task Scheduler task hub into Azurite, covering the entity, the export +// orchestration, both activities, the durable checkpoint, and the exported +// object layout. +// +// It requires both a Durable Task Scheduler endpoint and Azurite; it skips when +// either is missing. +func TestDTSExportHistoryEndToEnd(t *testing.T) { + store, container, blobReader := exportHistoryAzuriteStore(t) + + testCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSExportEcho", func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return "echo:" + input, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSExportSubject", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var echoed string + if err := ctx.CallActivity("DTSExportEcho", task.WithActivityInput(input)).Await(&echoed); err != nil { + return nil, err + } + return echoed, nil + })) + + // The export activities read management APIs through the same task hub the + // worker is connected to, so the client is created before the worker. + options := emulatorOptions(t) + managementClient, worker := startExportHistoryDTSWorker(t, registry, store, options) + + prefix := fmt.Sprintf("dts-export-%d/", time.Now().UTC().UnixNano()) + exportClient, err := exporthistory.NewClient(managementClient.TaskHubGrpcClient, exporthistory.ClientOptions{ + ContainerName: container, + Prefix: prefix, + }) + require.NoError(t, err) + + // The task hub is shared with every other test in this package, so the + // export window starts now and excludes instances those tests already + // completed. Otherwise the job would export the whole hub. + from := time.Now().UTC() + subjects := make([]api.InstanceID, 0, 3) + for i := 0; i < 3; i++ { + id, err := managementClient.ScheduleNewOrchestration( + testCtx, "DTSExportSubject", api.WithInput(fmt.Sprintf("subject-%d", i))) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(testCtx, id) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, metadata.RuntimeStatus) + subjects = append(subjects, id) + } + + // The service's instance-ID index can lag orchestration completion, and a + // batch job that lists an empty first page legitimately completes with + // nothing exported. Wait until the subjects are listable so the test + // exercises the export path rather than that race. + listable, err := waitForListableInstances(testCtx, managementClient, from, subjects, 90*time.Second) + require.NoError(t, err) + if !listable { + t.Skip("DTS emulator did not expose the completed subjects through ListInstanceIds") + } + + jobID := strings.TrimSuffix(prefix, "/") + job, err := exportClient.CreateJob(testCtx, exporthistory.JobCreationOptions{ + JobID: jobID, + Mode: exporthistory.ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: time.Now().UTC(), + MaxInstancesPerBatch: 2, + }) + require.NoError(t, err) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cleanupCancel() + if err := job.Delete(cleanupCtx); err != nil { + t.Logf("failed to delete export job: %v", err) + } + }) + + var description *exporthistory.ExportJobDescription + require.Eventually(t, func() bool { + description, err = job.Describe(testCtx) + return err == nil && description.Status == exporthistory.ExportJobStatusCompleted + }, 4*time.Minute, time.Second, "export job did not complete against the live service") + + assert.Empty(t, description.LastError) + assert.Equal(t, description.ScannedInstances, description.ExportedInstances) + assert.GreaterOrEqual(t, description.ScannedInstances, int64(len(subjects))) + require.NotNil(t, description.Config) + assert.Equal(t, container, description.Config.Destination.Container) + assert.Equal(t, prefix, description.Config.Destination.Prefix) + + // Every subject's history landed in the destination as gzip-compressed JSONL. + exported := listExportedInstances(t, blobReader, container, prefix) + for _, id := range subjects { + assert.Contains(t, exported, string(id), "instance %s was not exported", id) + } + for instanceID, events := range exported { + require.NotEmpty(t, events, "instance %s exported no events", instanceID) + var sawExecutionStarted, sawExecutionCompleted bool + for _, event := range events { + switch event.Type { + case api.HistoryEventExecutionStarted: + sawExecutionStarted = true + case api.HistoryEventExecutionCompleted: + sawExecutionCompleted = true + } + } + assert.True(t, sawExecutionStarted, "instance %s is missing ExecutionStarted", instanceID) + assert.True(t, sawExecutionCompleted, "instance %s is missing ExecutionCompleted", instanceID) + } + + // Listing and per-job reads agree with the job's own description. + listed, err := exportClient.ListJobs(testCtx, exporthistory.ExportJobQuery{JobIDPrefix: jobID}) + require.NoError(t, err) + require.Len(t, listed.Jobs, 1) + assert.Equal(t, exporthistory.ExportJobStatusCompleted, listed.Jobs[0].Status) + + require.NotEmpty(t, description.OrchestratorInstanceID) + firstRunID := api.InstanceID(description.OrchestratorInstanceID) + firstRun, err := managementClient.WaitForOrchestrationCompletion(testCtx, firstRunID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, firstRun.RuntimeStatus) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cleanupCancel() + // Recreation retains older histories; Delete cleans only the current run. + if err := managementClient.PurgeOrchestrationState(cleanupCtx, firstRunID); err != nil { + t.Logf("failed to purge the previous export run: %v", err) + } + }) + + // A completed job can be recreated in place, which resets its progress and + // starts a second run that must genuinely execute rather than leaving the + // job Active but idle. + require.NoError(t, job.Create(testCtx, exporthistory.JobCreationOptions{ + JobID: jobID, + Mode: exporthistory.ExportModeBatch, + CompletedTimeFrom: from, + CompletedTimeTo: time.Now().UTC(), + })) + recreated, err := job.Describe(testCtx) + require.NoError(t, err) + require.NotEmpty(t, recreated.OrchestratorInstanceID) + assert.NotEqual(t, description.OrchestratorInstanceID, recreated.OrchestratorInstanceID) + assert.Equal(t, description.CreatedAt, recreated.CreatedAt) + + retainedRun, err := managementClient.FetchOrchestrationMetadata(testCtx, firstRunID) + require.NoError(t, err, "recreating a job must retain the previous generation's orchestration history") + assert.Equal(t, api.RUNTIME_STATUS_COMPLETED, retainedRun.RuntimeStatus) + + var second *exporthistory.ExportJobDescription + require.Eventually(t, func() bool { + second, err = job.Describe(testCtx) + return err == nil && + second.Status == exporthistory.ExportJobStatusCompleted && + second.ScannedInstances >= int64(len(subjects)) + }, 4*time.Minute, time.Second, "the recreated export job did not run again against the live service") + assert.Empty(t, second.LastError) + assert.Equal(t, recreated.OrchestratorInstanceID, second.OrchestratorInstanceID) + assert.Equal(t, second.ScannedInstances, second.ExportedInstances) + assert.True(t, second.LastModifiedAt.After(description.LastModifiedAt), + "the recreated run must have made progress after the first one finished") + require.True(t, worker.Running()) +} + +// TestDTSExportHistoryJobNotFound covers the typed not-found error against the +// live service without needing Azurite. +func TestDTSExportHistoryJobNotFound(t *testing.T) { + options := emulatorOptions(t) + testCtx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + managementClient, err := durabletaskscheduler.NewClient(testCtx, options, api.DefaultLogger()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, managementClient.Close()) }) + exportClient, err := exporthistory.NewClient(managementClient.TaskHubGrpcClient, exporthistory.ClientOptions{ + ContainerName: "history-exports", + }) + require.NoError(t, err) + + var suffix [8]byte + _, err = rand.Read(suffix[:]) + require.NoError(t, err) + _, err = exportClient.GetJob(testCtx, "missing-"+hex.EncodeToString(suffix[:])) + require.ErrorIs(t, err, exporthistory.ErrJobNotFound) +} + +// waitForListableInstances blocks until every instance in want is visible to the +// management instance-ID query the export job uses. +func waitForListableInstances( + ctx context.Context, + client *durabletaskscheduler.Client, + from time.Time, + want []api.InstanceID, + timeout time.Duration, +) (bool, error) { + deadline := time.Now().Add(timeout) + for { + seen := map[api.InstanceID]bool{} + token := "" + for page := 0; page < 100; page++ { + if ctx.Err() != nil { + return false, ctx.Err() + } + if time.Now().After(deadline) { + return false, nil + } + result, err := client.ListInstanceIDs(ctx, api.InstanceIDQuery{ + RuntimeStatus: exporthistory.TerminalStatuses(), + CompletedTimeFrom: from, + PageSize: 100, + ContinuationToken: token, + }) + if err != nil { + return false, err + } + if len(result.InstanceIDs) == 0 { + break + } + for _, id := range result.InstanceIDs { + seen[id] = true + } + if result.ContinuationToken == "" { + break + } + token = result.ContinuationToken + } + allVisible := true + for _, id := range want { + if !seen[id] { + allVisible = false + break + } + } + if allVisible { + return true, nil + } + if ctx.Err() != nil || time.Now().After(deadline) { + return false, ctx.Err() + } + timer := time.NewTimer(time.Second) + select { + case <-ctx.Done(): + timer.Stop() + return false, ctx.Err() + case <-timer.C: + } + } +} + +// listExportedInstances reads every exported object under prefix and returns the +// decoded history keyed by instance ID. +func listExportedInstances( + t *testing.T, + client *azblob.Client, + container string, + prefix string, +) map[string][]api.HistoryEvent { + t.Helper() + ctx := context.Background() + exported := map[string][]api.HistoryEvent{} + pager := client.NewListBlobsFlatPager(container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: azblob.ListBlobsInclude{Metadata: true}, + }) + for pager.More() { + page, err := pager.NextPage(ctx) + require.NoError(t, err) + for _, item := range page.Segment.BlobItems { + name := *item.Name + assert.True(t, strings.HasSuffix(name, ".jsonl.gz"), name) + instanceID := "" + for key, value := range item.Metadata { + if strings.EqualFold(key, "instanceId") && value != nil { + instanceID = *value + } + } + require.NotEmpty(t, instanceID, "exported object %s has no instanceId metadata", name) + // The object is an opaque gzip file, so nothing transparently + // decompresses it and the download is always the gzip stream. + assert.Equal(t, "application/gzip", derefBlobString(item.Properties.ContentType), name) + assert.Empty(t, derefBlobString(item.Properties.ContentEncoding), name) + + response, err := client.DownloadStream(ctx, container, name, nil) + require.NoError(t, err) + payload, err := io.ReadAll(response.Body) + require.NoError(t, err) + require.NoError(t, response.Body.Close()) + exported[instanceID] = decodeExportedJSONL(t, payload) + } + } + return exported +} + +// derefBlobString reads an optional blob property. +func derefBlobString(value *string) string { + if value == nil { + return "" + } + return *value +} + +// decodeExportedJSONL decodes an exported object. The export never declares a +// content coding, so the downloaded bytes are always the gzip stream the object +// name promises and no raw-versus-decompressed guessing is needed. +func decodeExportedJSONL(t *testing.T, payload []byte) []api.HistoryEvent { + t.Helper() + reader, err := gzip.NewReader(bytes.NewReader(payload)) + require.NoError(t, err) + decompressed, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + + events := []api.HistoryEvent{} + for _, line := range strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") { + if line == "" { + continue + } + var event api.HistoryEvent + require.NoError(t, json.Unmarshal([]byte(line), &event)) + events = append(events, event) + } + return events +} + +// startExportHistoryDTSWorker starts a Durable Task Scheduler worker that runs +// both the application tasks in registry and the export history system tasks. +// The export activities read management APIs through the same client the caller +// uses, which is how a production worker is wired. +func startExportHistoryDTSWorker( + t *testing.T, + registry *task.TaskRegistry, + store exporthistory.Store, + options *durabletaskscheduler.Options, +) (*durabletaskscheduler.Client, *durabletaskclient.TaskHubGrpcWorker) { + t.Helper() + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + + require.NoError(t, exporthistory.Register(registry, exporthistory.WorkerOptions{ + Source: managementClient.TaskHubGrpcClient, + Store: store, + })) + + worker, err := durabletaskscheduler.NewWorker(options, registry, logger, + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), + durabletaskclient.WithMaxConcurrentActivityWorkItems(8), + durabletaskclient.WithMaxConcurrentEntityWorkItems(4), + durabletaskclient.WithWorkerSilentDisconnectTimeout(15*time.Second), + exporthistory.WithExportHistory(), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := worker.Shutdown(shutdownCtx); err != nil { + t.Logf("failed to shut down export history worker: %v", err) + } + require.NoError(t, managementClient.Close()) + }) + return managementClient, worker +} diff --git a/tests/durabletaskscheduler/dts_failure_and_management_test.go b/tests/durabletaskscheduler/dts_failure_and_management_test.go new file mode 100644 index 00000000..0ddf85dc --- /dev/null +++ b/tests/durabletaskscheduler/dts_failure_and_management_test.go @@ -0,0 +1,428 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/task" + "github.com/microsoft/durabletask-go/tests/failurechain" + "github.com/stretchr/testify/require" +) + +// dtsLeafError opts into every durable failure enrichment hook so the failure +// chain can be validated across a real Durable Task Scheduler service. +var dtsLeafError = &failurechain.LeafError{ + Message: "dts leaf boom", + ErrorType: "Contoso.DTSLeafError", + Stack: "Contoso.Leaf.Run(leaf.go:11)", + Properties: map[string]any{"code": "E11", "attempts": 4}, + IsNonRetriable: true, +} + +// TestDTSEmulatorDeepFailureChain asserts a Parent -> Sub-orchestration -> +// Activity failure keeps its error types, messages, custom properties, stack +// traces, non-retriable flags, and inner failures when it round-trips through +// the Durable Task Scheduler service. +func TestDTSEmulatorDeepFailureChain(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSFailureLeaf", func(task.ActivityContext) (any, error) { + return nil, dtsLeafError + })) + require.NoError(t, registry.AddOrchestratorN("DTSFailureChild", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("DTSFailureLeaf").Await(nil) + })) + require.NoError(t, registry.AddOrchestratorN("DTSFailureParent", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallSubOrchestrator( + "DTSFailureChild", + task.WithSubOrchestrationInstanceID(string(ctx.ID)+"-child"), + ).Await(nil) + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + parentID := uniqueInstanceID("dts-failure-chain") + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSFailureParent", + api.WithInstanceID(parentID), + ) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, parentID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, metadata.RuntimeStatus) + require.NotNil(t, metadata.FailureDetails) + + parent := metadata.FailureDetails + require.Equal(t, api.ErrorTypeTaskFailed, parent.ErrorType) + require.Contains(t, parent.ErrorMessage, "Task 'DTSFailureChild' (#0) failed with an unhandled exception") + require.Contains(t, parent.ErrorMessage, "dts leaf boom") + require.True(t, parent.IsNonRetriable) + + child := parent.InnerFailure + require.NotNil(t, child, "DTS must preserve the sub-orchestration failure frame") + require.Equal(t, api.ErrorTypeTaskFailed, child.ErrorType) + require.Contains(t, child.ErrorMessage, "Task 'DTSFailureLeaf' (#0) failed with an unhandled exception") + require.True(t, child.IsNonRetriable) + + leaf := child.InnerFailure + require.NotNil(t, leaf, "DTS must preserve the activity failure frame") + require.Equal(t, api.ErrorType("Contoso.DTSLeafError"), leaf.ErrorType) + require.Equal(t, "dts leaf boom", leaf.ErrorMessage) + require.True(t, leaf.IsNonRetriable) + require.Nil(t, leaf.InnerFailure) + require.True(t, metadata.FailureDetails.IsCausedBy("Contoso.DTSLeafError")) + + if leaf.StackTrace == "" { + t.Log("DTS emulator limitation: activity failure stack traces are not returned") + } else { + require.Equal(t, "Contoso.Leaf.Run(leaf.go:11)", leaf.StackTrace) + } + if len(leaf.Properties) == 0 { + t.Log("DTS emulator limitation: custom failure properties are not returned") + } else { + require.Equal(t, "E11", leaf.Properties["code"]) + require.Equal(t, float64(4), leaf.Properties["attempts"]) + } + + childMetadata, err := managementClient.FetchOrchestrationMetadata( + ctx, + parentID+"-child", + api.WithFetchPayloads(true), + ) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, childMetadata.RuntimeStatus) + require.NotNil(t, childMetadata.FailureDetails) + require.Equal(t, api.ErrorTypeTaskFailed, childMetadata.FailureDetails.ErrorType) + require.Equal( + t, + api.ErrorType("Contoso.DTSLeafError"), + childMetadata.FailureDetails.InnerFailure.ErrorType, + ) +} + +// TestDTSEmulatorMissingTaskBypassesRetryHandlers asserts an unregistered +// activity produces the canonical non-retriable not-found failure through DTS +// and that the retry handler is never consulted for it. +func TestDTSEmulatorMissingTaskBypassesRetryHandlers(t *testing.T) { + registry := task.NewTaskRegistry() + handlerCalls := make(chan struct{}, 8) + require.NoError(t, registry.AddOrchestratorN("DTSMissingHost", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("DTSMissingActivity", task.WithActivityRetryPolicy(&task.RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 10 * time.Millisecond, + BackoffCoefficient: 1, + Handle: func(task.RetryContext) bool { + select { + case handlerCalls <- struct{}{}: + default: + } + return true + }, + })).Await(nil) + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + id := uniqueInstanceID("dts-missing-task") + _, err := managementClient.ScheduleNewOrchestration(ctx, "DTSMissingHost", api.WithInstanceID(id)) + require.NoError(t, err) + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, metadata.RuntimeStatus) + require.NotNil(t, metadata.FailureDetails) + require.True(t, metadata.FailureDetails.Matches(api.ErrTaskNotRegistered)) + require.True(t, metadata.FailureDetails.IsCausedBy(api.ErrorTypeActivityTaskNotFound)) + require.True(t, metadata.FailureDetails.IsNonRetriable) + require.Empty(t, handlerCalls, "retry handler must be bypassed for non-retriable failures") +} + +// TestDTSEmulatorSuspendResumeNoOps asserts the redundant suspend and resume +// management contracts against a real service: suspending a completed instance +// and resuming a running instance must not change the instance. +func TestDTSEmulatorSuspendResumeNoOps(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSNoOpComplete", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSNoOpWait", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("release", -1).Await(nil) + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + completedID := uniqueInstanceID("dts-noop-completed") + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSNoOpComplete", + api.WithInstanceID(completedID), + api.WithInput("done"), + ) + require.NoError(t, err) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, completedID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completed.RuntimeStatus) + + if err := managementClient.SuspendOrchestration(ctx, completedID, "too late"); err != nil { + // Some services reject the request outright rather than dropping it. + t.Logf("DTS emulator limitation: suspending a completed instance returned %v", err) + } + if err := managementClient.ResumeOrchestration(ctx, completedID, "still too late"); err != nil { + t.Logf("DTS emulator limitation: resuming a completed instance returned %v", err) + } + require.Never(t, func() bool { + metadata, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, completedID) + return fetchErr != nil || metadata.RuntimeStatus != api.RUNTIME_STATUS_COMPLETED + }, 2*time.Second, 200*time.Millisecond) + + runningID := uniqueInstanceID("dts-noop-running") + _, err = managementClient.ScheduleNewOrchestration(ctx, "DTSNoOpWait", api.WithInstanceID(runningID)) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, runningID) + require.NoError(t, err) + + require.NoError(t, managementClient.ResumeOrchestration(ctx, runningID, "no-op")) + require.NoError(t, managementClient.ResumeOrchestration(ctx, runningID, "still a no-op")) + require.Never(t, func() bool { + metadata, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, runningID) + return fetchErr != nil || metadata.RuntimeStatus != api.RUNTIME_STATUS_RUNNING + }, 2*time.Second, 200*time.Millisecond) + + require.NoError(t, managementClient.RaiseEvent(ctx, runningID, "release")) + finished, err := managementClient.WaitForOrchestrationCompletion(ctx, runningID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, finished.RuntimeStatus) +} + +// TestDTSEmulatorTerminateWhileSuspended asserts termination is not blocked by +// suspension when the orchestration state is owned by a real service. +func TestDTSEmulatorTerminateWhileSuspended(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSSuspendWait", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.WaitForSingleEvent("release", -1).Await(nil) + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + id := uniqueInstanceID("dts-terminate-suspended") + _, err := managementClient.ScheduleNewOrchestration(ctx, "DTSSuspendWait", api.WithInstanceID(id)) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, id) + require.NoError(t, err) + require.NoError(t, managementClient.SuspendOrchestration(ctx, id, "hold")) + require.Eventually(t, func() bool { + metadata, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, id) + return fetchErr == nil && metadata.RuntimeStatus == api.RUNTIME_STATUS_SUSPENDED + }, 20*time.Second, 100*time.Millisecond) + + require.NoError(t, managementClient.TerminateOrchestration(ctx, id)) + terminated, err := managementClient.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_TERMINATED, terminated.RuntimeStatus) +} + +// TestDTSEmulatorRestartMissingInstanceIsTypedNotFound asserts management +// requests for unknown instances surface as typed API errors instead of opaque +// RPC failures. +func TestDTSEmulatorRestartMissingInstanceIsTypedNotFound(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSTypedErrors", func(*task.OrchestrationContext) (any, error) { + return nil, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + missingID := uniqueInstanceID("dts-missing-instance") + + _, err := managementClient.RestartInstance(ctx, missingID) + switch { + case errors.Is(err, api.ErrFeatureNotSupported): + t.Log("DTS emulator limitation: RestartInstance is not implemented") + case errors.Is(err, api.ErrInstanceNotFound): + default: + t.Fatalf("RestartInstance() error = %v, want api.ErrInstanceNotFound", err) + } + + _, err = managementClient.FetchOrchestrationMetadata(ctx, missingID) + require.ErrorIs(t, err, api.ErrInstanceNotFound) + + err = managementClient.PurgeOrchestrationState(ctx, missingID) + if err != nil { + require.ErrorIs(t, err, api.ErrInstanceNotFound) + } else { + t.Log("DTS emulator limitation: purging a missing instance succeeds instead of reporting not-found") + } +} + +// TestDTSEmulatorPaginationContinuationEquivalence asserts paging through a +// service query with small pages yields the same instance set as one large page. +func TestDTSEmulatorPaginationContinuationEquivalence(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSPage", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + prefix := string(uniqueInstanceID("dts-page")) + const instanceCount = 6 + expected := make(map[api.InstanceID]struct{}, instanceCount) + for index := range instanceCount { + id := api.InstanceID(fmt.Sprintf("%s-%02d", prefix, index)) + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSPage", + api.WithInstanceID(id), + api.WithInput(fmt.Sprintf("value-%d", index)), + ) + require.NoError(t, err) + completed, err := managementClient.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completed.RuntimeStatus) + expected[id] = struct{}{} + } + + collect := func(pageSize int) (map[api.InstanceID]struct{}, error) { + collected := make(map[api.InstanceID]struct{}, instanceCount) + token := "" + for pages := 0; pages < instanceCount+2; pages++ { + page, err := managementClient.QueryInstances(ctx, api.OrchestrationQuery{ + InstanceIDPrefix: prefix, + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + PageSize: pageSize, + ContinuationToken: token, + }) + if err != nil { + return nil, err + } + if len(page.Orchestrations) > pageSize { + return nil, fmt.Errorf("page returned %d items for page size %d", len(page.Orchestrations), pageSize) + } + for _, item := range page.Orchestrations { + collected[item.InstanceID] = struct{}{} + } + token = page.ContinuationToken + if token == "" { + return collected, nil + } + } + return nil, errors.New("pagination did not terminate") + } + + single, err := collect(instanceCount + 4) + if errors.Is(err, api.ErrFeatureNotSupported) { + t.Log("DTS emulator limitation: QueryInstances is not implemented") + return + } + require.NoError(t, err) + require.Equal(t, expected, single) + + for _, pageSize := range []int{1, 2, 4} { + paged, err := collect(pageSize) + require.NoErrorf(t, err, "page size %d", pageSize) + require.Equalf(t, single, paged, "page size %d", pageSize) + } +} + +// TestDTSEmulatorOrchestrationIDReuse asserts the current status-based instance +// ID deduplication contract against a real service. +func TestDTSEmulatorOrchestrationIDReuse(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSReuseComplete", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + // scheduleDuplicate creates a completed instance and then re-schedules the + // same ID with the supplied options, returning the resulting error. + scheduleDuplicate := func(t *testing.T, prefix string, extra ...api.NewOrchestrationOptions) (api.InstanceID, error) { + t.Helper() + id := uniqueInstanceID(prefix) + _, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSReuseComplete", + api.WithInstanceID(id), + api.WithInput("original"), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + + options := append([]api.NewOrchestrationOptions{ + api.WithInstanceID(id), + api.WithInput("replacement"), + }, extra...) + _, err = managementClient.ScheduleNewOrchestration(ctx, "DTSReuseComplete", options...) + return id, err + } + + t.Run("empty-dedupe-set-replaces-completed-instance", func(t *testing.T) { + id, err := scheduleDuplicate(t, "dts-reuse-all", api.WithOrchestrationIDReusePolicy( + &api.OrchestrationIDReusePolicy{ + DedupeStatuses: []api.OrchestrationStatus{}, + }, + )) + require.NoError(t, err) + require.Eventually(t, func() bool { + metadata, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + return fetchErr == nil && + metadata.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED && + metadata.SerializedOutput == `"replacement"` + }, 20*time.Second, 100*time.Millisecond) + }) + + t.Run("dedupe-status-rejects-completed-instance", func(t *testing.T) { + id, err := scheduleDuplicate(t, "dts-reuse-dedupe", api.WithOrchestrationIDReusePolicy( + &api.OrchestrationIDReusePolicy{ + DedupeStatuses: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + }, + )) + require.ErrorIs(t, err, api.ErrDuplicateInstance) + current, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, fetchErr) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, current.RuntimeStatus) + require.Equal(t, `"original"`, current.SerializedOutput) + }) + + t.Run("all-dedupe-statuses-replace-nothing", func(t *testing.T) { + _, err := scheduleDuplicate(t, "dts-reuse-none", api.WithOrchestrationIDReusePolicy( + &api.OrchestrationIDReusePolicy{ + DedupeStatuses: []api.OrchestrationStatus{ + api.RUNTIME_STATUS_RUNNING, + api.RUNTIME_STATUS_COMPLETED, + api.RUNTIME_STATUS_FAILED, + api.RUNTIME_STATUS_CANCELED, + api.RUNTIME_STATUS_TERMINATED, + api.RUNTIME_STATUS_PENDING, + api.RUNTIME_STATUS_SUSPENDED, + }, + }, + )) + require.ErrorIs(t, err, api.ErrDuplicateInstance) + }) +} diff --git a/tests/durabletaskscheduler/dts_tracing_tree_test.go b/tests/durabletaskscheduler/dts_tracing_tree_test.go new file mode 100644 index 00000000..ff39eaa9 --- /dev/null +++ b/tests/durabletaskscheduler/dts_tracing_tree_test.go @@ -0,0 +1,499 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/task" + "github.com/microsoft/durabletask-go/tests/tracingtree" + "github.com/stretchr/testify/require" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// Tracing on the Durable Task Scheduler surface is owned by the scheduler service, not by +// this SDK: the Go process only runs the client and the worker, and neither creates +// orchestration, activity, or timer spans. These tests therefore assert the two halves of +// the contract that this SDK is actually responsible for: +// +// 1. The caller's W3C trace context reaches the scheduler and is durably recorded on the +// history, so the scheduler-side span tree is rooted in the caller's trace. +// 2. The worker process itself never emits duplicate durabletask spans for work it runs. +// +// Emulator limitations that are asserted loosely on purpose are called out inline. + +func startDTSCallerSpan(t *testing.T, parent context.Context, name string) (context.Context, trace.Span) { + t.Helper() + return tracingtree.StartCallerSpan(t, "tests/durabletaskscheduler", parent, name) +} + +// expectedTraceParent renders the traceparent header the SDK must send for a span. +func expectedTraceParent(span trace.Span) string { + spanContext := span.SpanContext() + return fmt.Sprintf("00-%s-%s-%s", spanContext.TraceID(), spanContext.SpanID(), spanContext.TraceFlags()) +} + +func traceParentSpanID(t *testing.T, traceParent string) string { + t.Helper() + require.Len(t, traceParent, 55, "unexpected traceparent format %q", traceParent) + return traceParent[36:52] +} + +func fetchHistory(t *testing.T, ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, executionID string) []*api.HistoryEvent { + t.Helper() + history, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID}) + require.NoError(t, err) + return history.Events +} + +// executionStarted returns the single ExecutionStarted record of a history. +func executionStarted(t *testing.T, events []*api.HistoryEvent) *api.HistoryExecutionStartedEvent { + t.Helper() + var started *api.HistoryExecutionStartedEvent + for _, event := range events { + if event.ExecutionStarted != nil { + require.Nil(t, started, "history contains more than one ExecutionStarted event") + started = event.ExecutionStarted + } + } + require.NotNil(t, started, "history has no ExecutionStarted event") + return started +} + +func countEvents(events []*api.HistoryEvent, eventType api.HistoryEventType) int { + count := 0 + for _, event := range events { + if event.Type == eventType { + count++ + } + } + return count +} + +// requireTraceContextsShareTrace asserts that every trace context recorded on a history +// belongs to wantTraceID. Records without a trace context are skipped: the emulator does +// not stamp one on every event type, and that gap is asserted separately where it matters. +func requireTraceContextsShareTrace(t *testing.T, events []*api.HistoryEvent, wantTraceID string) { + t.Helper() + check := func(label string, tc *api.HistoryTraceContext) { + if tc == nil || tc.TraceParent == "" { + return + } + require.Containsf(t, tc.TraceParent, wantTraceID, + "%s carries trace context %q, which is outside the caller's trace %s", label, tc.TraceParent, wantTraceID) + } + for _, event := range events { + switch { + case event.ExecutionStarted != nil: + check("ExecutionStarted", event.ExecutionStarted.ParentTraceContext) + case event.TaskScheduled != nil: + check("TaskScheduled("+event.TaskScheduled.Name+")", event.TaskScheduled.ParentTraceContext) + case event.SubOrchestrationInstanceCreated != nil: + check("SubOrchestrationInstanceCreated("+event.SubOrchestrationInstanceCreated.Name+")", + event.SubOrchestrationInstanceCreated.ParentTraceContext) + } + } +} + +// requireNoWorkerEmittedSpans asserts the documented Durable Task Scheduler limitation: the +// Go worker and client do not emit durabletask spans, because the scheduler service owns +// orchestration, activity, and timer span emission. This also guards against a regression +// that would duplicate the service-side spans locally. +func requireNoWorkerEmittedSpans(t *testing.T, exporter *tracetest.InMemoryExporter) { + t.Helper() + spans := exporter.GetSpans().Snapshots() + durableSpans := make([]sdktrace.ReadOnlySpan, 0, len(spans)) + for _, span := range spans { + if span.InstrumentationScope().Name == "durabletask" { + durableSpans = append(durableSpans, span) + } + } + require.Emptyf(t, durableSpans, + "the Durable Task Scheduler worker must not emit durabletask spans locally; got: %s", + tracingtree.Describe(durableSpans)) +} + +// TestDTSEmulatorTracingTreeActivityFailureSubOrchestrationsAndEvents drives one +// orchestration through every durable construct that participates in the trace tree +// (activity failure, sub-orchestration success and failure, durable timer, client-raised +// event, and an orchestration-sent event) and verifies the caller's trace context is what +// the scheduler persists. +func TestDTSEmulatorTracingTreeActivityFailureSubOrchestrationsAndEvents(t *testing.T) { + exporter := tracingtree.Init() + + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN("DTSTraceEcho", func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil + })) + require.NoError(t, registry.AddActivityN("DTSTraceFailingActivity", func(task.ActivityContext) (any, error) { + return nil, errors.New("dts activity exploded") + })) + require.NoError(t, registry.AddOrchestratorN("DTSTraceChildOK", func(ctx *task.OrchestrationContext) (any, error) { + var output string + if err := ctx.CallActivity("DTSTraceEcho", task.WithActivityInput("child")).Await(&output); err != nil { + return nil, err + } + return output, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSTraceChildFailed", func(ctx *task.OrchestrationContext) (any, error) { + return nil, ctx.CallActivity("DTSTraceFailingActivity").Await(nil) + })) + require.NoError(t, registry.AddOrchestratorN("DTSTraceReceiver", func(ctx *task.OrchestrationContext) (any, error) { + var payload string + if err := ctx.WaitForSingleEvent("ping", 60*time.Second).Await(&payload); err != nil { + return nil, err + } + return payload, nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSTraceParent", func(ctx *task.OrchestrationContext) (any, error) { + var receiver api.InstanceID + if err := ctx.GetInput(&receiver); err != nil { + return nil, err + } + var completed string + if err := ctx.CallSubOrchestrator( + "DTSTraceChildOK", + task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_ok"), + ).Await(&completed); err != nil { + return nil, err + } + if err := ctx.CreateTimer(10 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var signal string + if err := ctx.WaitForSingleEvent("proceed", 60*time.Second).Await(&signal); err != nil { + return nil, err + } + if err := ctx.SendEvent(receiver, "ping", "pong"); err != nil { + return nil, err + } + if err := ctx.CallSubOrchestrator( + "DTSTraceChildFailed", + task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_failed"), + ).Await(nil); err == nil { + return nil, errors.New("expected the child orchestration to fail") + } + return completed + "+" + signal, nil + })) + managementClient, _, _ := startEmulatorClientAndWorker(t, registry) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + receiverID, err := managementClient.ScheduleNewOrchestration( + ctx, + "DTSTraceReceiver", + api.WithInstanceID(uniqueInstanceID("go-trace-receiver")), + ) + require.NoError(t, err) + _, err = managementClient.WaitForOrchestrationStart(ctx, receiverID) + require.NoError(t, err) + + callerCtx, callerSpan := startDTSCallerSpan(t, ctx, "caller/dts-trace-tree") + parentID, err := managementClient.ScheduleNewOrchestration( + callerCtx, + "DTSTraceParent", + api.WithInstanceID(uniqueInstanceID("go-trace-parent")), + api.WithInput(receiverID), + ) + require.NoError(t, err) + callerSpan.End() + callerTraceID := callerSpan.SpanContext().TraceID().String() + + _, err = managementClient.WaitForOrchestrationStart(ctx, parentID) + require.NoError(t, err) + require.NoError(t, managementClient.RaiseEvent(ctx, parentID, "proceed", api.WithEventPayload("go"))) + + parentMetadata, err := managementClient.WaitForOrchestrationCompletion(ctx, parentID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, parentMetadata.RuntimeStatus) + require.Equal(t, `"child+go"`, parentMetadata.SerializedOutput) + receiverMetadata, err := managementClient.WaitForOrchestrationCompletion(ctx, receiverID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, `"pong"`, receiverMetadata.SerializedOutput) + failedChild, err := managementClient.WaitForOrchestrationCompletion(ctx, parentID+"_failed") + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, failedChild.RuntimeStatus) + require.NotNil(t, failedChild.FailureDetails) + require.Contains(t, failedChild.FailureDetails.ErrorMessage, "dts activity exploded") + + // The caller's trace context must be persisted verbatim on the orchestration that the + // client scheduled. This is the hand-off point between the caller's trace and the + // scheduler-side span tree. + parentEvents := fetchHistory(t, ctx, managementClient, parentID, parentMetadata.ExecutionID) + started := executionStarted(t, parentEvents) + require.NotNil(t, started.ParentTraceContext, "the scheduler did not persist the caller's trace context") + require.Equal(t, expectedTraceParent(callerSpan), started.ParentTraceContext.TraceParent) + requireTraceContextsShareTrace(t, parentEvents, callerTraceID) + actionSpanIDs := make(map[string]struct{}) + childTraceParents := make(map[api.InstanceID]string) + + // Durable timers and raised events are recorded on the parent's history, but the + // emulator does not stamp a trace context on those records, so they can only be + // correlated through the orchestration instance. + require.Equal(t, 1, countEvents(parentEvents, api.HistoryEventTimerCreated)) + require.Equal(t, 1, countEvents(parentEvents, api.HistoryEventTimerFired)) + require.Equal(t, 1, countEvents(parentEvents, api.HistoryEventEventRaised)) + require.Equal(t, 2, countEvents(parentEvents, api.HistoryEventSubOrchestrationInstanceCreated)) + for _, event := range parentEvents { + if created := event.SubOrchestrationInstanceCreated; created != nil { + require.NotNil(t, created.ParentTraceContext) + require.Contains(t, created.ParentTraceContext.TraceParent, callerTraceID) + spanID := traceParentSpanID(t, created.ParentTraceContext.TraceParent) + require.NotEqual(t, callerSpan.SpanContext().SpanID().String(), spanID) + require.NotEqual(t, started.OrchestrationSpanID, spanID) + require.NotContains(t, actionSpanIDs, spanID) + actionSpanIDs[spanID] = struct{}{} + childTraceParents[created.InstanceID] = created.ParentTraceContext.TraceParent + } + } + + // The orchestration-sent event lands on the receiver as a raised event. Cross-instance + // causality is not carried on the wire by the emulator, so the receiving instance keeps + // its own trace; only the presence of the event is asserted here. + receiverEvents := fetchHistory(t, ctx, managementClient, receiverID, receiverMetadata.ExecutionID) + require.Equal(t, 1, countEvents(receiverEvents, api.HistoryEventEventRaised)) + + for _, childID := range []api.InstanceID{parentID + "_ok", parentID + "_failed"} { + childEvents := fetchHistory(t, ctx, managementClient, childID, "") + childStarted := executionStarted(t, childEvents) + require.NotNil(t, childStarted.ParentTraceContext) + require.Contains(t, childStarted.ParentTraceContext.TraceParent, callerTraceID) + require.Equal(t, childTraceParents[childID], childStarted.ParentTraceContext.TraceParent) + var foundActivityTrace bool + for _, event := range childEvents { + if scheduled := event.TaskScheduled; scheduled != nil { + require.NotNil(t, scheduled.ParentTraceContext) + require.Contains(t, scheduled.ParentTraceContext.TraceParent, callerTraceID) + spanID := traceParentSpanID(t, scheduled.ParentTraceContext.TraceParent) + require.NotEqual(t, callerSpan.SpanContext().SpanID().String(), spanID) + require.NotEqual(t, childStarted.OrchestrationSpanID, spanID) + require.NotContains(t, actionSpanIDs, spanID) + actionSpanIDs[spanID] = struct{}{} + foundActivityTrace = true + } + } + require.True(t, foundActivityTrace) + requireTraceContextsShareTrace(t, childEvents, callerTraceID) + } + + requireNoWorkerEmittedSpans(t, exporter) +} + +// TestDTSEmulatorTracingTreeVersionMigration verifies that a version-migrating +// continue-as-new keeps the caller's trace context, so every generation of an instance +// stays inside the trace that started it. +func TestDTSEmulatorTracingTreeVersionMigration(t *testing.T) { + exporter := tracingtree.Init() + + options := emulatorOptions(t) + options.Versioning = &task.VersioningOptions{ + Version: "1.0", + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchNone, + FailureStrategy: task.VersionFailureFail, + } + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityNVersion("DTSTraceVersionedActivity", "1.0", func(task.ActivityContext) (any, error) { + return "v1-activity", nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("DTSTraceVersioned", "1.0", func(ctx *task.OrchestrationContext) (any, error) { + var output string + if err := ctx.CallActivity("DTSTraceVersionedActivity").Await(&output); err != nil { + return nil, err + } + ctx.ContinueAsNew(output, task.WithContinueAsNewVersion("2.0")) + return nil, nil + })) + require.NoError(t, registry.AddOrchestratorNVersion("DTSTraceVersioned", "2.0", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + "+v2", nil + })) + + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker(options, registry, logger) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + callerCtx, callerSpan := startDTSCallerSpan(t, ctx, "caller/dts-version-migration") + instanceID, err := managementClient.ScheduleNewOrchestration( + callerCtx, + "DTSTraceVersioned", + api.WithInstanceID(uniqueInstanceID("go-trace-version")), + api.WithTags(map[string]string{"migration": "preserved"}), + ) + require.NoError(t, err) + callerSpan.End() + + metadata, err := managementClient.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, metadata.RuntimeStatus) + require.Equal(t, "2.0", metadata.Version) + require.Equal(t, `"v1-activity+v2"`, metadata.SerializedOutput) + require.Equal(t, "preserved", metadata.Tags["migration"]) + + // The migrated generation is a new execution, and it must still carry the trace context + // of the client call that started generation 1. + events := fetchHistory(t, ctx, managementClient, instanceID, metadata.ExecutionID) + started := executionStarted(t, events) + require.Equal(t, "2.0", started.Version) + require.NotNil(t, started.ParentTraceContext, "continue-as-new dropped the caller's trace context") + require.Equal(t, expectedTraceParent(callerSpan), started.ParentTraceContext.TraceParent) + requireTraceContextsShareTrace(t, events, callerSpan.SpanContext().TraceID().String()) + + // Emulator limitation: only the latest generation's history is retained, so the 1.0 + // generation cannot be re-read after the migration completes. + require.Equal(t, 0, countEvents(events, api.HistoryEventTaskScheduled)) + + requireNoWorkerEmittedSpans(t, exporter) +} + +// TestDTSEmulatorTracingTreeScheduledTask verifies the trace shape of orchestrations +// started by the scheduled-task (schedule entity) surface rather than by a client call. +func TestDTSEmulatorTracingTreeScheduledTask(t *testing.T) { + exporter := tracingtree.Init() + + options := emulatorOptions(t) + options.Versioning = &task.VersioningOptions{ + Version: "1.0", + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureFail, + } + registry := task.NewTaskRegistry() + targetName := "DTSTraceScheduled" + uuid.NewString()[:8] + require.NoError(t, registry.AddOrchestratorNVersion(targetName, "1.0", func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output string + if err := ctx.CallActivity("DTSTraceScheduledActivity", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil + })) + require.NoError(t, registry.AddActivityNVersion("DTSTraceScheduledActivity", "1.0", func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input + "-done", nil + })) + require.NoError(t, durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion(registry, "1.0")) + + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + logger, + durabletaskscheduler.WithScheduledTasks(), + durabletaskclient.WithAutoWorkItemFilters(), + ) + require.NoError(t, err) + require.NoError(t, worker.Start(context.Background())) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + require.NoError(t, worker.Shutdown(shutdownCtx)) + require.NoError(t, managementClient.Close()) + }) + + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + createdFrom := time.Now().UTC().Add(-time.Second) + scheduleID := "go-trace-schedule-" + uuid.NewString() + scheduleTag := "trace-" + uuid.NewString() + + callerCtx, callerSpan := startDTSCallerSpan(t, ctx, "caller/dts-scheduled-task") + handle, err := managementClient.ScheduledTasks().Create(callerCtx, durabletaskscheduler.ScheduleCreationOptions{ + ScheduleID: scheduleID, + OrchestrationName: targetName, + TypedOrchestrationInput: "scheduled", + Interval: 2 * time.Second, + StartImmediatelyIfLate: true, + Tags: map[string]string{"source": scheduleTag}, + }) + require.NoError(t, err) + callerSpan.End() + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cleanupCancel() + _ = handle.Delete(cleanupCtx) + }) + + var target *api.OrchestrationMetadata + deadline := time.Now().Add(60 * time.Second) + for target == nil && time.Now().Before(deadline) { + result, queryErr := managementClient.QueryInstances(ctx, api.OrchestrationQuery{ + CreatedTimeFrom: createdFrom, + PageSize: 100, + }) + if queryErr == nil { + for _, orchestration := range result.Orchestrations { + if orchestration.Name == targetName && + orchestration.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED && + orchestration.Tags["source"] == scheduleTag { + target = orchestration + break + } + } + } + if target == nil { + time.Sleep(200 * time.Millisecond) + } + } + require.NotNil(t, target, "the scheduled orchestration did not complete") + + events := fetchHistory(t, ctx, managementClient, target.InstanceID, target.ExecutionID) + started := executionStarted(t, events) + require.Equal(t, "1.0", started.Version) + require.Equal(t, 1, countEvents(events, api.HistoryEventTaskScheduled)) + + // Emulator limitation: schedule-triggered runs are started by the schedule entity, not + // by a client call, so they do not inherit the trace context of whoever created the + // schedule. Each run is its own trace root. Assert that shape explicitly, and assert + // that whatever trace context the run does get is used consistently across its history. + if started.ParentTraceContext != nil && started.ParentTraceContext.TraceParent != "" { + require.NotEqual(t, expectedTraceParent(callerSpan), started.ParentTraceContext.TraceParent, + "schedule-triggered runs are not expected to inherit the schedule creator's span") + traceID := traceIDFromTraceParent(t, started.ParentTraceContext.TraceParent) + requireTraceContextsShareTrace(t, events, traceID) + } + + requireNoWorkerEmittedSpans(t, exporter) +} + +// traceIDFromTraceParent extracts the trace ID from a W3C traceparent header. +func traceIDFromTraceParent(t *testing.T, traceParent string) string { + t.Helper() + require.Len(t, traceParent, 55, "unexpected traceparent format %q", traceParent) + return traceParent[3:35] +} diff --git a/tests/entity_executor_test.go b/tests/entity_executor_test.go new file mode 100644 index 00000000..41365191 --- /dev/null +++ b/tests/entity_executor_test.go @@ -0,0 +1,865 @@ +package tests + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/structpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func newEntityExecutor(r *task.TaskRegistry) task.EntityExecutor { + return task.NewTaskExecutor(r).(task.EntityExecutor) +} + +func Test_Executor_EntityActionsInheritOperationTraceContext(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("router", func(ctx *task.EntityContext) (any, error) { + if err := ctx.SignalEntity(api.NewEntityID("target", "one"), "signal", nil); err != nil { + return nil, err + } + return nil, ctx.StartNewOrchestration( + "child", + task.WithEntityStartOrchestrationInstanceID("child-instance"), + ) + })) + parent := &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + TraceState: wrapperspb.String("vendor=value"), + } + result, err := newEntityExecutor(registry).ExecuteEntity( + context.Background(), + &protos.EntityBatchRequest{ + InstanceId: "@router@key", + Operations: []*protos.OperationRequest{{ + Operation: "route", + RequestId: "request", + TraceContext: parent, + }}, + }, + ) + require.NoError(t, err) + require.Len(t, result.Actions, 2) + require.Equal(t, parent, result.Actions[0].GetSendSignal().GetParentTraceContext()) + require.Equal(t, parent, result.Actions[1].GetStartNewOrchestration().GetParentTraceContext()) + require.NotSame(t, parent, result.Actions[0].GetSendSignal().GetParentTraceContext()) + require.NotSame(t, parent, result.Actions[1].GetStartNewOrchestration().GetParentTraceContext()) +} + +func Test_Executor_EntityBasicOperation(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var count int + if ctx.HasState() { + if err := ctx.GetState(&count); err != nil { + return nil, err + } + } + + switch ctx.Operation { + case "add": + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + count += amount + case "get": + return count, ctx.SetState(count) + case "reset": + count = 0 + } + + if err := ctx.SetState(count); err != nil { + return nil, err + } + return count, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + // Test "add" operation with no initial state + req := &protos.EntityBatchRequest{ + InstanceId: "@counter@myCounter", + Operations: []*protos.OperationRequest{ + { + Operation: "add", + RequestId: "req1", + Input: wrapperspb.String("5"), + }, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, "5", result.Results[0].GetSuccess().GetResult().GetValue()) + assert.Equal(t, "5", result.EntityState.GetValue()) + + // Test "add" again with existing state + req2 := &protos.EntityBatchRequest{ + InstanceId: "@counter@myCounter", + EntityState: result.EntityState, + Operations: []*protos.OperationRequest{ + { + Operation: "add", + RequestId: "req2", + Input: wrapperspb.String("3"), + }, + }, + } + + result2, err := executor.ExecuteEntity(entityCtx, req2) + require.NoError(t, err) + require.Len(t, result2.Results, 1) + require.NotNil(t, result2.Results[0].GetSuccess()) + assert.Equal(t, "8", result2.Results[0].GetSuccess().GetResult().GetValue()) + assert.Equal(t, "8", result2.EntityState.GetValue()) + + // Test "get" operation + req3 := &protos.EntityBatchRequest{ + InstanceId: "@counter@myCounter", + EntityState: result2.EntityState, + Operations: []*protos.OperationRequest{ + { + Operation: "get", + RequestId: "req3", + }, + }, + } + + result3, err := executor.ExecuteEntity(entityCtx, req3) + require.NoError(t, err) + require.Len(t, result3.Results, 1) + require.NotNil(t, result3.Results[0].GetSuccess()) + assert.Equal(t, "8", result3.Results[0].GetSuccess().GetResult().GetValue()) +} + +func Test_Executor_EntityBatchOperations(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var count int + if ctx.HasState() { + if err := ctx.GetState(&count); err != nil { + return nil, err + } + } + + switch ctx.Operation { + case "add": + var amount int + if err := ctx.GetInput(&amount); err != nil { + return nil, err + } + count += amount + case "get": + return count, ctx.SetState(count) + } + + if err := ctx.SetState(count); err != nil { + return nil, err + } + return count, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + // Batch multiple operations + req := &protos.EntityBatchRequest{ + InstanceId: "@counter@myCounter", + Operations: []*protos.OperationRequest{ + {Operation: "add", RequestId: "req1", Input: wrapperspb.String("10")}, + {Operation: "add", RequestId: "req2", Input: wrapperspb.String("20")}, + {Operation: "get", RequestId: "req3"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 3) + + // First add: 0 + 10 = 10 + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, "10", result.Results[0].GetSuccess().GetResult().GetValue()) + + // Second add: 10 + 20 = 30 + require.NotNil(t, result.Results[1].GetSuccess()) + assert.Equal(t, "30", result.Results[1].GetSuccess().GetResult().GetValue()) + + // Get: 30 + require.NotNil(t, result.Results[2].GetSuccess()) + assert.Equal(t, "30", result.Results[2].GetSuccess().GetResult().GetValue()) + + // Final state should be 30 + assert.Equal(t, "30", result.EntityState.GetValue()) +} + +func Test_Executor_EntityOperationError(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("faulty", func(ctx *task.EntityContext) (any, error) { + var count int + if ctx.HasState() { + _ = ctx.GetState(&count) + } + + switch ctx.Operation { + case "fail": + return nil, assert.AnError + case "add": + count++ + _ = ctx.SetState(count) + return count, nil + } + return nil, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + // Batch: add, fail, add — the "fail" operation should not affect state + req := &protos.EntityBatchRequest{ + InstanceId: "@faulty@key1", + Operations: []*protos.OperationRequest{ + {Operation: "add", RequestId: "req1"}, + {Operation: "fail", RequestId: "req2"}, + {Operation: "add", RequestId: "req3"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 3) + + // First add succeeds + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, "1", result.Results[0].GetSuccess().GetResult().GetValue()) + + // Second op fails + require.NotNil(t, result.Results[1].GetFailure()) + + // Third add succeeds (state from first op is preserved, failure is rolled back) + require.NotNil(t, result.Results[2].GetSuccess()) + assert.Equal(t, "2", result.Results[2].GetSuccess().GetResult().GetValue()) + + // Final state is 2 + assert.Equal(t, "2", result.EntityState.GetValue()) +} + +func Test_Executor_EntityPanic(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("panicky", func(ctx *task.EntityContext) (any, error) { + panic("oh no!") + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@panicky@key1", + Operations: []*protos.OperationRequest{ + {Operation: "test", RequestId: "req1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetFailure()) + assert.Contains(t, result.Results[0].GetFailure().GetFailureDetails().GetErrorMessage(), "oh no!") +} + +type reflectedPanickyEntity struct{} + +func (*reflectedPanickyEntity) Explode() { + panic("reflected boom") +} + +func Test_Executor_ReflectedEntityPanicPreservesClassificationAndStack(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("panicky", task.NewEntityFor[reflectedPanickyEntity]())) + + result, err := newEntityExecutor(registry).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@panicky@key", + Operations: []*protos.OperationRequest{{Operation: "Explode"}}, + }) + require.NoError(t, err) + failure := result.Results[0].GetFailure().GetFailureDetails() + require.Equal(t, "EntityOperationPanic", failure.ErrorType) + require.Contains(t, failure.ErrorMessage, "reflected boom") + require.Contains(t, failure.GetStackTrace().GetValue(), "reflectedPanickyEntity).Explode") +} + +func Test_Executor_EntityConversionFailuresStayPerOperation(t *testing.T) { + t.Run("malformed state", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var state int + if err := ctx.GetState(&state); err != nil { + return nil, err + } + return state, nil + })) + result, err := newEntityExecutor(registry).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + EntityState: wrapperspb.String("not-json"), + Operations: []*protos.OperationRequest{{Operation: "get"}}, + }) + require.NoError(t, err) + require.Nil(t, result.FailureDetails) + require.NotNil(t, result.Results[0].GetFailure()) + require.Equal(t, "not-json", result.EntityState.GetValue()) + }) + + t.Run("result serialization", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + if ctx.Operation == "bad" { + return make(chan int), nil + } + return 1, nil + })) + result, err := newEntityExecutor(registry).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{ + {Operation: "bad"}, + {Operation: "good"}, + }, + }) + require.NoError(t, err) + require.Nil(t, result.FailureDetails) + require.NotNil(t, result.Results[0].GetFailure()) + require.Equal(t, "1", result.Results[1].GetSuccess().GetResult().GetValue()) + }) + + t.Run("typed nil result", func(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddEntityN("counter", func(*task.EntityContext) (any, error) { + var result *int + return result, nil + })) + result, err := newEntityExecutor(registry).ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "get"}}, + }) + require.NoError(t, err) + require.NotNil(t, result.Results[0].GetSuccess()) + require.Nil(t, result.Results[0].GetSuccess().Result) + }) +} + +func Test_Executor_EntityNotRegistered(t *testing.T) { + r := task.NewTaskRegistry() + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@unknown@key1", + Operations: []*protos.OperationRequest{ + {Operation: "test", RequestId: "req1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetFailure()) + assert.Equal(t, "EntityTaskNotFound", result.Results[0].GetFailure().FailureDetails.ErrorType) + assert.True(t, result.Results[0].GetFailure().FailureDetails.IsNonRetriable) +} + +func Test_Executor_EntitySignalAction(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("sender", func(ctx *task.EntityContext) (any, error) { + return nil, ctx.SignalEntity(api.NewEntityID("receiver", "key2"), "notify", "hello") + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@sender@key1", + Operations: []*protos.OperationRequest{ + {Operation: "send", RequestId: "req1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetSuccess()) + + // Check that a signal action was emitted + require.Len(t, result.Actions, 1) + signal := result.Actions[0].GetSendSignal() + require.NotNil(t, signal) + assert.Equal(t, "@receiver@key2", signal.InstanceId) + assert.Equal(t, "notify", signal.Name) + assert.Equal(t, `"hello"`, signal.Input.GetValue()) +} + +func Test_Executor_EntityDeleteState(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("deletable", func(ctx *task.EntityContext) (any, error) { + switch ctx.Operation { + case "set": + return nil, ctx.SetState(42) + case "delete": + return nil, ctx.SetState(nil) + } + return nil, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + // First set state + req := &protos.EntityBatchRequest{ + InstanceId: "@deletable@key1", + Operations: []*protos.OperationRequest{ + {Operation: "set", RequestId: "req1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + assert.Equal(t, "42", result.EntityState.GetValue()) + + // Then delete state + req2 := &protos.EntityBatchRequest{ + InstanceId: "@deletable@key1", + EntityState: result.EntityState, + Operations: []*protos.OperationRequest{ + {Operation: "delete", RequestId: "req2"}, + }, + } + + result2, err := executor.ExecuteEntity(entityCtx, req2) + require.NoError(t, err) + assert.Nil(t, result2.EntityState) +} + +// Tests that state persists correctly across multiple batch requests +func Test_Executor_EntityStatePersistsAcrossBatches(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { + var count int + if ctx.HasState() { + _ = ctx.GetState(&count) + } + switch ctx.Operation { + case "increment": + count++ + case "get": + // no-op + } + _ = ctx.SetState(count) + return count, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + // Batch 1: increment 3 times + req := &protos.EntityBatchRequest{ + InstanceId: "@counter@persist", + Operations: []*protos.OperationRequest{ + {Operation: "increment", RequestId: "r1"}, + {Operation: "increment", RequestId: "r2"}, + {Operation: "increment", RequestId: "r3"}, + }, + } + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 3) + assert.Equal(t, "3", result.EntityState.GetValue()) + + // Batch 2: use state from batch 1, increment 2 more times + req2 := &protos.EntityBatchRequest{ + InstanceId: "@counter@persist", + EntityState: result.EntityState, + Operations: []*protos.OperationRequest{ + {Operation: "increment", RequestId: "r4"}, + {Operation: "get", RequestId: "r5"}, + }, + } + result2, err := executor.ExecuteEntity(entityCtx, req2) + require.NoError(t, err) + require.Len(t, result2.Results, 2) + // After 4th increment: 4 + assert.Equal(t, "4", result2.Results[0].GetSuccess().GetResult().GetValue()) + // Get returns 4 + assert.Equal(t, "4", result2.Results[1].GetSuccess().GetResult().GetValue()) + assert.Equal(t, "4", result2.EntityState.GetValue()) +} + +// When an operation fails, state rolls back to the last successful commit +func Test_Executor_EntityErrorRollbackInBatch(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("rollback", func(ctx *task.EntityContext) (any, error) { + var val int + if ctx.HasState() { + _ = ctx.GetState(&val) + } + switch ctx.Operation { + case "set": + var newVal int + if err := ctx.GetInput(&newVal); err != nil { + return nil, err + } + val = newVal + _ = ctx.SetState(val) + return val, nil + case "fail_after_set": + val = 999 // modify state... + _ = ctx.SetState(val) + return nil, fmt.Errorf("intentional failure") // ...then fail + case "get": + _ = ctx.SetState(val) + return val, nil + } + return nil, fmt.Errorf("unknown op") + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@rollback@key1", + Operations: []*protos.OperationRequest{ + {Operation: "set", RequestId: "r1", Input: wrapperspb.String("10")}, + {Operation: "fail_after_set", RequestId: "r2"}, // fails, state should rollback + {Operation: "get", RequestId: "r3"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 3) + + // First op succeeds with value 10 + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, "10", result.Results[0].GetSuccess().GetResult().GetValue()) + + // Second op fails + require.NotNil(t, result.Results[1].GetFailure()) + assert.Contains(t, result.Results[1].GetFailure().GetFailureDetails().GetErrorMessage(), "intentional failure") + + // Third op sees state 10 (rolled back from 999) + require.NotNil(t, result.Results[2].GetSuccess()) + assert.Equal(t, "10", result.Results[2].GetSuccess().GetResult().GetValue()) + + // Final state is 10 (not 999) + assert.Equal(t, "10", result.EntityState.GetValue()) +} + +func Test_Executor_EntityWildcardRegistration(t *testing.T) { + r := task.NewTaskRegistry() + // Register a wildcard entity that handles any entity name + require.NoError(t, r.AddEntityN("*", func(ctx *task.EntityContext) (any, error) { + return fmt.Sprintf("handled %s on %s", ctx.Operation, ctx.ID.Name), nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@anything@key1", + Operations: []*protos.OperationRequest{ + {Operation: "test", RequestId: "r1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, `"handled test on anything"`, result.Results[0].GetSuccess().GetResult().GetValue()) +} + +func Test_Executor_EntitySignalAndStartOrchestrationActions(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("coordinator", func(ctx *task.EntityContext) (any, error) { + if ctx.Operation == "notify_all" { + // Signal another entity + _ = ctx.SignalEntity(api.NewEntityID("worker", "w1"), "process", nil) + _ = ctx.SignalEntity(api.NewEntityID("worker", "w2"), "process", nil) + // Start an orchestration + _ = ctx.StartNewOrchestration("CleanupOrchestrator", + task.WithEntityStartOrchestrationInstanceID("cleanup-1"), + task.WithEntityStartOrchestrationInput("batch-42"), + ) + return "notified", nil + } + return nil, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@coordinator@main", + Operations: []*protos.OperationRequest{ + {Operation: "notify_all", RequestId: "r1"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetSuccess()) + + // Should have 3 actions: 2 signals + 1 start orchestration + require.Len(t, result.Actions, 3) + + signal1 := result.Actions[0].GetSendSignal() + require.NotNil(t, signal1) + assert.Equal(t, "@worker@w1", signal1.InstanceId) + assert.Equal(t, "process", signal1.Name) + + signal2 := result.Actions[1].GetSendSignal() + require.NotNil(t, signal2) + assert.Equal(t, "@worker@w2", signal2.InstanceId) + + startOrch := result.Actions[2].GetStartNewOrchestration() + require.NotNil(t, startOrch) + assert.Equal(t, "CleanupOrchestrator", startOrch.Name) + assert.Equal(t, "cleanup-1", startOrch.InstanceId) + assert.Equal(t, `"batch-42"`, startOrch.Input.GetValue()) +} + +func Test_Executor_EntityComplexState(t *testing.T) { + type item struct { + Name string `json:"name"` + Price int `json:"price"` + } + type cart struct { + Items []item `json:"items"` + Total int `json:"total"` + } + + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("cart", func(ctx *task.EntityContext) (any, error) { + var state cart + if ctx.HasState() { + _ = ctx.GetState(&state) + } + switch ctx.Operation { + case "add_item": + var i item + if err := ctx.GetInput(&i); err != nil { + return nil, err + } + state.Items = append(state.Items, i) + state.Total += i.Price + _ = ctx.SetState(state) + return len(state.Items), nil + case "get": + _ = ctx.SetState(state) + return state, nil + case "clear": + _ = ctx.SetState(nil) + return nil, nil + } + return nil, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@cart@user1", + Operations: []*protos.OperationRequest{ + {Operation: "add_item", RequestId: "r1", Input: wrapperspb.String(`{"name":"apple","price":3}`)}, + {Operation: "add_item", RequestId: "r2", Input: wrapperspb.String(`{"name":"banana","price":2}`)}, + {Operation: "get", RequestId: "r3"}, + }, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + require.Len(t, result.Results, 3) + + // 1 item after first add + assert.Equal(t, "1", result.Results[0].GetSuccess().GetResult().GetValue()) + // 2 items after second add + assert.Equal(t, "2", result.Results[1].GetSuccess().GetResult().GetValue()) + // Get returns full cart + getResult := result.Results[2].GetSuccess().GetResult().GetValue() + assert.Contains(t, getResult, `"apple"`) + assert.Contains(t, getResult, `"banana"`) + assert.Contains(t, getResult, `"total":5`) +} + +func Test_Executor_EntityEmptyBatch(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("noop", func(ctx *task.EntityContext) (any, error) { + return nil, nil + })) + + executor := newEntityExecutor(r) + entityCtx := context.Background() + + req := &protos.EntityBatchRequest{ + InstanceId: "@noop@key1", + Operations: []*protos.OperationRequest{}, + } + + result, err := executor.ExecuteEntity(entityCtx, req) + require.NoError(t, err) + assert.Empty(t, result.Results) + assert.Empty(t, result.Actions) +} + +func Test_Executor_EntityRollbackDiscardsActionsAndReusesDeterministicIDs(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("transactional", func(ctx *task.EntityContext) (any, error) { + target := api.NewEntityID("receiver", ctx.Operation) + if err := ctx.SignalEntity(target, "notify", nil); err != nil { + return nil, err + } + if ctx.Operation == "fail" { + return nil, errors.New("rollback") + } + return ctx.Operation, nil + })) + executor := newEntityExecutor(r) + request := &protos.EntityBatchRequest{ + InstanceId: "@transactional@key", + Operations: []*protos.OperationRequest{ + {Operation: "first", RequestId: uuid.NewString()}, + {Operation: "fail", RequestId: uuid.NewString()}, + {Operation: "third", RequestId: uuid.NewString()}, + }, + } + result, err := executor.ExecuteEntity(context.Background(), request) + require.NoError(t, err) + require.Len(t, result.Actions, 2) + assert.Equal(t, int32(0), result.Actions[0].Id) + assert.Equal(t, int32(1), result.Actions[1].Id) + assert.Equal(t, "@receiver@first", result.Actions[0].GetSendSignal().InstanceId) + assert.Equal(t, "@receiver@third", result.Actions[1].GetSendSignal().InstanceId) + require.NotNil(t, result.Results[1].GetFailure()) +} + +func Test_Executor_EntityStartOrchestrationIDIsStableAcrossRetry(t *testing.T) { + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("starter", func(ctx *task.EntityContext) (any, error) { + return nil, ctx.StartNewOrchestration("child") + })) + executor := newEntityExecutor(r) + request := &protos.EntityBatchRequest{ + InstanceId: "@starter@key", + Operations: []*protos.OperationRequest{{ + Operation: "start", + RequestId: uuid.NewString(), + }}, + } + first, err := executor.ExecuteEntity(context.Background(), request) + require.NoError(t, err) + second, err := executor.ExecuteEntity(context.Background(), request) + require.NoError(t, err) + require.Len(t, first.Actions, 1) + require.Len(t, second.Actions, 1) + assert.Equal( + t, + first.Actions[0].GetStartNewOrchestration().InstanceId, + second.Actions[0].GetStartNewOrchestration().InstanceId, + ) +} + +// The scheduler signals an elided entity state with the "IncludeState" property, +// matching Microsoft.DurableTask's GrpcInstanceRunnerUtils. +func Test_Executor_EntityStateElisionRequestsState(t *testing.T) { + var invoked bool + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("cached", func(*task.EntityContext) (any, error) { + invoked = true + return nil, nil + })) + executor := newEntityExecutor(r) + result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@cached@key", + Operations: []*protos.OperationRequest{{ + Operation: "get", + RequestId: uuid.NewString(), + }}, + Properties: map[string]*structpb.Value{ + "IncludeState": structpb.NewBoolValue(false), + }, + }) + require.NoError(t, err) + assert.True(t, result.RequiresState) + assert.False(t, invoked) + assert.Empty(t, result.Results) + assert.Nil(t, result.EntityState) +} + +// A state request is driven by the property alone, so an attached state does not +// suppress it and an unregistered entity does not mask it. +func Test_Executor_EntityStateElisionIgnoresAttachedStateAndRegistration(t *testing.T) { + executor := newEntityExecutor(task.NewTaskRegistry()) + result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ + InstanceId: "@unregistered@key", + EntityState: wrapperspb.String("42"), + Operations: []*protos.OperationRequest{{ + Operation: "get", + RequestId: uuid.NewString(), + }}, + Properties: map[string]*structpb.Value{ + "IncludeState": structpb.NewBoolValue(false), + }, + }) + require.NoError(t, err) + assert.True(t, result.RequiresState) + assert.Empty(t, result.Results) +} + +// A missing or non-boolean property means the state was included. +func Test_Executor_EntityStateElisionDefaultsToIncluded(t *testing.T) { + properties := map[string]map[string]*structpb.Value{ + "absent": nil, + "wrong-type": { + "IncludeState": structpb.NewStringValue("false"), + }, + "misspelled": { + "includestate": structpb.NewBoolValue(false), + }, + "true": { + "IncludeState": structpb.NewBoolValue(true), + }, + } + for name, property := range properties { + t.Run(name, func(t *testing.T) { + var invoked bool + r := task.NewTaskRegistry() + require.NoError(t, r.AddEntityN("cached", func(*task.EntityContext) (any, error) { + invoked = true + return nil, nil + })) + result, err := newEntityExecutor(r).ExecuteEntity( + context.Background(), + &protos.EntityBatchRequest{ + InstanceId: "@cached@key", + Operations: []*protos.OperationRequest{{ + Operation: "get", + RequestId: uuid.NewString(), + }}, + Properties: property, + }, + ) + require.NoError(t, err) + assert.False(t, result.RequiresState) + assert.True(t, invoked) + }) + } +} diff --git a/tests/failurechain/failurechain.go b/tests/failurechain/failurechain.go new file mode 100644 index 00000000..c33ec0f2 --- /dev/null +++ b/tests/failurechain/failurechain.go @@ -0,0 +1,88 @@ +// Package failurechain provides a shared, table-driven assertion for durable +// failure chains, so the Durable Task Scheduler tests validate the same +// cross-language contract the other language SDKs do. +package failurechain + +import ( + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/require" +) + +// LeafError is an application error that opts into every durable failure +// enrichment hook: a stable cross-language error type, a stack trace, custom +// properties, and the non-retriable marker. Tests reuse it so the same leaf +// contract is exercised against a live service. +type LeafError struct { + Message string + ErrorType api.ErrorType + Stack string + Properties map[string]any + // IsNonRetriable backs the NonRetriable marker method, which cannot share + // the field's name. + IsNonRetriable bool +} + +func (e *LeafError) Error() string { return e.Message } + +func (e *LeafError) DurableTaskErrorType() api.ErrorType { return e.ErrorType } + +func (e *LeafError) DurableTaskStackTrace() string { return e.Stack } + +func (e *LeafError) DurableTaskErrorProperties() map[string]any { return e.Properties } + +// NonRetriable reports the marker the durable retry policy consults. +func (e *LeafError) NonRetriable() bool { return e.IsNonRetriable } + +var ( + _ api.DurableTaskErrorTypeProvider = (*LeafError)(nil) + _ api.DurableTaskStackTraceProvider = (*LeafError)(nil) + _ api.DurableTaskErrorPropertiesProvider = (*LeafError)(nil) + _ api.NonRetriable = (*LeafError)(nil) +) + +// Frame describes one expected frame of a durable failure chain. +type Frame struct { + ErrorType api.ErrorType + MessageContains []string + // StackContains is only checked when ExpectStack is true. + StackContains string + ExpectStack bool + NonRetriable bool + // Properties are checked by key. A nil map asserts the frame carries no + // properties at all. + Properties map[string]any +} + +// Assert walks a failure chain and asserts it matches frames exactly, including +// that the chain is neither shorter nor deeper than expected. +func Assert(t *testing.T, details *api.FailureDetails, frames []Frame) { + t.Helper() + current := details + for i, frame := range frames { + require.NotNilf(t, current, "failure chain ended before frame %d", i) + require.Equalf(t, frame.ErrorType, current.ErrorType, "frame %d error type", i) + for _, fragment := range frame.MessageContains { + require.Containsf(t, current.ErrorMessage, fragment, "frame %d message", i) + } + require.Equalf(t, frame.NonRetriable, current.IsNonRetriable, "frame %d non-retriable flag", i) + if frame.ExpectStack { + require.NotEmptyf(t, current.StackTrace, "frame %d stack trace", i) + if frame.StackContains != "" { + require.Containsf(t, current.StackTrace, frame.StackContains, "frame %d stack trace", i) + } + } else { + require.Emptyf(t, current.StackTrace, "frame %d stack trace", i) + } + if frame.Properties == nil { + require.Emptyf(t, current.Properties, "frame %d properties", i) + } else { + for key, want := range frame.Properties { + require.Equalf(t, want, current.Properties[key], "frame %d property %q", i, key) + } + } + current = current.InnerFailure + } + require.Nil(t, current, "failure chain is deeper than expected") +} diff --git a/tests/grpc/grpc_test.go b/tests/grpc/grpc_test.go deleted file mode 100644 index e5bf6d35..00000000 --- a/tests/grpc/grpc_test.go +++ /dev/null @@ -1,510 +0,0 @@ -package tests_grpc - -import ( - "context" - "errors" - "fmt" - "log" - "net" - "os" - "testing" - "time" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" - "github.com/microsoft/durabletask-go/client" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/microsoft/durabletask-go/task" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - "google.golang.org/grpc/credentials/insecure" -) - -var ( - grpcClient *client.TaskHubGrpcClient - ctx = context.Background() -) - -// TestMain is the entry point for the test suite. We use this to set up a gRPC server and client instance -// which are used by all tests in the suite. -func TestMain(m *testing.M) { - sqliteOptions := sqlite.NewSqliteOptions("") - logger := backend.DefaultLogger() - be := sqlite.NewSqliteBackend(sqliteOptions, logger) - grpcServer := grpc.NewServer() - grpcExecutor, registerFn := backend.NewGrpcExecutor(be, logger) - registerFn(grpcServer) - orchestrationWorker := backend.NewOrchestrationWorker(be, grpcExecutor, logger) - activityWorker := backend.NewActivityTaskWorker(be, grpcExecutor, logger) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - if err := taskHubWorker.Start(ctx); err != nil { - log.Fatalf("failed to start worker: %v", err) - } - - lis, err := net.Listen("tcp", ":0") - if err != nil { - log.Fatalf("failed to listen: %v", err) - } - - go func() { - if err := grpcServer.Serve(lis); err != nil { - log.Fatalf("failed to serve: %v", err) - } - fmt.Printf("server listening at %v\n", lis.Addr()) - }() - - time.Sleep(1 * time.Second) - - conn, err := grpc.Dial(lis.Addr().String(), grpc.WithTransportCredentials(insecure.NewCredentials())) - if err != nil { - log.Fatalf("failed to connect to gRPC server: %v", err) - } - defer func() { - if err := conn.Close(); err != nil { - log.Printf("failed to close connection: %v", err) - } - }() - grpcClient = client.NewTaskHubGrpcClient(conn, logger) - - // Run the test exitCode - exitCode := m.Run() - - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - err = grpcExecutor.Shutdown(timeoutCtx) - if err != nil { - log.Printf("failed to shutdown grpc Executor: %v", err) - } - - timeoutCtx, cancel = context.WithTimeout(ctx, 5*time.Second) - defer cancel() - if err := taskHubWorker.Shutdown(timeoutCtx); err != nil { - log.Printf("failed to shutdown worker: %v", err) - } - grpcServer.Stop() - os.Exit(exitCode) //nolint:gocritic // os.Exit in TestMain is required by Go testing -} - -func startGrpcListener(t *testing.T, r *task.TaskRegistry) context.CancelFunc { - cancelCtx, cancel := context.WithCancel(ctx) - require.NoError(t, grpcClient.StartWorkItemListener(cancelCtx, r)) - return cancel -} - -func Test_Grpc_WaitForInstanceStart_Timeout(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("WaitForInstanceStartThrowsException", func(ctx *task.OrchestrationContext) (any, error) { - // sleep 5 seconds - time.Sleep(5 * time.Second) - return 42, nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "WaitForInstanceStartThrowsException", api.WithInput("世界")) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, time.Second) - defer cancelTimeout() - _, err = grpcClient.WaitForOrchestrationStart(timeoutCtx, id, api.WithFetchPayloads(true)) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "context deadline exceeded") - } - time.Sleep(1 * time.Second) -} - -func Test_Grpc_WaitForInstanceStart_ConnectionResume(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("WaitForInstanceStartThrowsException", func(ctx *task.OrchestrationContext) (any, error) { - // sleep 5 seconds - time.Sleep(5 * time.Second) - return 42, nil - })) - - cancelListener := startGrpcListener(t, r) - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "WaitForInstanceStartThrowsException", api.WithInput("世界")) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, time.Second) - defer cancelTimeout() - _, err = grpcClient.WaitForOrchestrationStart(timeoutCtx, id, api.WithFetchPayloads(true)) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "context deadline exceeded") - } - cancelListener() - time.Sleep(2 * time.Second) - - // reconnect - cancelListener = startGrpcListener(t, r) - defer cancelListener() - - // workitem should be retried and completed. - timeoutCtx, cancelTimeout = context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - assert.Equal(t, "42", metadata.SerializedOutput) - time.Sleep(1 * time.Second) -} - -func Test_Grpc_HelloOrchestration(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - ctx.SetCustomStatus("hello-test") - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界")) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - assert.Equal(t, `"Hello, 世界!"`, metadata.SerializedOutput) - assert.Equal(t, "hello-test", metadata.SerializedCustomStatus) - time.Sleep(1 * time.Second) -} - -func Test_Grpc_SuspendResume(t *testing.T) { - const eventCount = 10 - - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SuspendResumeOrchestration", func(ctx *task.OrchestrationContext) (any, error) { - for i := 0; i < eventCount; i++ { - var value int - if err := ctx.WaitForSingleEvent("MyEvent", 5*time.Second).Await(&value); err != nil { - return false, err - } - if value != i { - return false, errors.New("Unexpected value") - } - } - return true, nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - - // Run the orchestration, which will block waiting for external events - id, err := grpcClient.ScheduleNewOrchestration(ctx, "SuspendResumeOrchestration", api.WithInput(0)) - require.NoError(t, err) - - // Suspend the orchestration - require.NoError(t, grpcClient.SuspendOrchestration(ctx, id, "")) - - // Raise a bunch of events to the orchestration (they should get buffered but not consumed) - for i := 0; i < eventCount; i++ { - opts := api.WithEventPayload(i) - require.NoError(t, grpcClient.RaiseEvent(ctx, id, "MyEvent", opts)) - } - - // Make sure the orchestration *doesn't* complete - timeoutCtx, cancelWait := context.WithTimeout(ctx, 3*time.Second) - defer cancelWait() - _, err = grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id) - require.ErrorIs(t, err, timeoutCtx.Err()) - - var metadata *api.OrchestrationMetadata - metadata, err = grpcClient.FetchOrchestrationMetadata(ctx, id) - require.NoError(t, err) - require.True(t, metadata.IsRunning()) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, metadata.RuntimeStatus) - - // Resume the orchestration and wait for it to complete - require.NoError(t, grpcClient.ResumeOrchestration(ctx, id, "")) - timeoutCtx, cancelWait = context.WithTimeout(ctx, 3*time.Second) - defer cancelWait() - _, err = grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - time.Sleep(1 * time.Second) -} - -func Test_Grpc_Terminate_Recursive(t *testing.T) { - delayTime := 4 * time.Second - executedActivity := false - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Root", func(ctx *task.OrchestrationContext) (any, error) { - tasks := []task.Task{} - for i := 0; i < 5; i++ { - task := ctx.CallSubOrchestrator("L1") - tasks = append(tasks, task) - } - for _, task := range tasks { - if err := task.Await(nil); err != nil { - return nil, err - } - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L1", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallSubOrchestrator("L2").Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L2", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - if err := ctx.CallActivity("Fail").Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddActivityN("Fail", func(ctx task.ActivityContext) (any, error) { - executedActivity = true - return nil, errors.New("Failed: Should not have executed the activity") - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - - // Test terminating with and without recursion - for _, recurse := range []bool{true, false} { - t.Run(fmt.Sprintf("Recurse = %v", recurse), func(t *testing.T) { - // Run the orchestration, which will block waiting for external events - id, err := grpcClient.ScheduleNewOrchestration(ctx, "Root") - require.NoError(t, err) - - // Wait long enough to ensure all orchestrations have started (but not longer than the timer delay) - time.Sleep(2 * time.Second) - - // Terminate the root orchestration and mark whether a recursive termination - output := fmt.Sprintf("Recursive termination = %v", recurse) - opts := []api.TerminateOptions{api.WithOutput(output), api.WithRecursiveTerminate(recurse)} - require.NoError(t, grpcClient.TerminateOrchestration(ctx, id, opts...)) - - // Wait for the root orchestration to complete and verify its terminated status - metadata, err := grpcClient.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, metadata.RuntimeStatus) - require.Equal(t, fmt.Sprintf("\"%s\"", output), metadata.SerializedOutput) - - // Wait longer to ensure that none of the sub-orchestrations continued to the next step - // of executing the activity function. - time.Sleep(delayTime) - assert.NotEqual(t, recurse, executedActivity) - }) - } -} - -func Test_Grpc_ReuseInstanceIDIgnore(t *testing.T) { - delayTime := 2 * time.Second - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - instanceID := api.InstanceID("SKIP_IF_RUNNING_OR_COMPLETED") - reuseIdPolicy := &api.OrchestrationIdReusePolicy{ - Action: api.REUSE_ID_ACTION_IGNORE, - OperationStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_COMPLETED, api.RUNTIME_STATUS_PENDING}, - } - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - // wait orchestration to start - _, err = grpcClient.WaitForOrchestrationStart(ctx, id) - require.NoError(t, err) - pivotTime := time.Now() - // schedule again, it should ignore creating the new orchestration - id, err = grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(id), api.WithOrchestrationIdReusePolicy(reuseIdPolicy)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - // the first orchestration should complete as the second one is ignored - assert.Equal(t, `"Hello, 世界!"`, metadata.SerializedOutput) - // assert the orchestration created timestamp - assert.True(t, pivotTime.After(metadata.CreatedAt)) -} - -func Test_Grpc_ReuseInstanceIDTerminate(t *testing.T) { - delayTime := 2 * time.Second - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - instanceID := api.InstanceID("TERMINATE_IF_RUNNING_OR_COMPLETED") - reuseIdPolicy := &api.OrchestrationIdReusePolicy{ - Action: api.REUSE_ID_ACTION_TERMINATE, - OperationStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_COMPLETED, api.RUNTIME_STATUS_PENDING}, - } - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - // wait orchestration to start - _, err = grpcClient.WaitForOrchestrationStart(ctx, id) - require.NoError(t, err) - pivotTime := time.Now() - // schedule again, it should terminate the first orchestration and start a new one - id, err = grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(id), api.WithOrchestrationIdReusePolicy(reuseIdPolicy)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - // the second orchestration should complete. - assert.Equal(t, `"Hello, World!"`, metadata.SerializedOutput) - // assert the orchestration created timestamp - assert.True(t, pivotTime.Before(metadata.CreatedAt)) -} - -func Test_Grpc_ReuseInstanceIDError(t *testing.T) { - delayTime := 4 * time.Second - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - instanceID := api.InstanceID("THROW_IF_RUNNING_OR_COMPLETED") - - _, err := grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - _, err = grpcClient.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(instanceID)) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "orchestration instance already exists") - } -} - -func Test_Grpc_ActivityRetries(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ActivityRetries", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity("FailActivity", task.WithActivityRetryPolicy(&task.RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 10 * time.Millisecond, - })).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddActivityN("FailActivity", func(ctx task.ActivityContext) (any, error) { - return nil, errors.New("activity failure") - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - instanceID := api.InstanceID("activity_retries") - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "ActivityRetries", api.WithInstanceID(instanceID)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - // With 3 max attempts there will be two retries with 10 millis delay before each - require.GreaterOrEqual(t, metadata.LastUpdatedAt, metadata.CreatedAt.Add(2*10*time.Millisecond)) -} - -func Test_Grpc_SubOrchestratorRetries(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Parent", func(ctx *task.OrchestrationContext) (any, error) { - err := ctx.CallSubOrchestrator( - "Child", - task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_child"), - task.WithSubOrchestrationRetryPolicy(&task.RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 10 * time.Millisecond, - BackoffCoefficient: 2, - })).Await(nil) - return nil, err - })) - require.NoError(t, r.AddOrchestratorN("Child", func(ctx *task.OrchestrationContext) (any, error) { - return nil, errors.New("Child failed") - })) - - cancelListener := startGrpcListener(t, r) - defer cancelListener() - instanceID := api.InstanceID("orchestrator_retries") - - id, err := grpcClient.ScheduleNewOrchestration(ctx, "Parent", api.WithInstanceID(instanceID)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := grpcClient.WaitForOrchestrationCompletion(timeoutCtx, id, api.WithFetchPayloads(true)) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - // With 3 max attempts there will be two retries with 10 millis delay before each - require.GreaterOrEqual(t, metadata.LastUpdatedAt, metadata.CreatedAt.Add(2*10*time.Millisecond)) -} diff --git a/tests/metadata_test.go b/tests/metadata_test.go index b9f1e76d..2c57a8c5 100644 --- a/tests/metadata_test.go +++ b/tests/metadata_test.go @@ -8,7 +8,6 @@ import ( "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/internal/protos" "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/types/known/wrapperspb" ) func Test_OrchestrationMetadata_Serialization(t *testing.T) { @@ -21,11 +20,13 @@ func Test_OrchestrationMetadata_Serialization(t *testing.T) { "\"World\"", "\"Hello, World!\"", "", - &protos.TaskFailureDetails{ - ErrorType: "MyError", - ErrorMessage: "Kah-BOOOOM!!!", - StackTrace: wrapperspb.String("stack trace"), - InnerFailure: &protos.TaskFailureDetails{ + &api.FailureDetails{ + ErrorType: "MyError", + ErrorMessage: "Kah-BOOOOM!!!", + StackTrace: "stack trace", + IsNonRetriable: true, + Properties: map[string]any{"attempt": float64(2)}, + InnerFailure: &api.FailureDetails{ ErrorType: "InnerError", ErrorMessage: "Fuse lit", }, @@ -45,11 +46,13 @@ func Test_OrchestrationMetadata_Serialization(t *testing.T) { if assert.NotNil(t, metadata2.FailureDetails) { assert.Equal(t, metadata.FailureDetails.ErrorType, metadata2.FailureDetails.ErrorType) assert.Equal(t, metadata.FailureDetails.ErrorMessage, metadata2.FailureDetails.ErrorMessage) - assert.Equal(t, metadata.FailureDetails.StackTrace.GetValue(), metadata2.FailureDetails.StackTrace.GetValue()) + assert.Equal(t, metadata.FailureDetails.StackTrace, metadata2.FailureDetails.StackTrace) + assert.Equal(t, metadata.FailureDetails.IsNonRetriable, metadata2.FailureDetails.IsNonRetriable) + assert.Equal(t, metadata.FailureDetails.Properties, metadata2.FailureDetails.Properties) if assert.NotNil(t, metadata2.FailureDetails.InnerFailure) { assert.Equal(t, metadata.FailureDetails.InnerFailure.ErrorType, metadata2.FailureDetails.InnerFailure.ErrorType) assert.Equal(t, metadata.FailureDetails.InnerFailure.ErrorMessage, metadata2.FailureDetails.InnerFailure.ErrorMessage) - assert.Nil(t, metadata2.FailureDetails.InnerFailure.StackTrace) + assert.Empty(t, metadata2.FailureDetails.InnerFailure.StackTrace) assert.Nil(t, metadata2.FailureDetails.InnerFailure.InnerFailure) } } diff --git a/tests/mocks/Backend.go b/tests/mocks/Backend.go deleted file mode 100644 index a59f0f1b..00000000 --- a/tests/mocks/Backend.go +++ /dev/null @@ -1,804 +0,0 @@ -// Code generated by mockery v2.38.0. DO NOT EDIT. - -package mocks - -import ( - api "github.com/microsoft/durabletask-go/api" - backend "github.com/microsoft/durabletask-go/backend" - - context "context" - - mock "github.com/stretchr/testify/mock" - - protos "github.com/microsoft/durabletask-go/internal/protos" -) - -// Backend is an autogenerated mock type for the Backend type -type Backend struct { - mock.Mock -} - -type Backend_Expecter struct { - mock *mock.Mock -} - -func (_m *Backend) EXPECT() *Backend_Expecter { - return &Backend_Expecter{mock: &_m.Mock} -} - -// AbandonActivityWorkItem provides a mock function with given fields: _a0, _a1 -func (_m *Backend) AbandonActivityWorkItem(_a0 context.Context, _a1 *backend.ActivityWorkItem) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for AbandonActivityWorkItem") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *backend.ActivityWorkItem) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_AbandonActivityWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonActivityWorkItem' -type Backend_AbandonActivityWorkItem_Call struct { - *mock.Call -} - -// AbandonActivityWorkItem is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *backend.ActivityWorkItem -func (_e *Backend_Expecter) AbandonActivityWorkItem(_a0 interface{}, _a1 interface{}) *Backend_AbandonActivityWorkItem_Call { - return &Backend_AbandonActivityWorkItem_Call{Call: _e.mock.On("AbandonActivityWorkItem", _a0, _a1)} -} - -func (_c *Backend_AbandonActivityWorkItem_Call) Run(run func(_a0 context.Context, _a1 *backend.ActivityWorkItem)) *Backend_AbandonActivityWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*backend.ActivityWorkItem)) - }) - return _c -} - -func (_c *Backend_AbandonActivityWorkItem_Call) Return(_a0 error) *Backend_AbandonActivityWorkItem_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_AbandonActivityWorkItem_Call) RunAndReturn(run func(context.Context, *backend.ActivityWorkItem) error) *Backend_AbandonActivityWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// AbandonOrchestrationWorkItem provides a mock function with given fields: _a0, _a1 -func (_m *Backend) AbandonOrchestrationWorkItem(_a0 context.Context, _a1 *backend.OrchestrationWorkItem) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for AbandonOrchestrationWorkItem") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *backend.OrchestrationWorkItem) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_AbandonOrchestrationWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AbandonOrchestrationWorkItem' -type Backend_AbandonOrchestrationWorkItem_Call struct { - *mock.Call -} - -// AbandonOrchestrationWorkItem is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *backend.OrchestrationWorkItem -func (_e *Backend_Expecter) AbandonOrchestrationWorkItem(_a0 interface{}, _a1 interface{}) *Backend_AbandonOrchestrationWorkItem_Call { - return &Backend_AbandonOrchestrationWorkItem_Call{Call: _e.mock.On("AbandonOrchestrationWorkItem", _a0, _a1)} -} - -func (_c *Backend_AbandonOrchestrationWorkItem_Call) Run(run func(_a0 context.Context, _a1 *backend.OrchestrationWorkItem)) *Backend_AbandonOrchestrationWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*backend.OrchestrationWorkItem)) - }) - return _c -} - -func (_c *Backend_AbandonOrchestrationWorkItem_Call) Return(_a0 error) *Backend_AbandonOrchestrationWorkItem_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_AbandonOrchestrationWorkItem_Call) RunAndReturn(run func(context.Context, *backend.OrchestrationWorkItem) error) *Backend_AbandonOrchestrationWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// AddNewOrchestrationEvent provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Backend) AddNewOrchestrationEvent(_a0 context.Context, _a1 api.InstanceID, _a2 *protos.HistoryEvent) error { - ret := _m.Called(_a0, _a1, _a2) - - if len(ret) == 0 { - panic("no return value specified for AddNewOrchestrationEvent") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID, *protos.HistoryEvent) error); ok { - r0 = rf(_a0, _a1, _a2) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_AddNewOrchestrationEvent_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AddNewOrchestrationEvent' -type Backend_AddNewOrchestrationEvent_Call struct { - *mock.Call -} - -// AddNewOrchestrationEvent is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 api.InstanceID -// - _a2 *protos.HistoryEvent -func (_e *Backend_Expecter) AddNewOrchestrationEvent(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Backend_AddNewOrchestrationEvent_Call { - return &Backend_AddNewOrchestrationEvent_Call{Call: _e.mock.On("AddNewOrchestrationEvent", _a0, _a1, _a2)} -} - -func (_c *Backend_AddNewOrchestrationEvent_Call) Run(run func(_a0 context.Context, _a1 api.InstanceID, _a2 *protos.HistoryEvent)) *Backend_AddNewOrchestrationEvent_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(api.InstanceID), args[2].(*protos.HistoryEvent)) - }) - return _c -} - -func (_c *Backend_AddNewOrchestrationEvent_Call) Return(_a0 error) *Backend_AddNewOrchestrationEvent_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_AddNewOrchestrationEvent_Call) RunAndReturn(run func(context.Context, api.InstanceID, *protos.HistoryEvent) error) *Backend_AddNewOrchestrationEvent_Call { - _c.Call.Return(run) - return _c -} - -// CompleteActivityWorkItem provides a mock function with given fields: _a0, _a1 -func (_m *Backend) CompleteActivityWorkItem(_a0 context.Context, _a1 *backend.ActivityWorkItem) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for CompleteActivityWorkItem") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *backend.ActivityWorkItem) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_CompleteActivityWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompleteActivityWorkItem' -type Backend_CompleteActivityWorkItem_Call struct { - *mock.Call -} - -// CompleteActivityWorkItem is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *backend.ActivityWorkItem -func (_e *Backend_Expecter) CompleteActivityWorkItem(_a0 interface{}, _a1 interface{}) *Backend_CompleteActivityWorkItem_Call { - return &Backend_CompleteActivityWorkItem_Call{Call: _e.mock.On("CompleteActivityWorkItem", _a0, _a1)} -} - -func (_c *Backend_CompleteActivityWorkItem_Call) Run(run func(_a0 context.Context, _a1 *backend.ActivityWorkItem)) *Backend_CompleteActivityWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*backend.ActivityWorkItem)) - }) - return _c -} - -func (_c *Backend_CompleteActivityWorkItem_Call) Return(_a0 error) *Backend_CompleteActivityWorkItem_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_CompleteActivityWorkItem_Call) RunAndReturn(run func(context.Context, *backend.ActivityWorkItem) error) *Backend_CompleteActivityWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// CompleteOrchestrationWorkItem provides a mock function with given fields: _a0, _a1 -func (_m *Backend) CompleteOrchestrationWorkItem(_a0 context.Context, _a1 *backend.OrchestrationWorkItem) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for CompleteOrchestrationWorkItem") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *backend.OrchestrationWorkItem) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_CompleteOrchestrationWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CompleteOrchestrationWorkItem' -type Backend_CompleteOrchestrationWorkItem_Call struct { - *mock.Call -} - -// CompleteOrchestrationWorkItem is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *backend.OrchestrationWorkItem -func (_e *Backend_Expecter) CompleteOrchestrationWorkItem(_a0 interface{}, _a1 interface{}) *Backend_CompleteOrchestrationWorkItem_Call { - return &Backend_CompleteOrchestrationWorkItem_Call{Call: _e.mock.On("CompleteOrchestrationWorkItem", _a0, _a1)} -} - -func (_c *Backend_CompleteOrchestrationWorkItem_Call) Run(run func(_a0 context.Context, _a1 *backend.OrchestrationWorkItem)) *Backend_CompleteOrchestrationWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*backend.OrchestrationWorkItem)) - }) - return _c -} - -func (_c *Backend_CompleteOrchestrationWorkItem_Call) Return(_a0 error) *Backend_CompleteOrchestrationWorkItem_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_CompleteOrchestrationWorkItem_Call) RunAndReturn(run func(context.Context, *backend.OrchestrationWorkItem) error) *Backend_CompleteOrchestrationWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// CreateOrchestrationInstance provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Backend) CreateOrchestrationInstance(_a0 context.Context, _a1 *protos.HistoryEvent, _a2 ...backend.OrchestrationIdReusePolicyOptions) error { - _va := make([]interface{}, len(_a2)) - for _i := range _a2 { - _va[_i] = _a2[_i] - } - var _ca []interface{} - _ca = append(_ca, _a0, _a1) - _ca = append(_ca, _va...) - ret := _m.Called(_ca...) - - if len(ret) == 0 { - panic("no return value specified for CreateOrchestrationInstance") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, *protos.HistoryEvent, ...backend.OrchestrationIdReusePolicyOptions) error); ok { - r0 = rf(_a0, _a1, _a2...) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_CreateOrchestrationInstance_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateOrchestrationInstance' -type Backend_CreateOrchestrationInstance_Call struct { - *mock.Call -} - -// CreateOrchestrationInstance is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *protos.HistoryEvent -// - _a2 ...backend.OrchestrationIdReusePolicyOptions -func (_e *Backend_Expecter) CreateOrchestrationInstance(_a0 interface{}, _a1 interface{}, _a2 ...interface{}) *Backend_CreateOrchestrationInstance_Call { - return &Backend_CreateOrchestrationInstance_Call{Call: _e.mock.On("CreateOrchestrationInstance", - append([]interface{}{_a0, _a1}, _a2...)...)} -} - -func (_c *Backend_CreateOrchestrationInstance_Call) Run(run func(_a0 context.Context, _a1 *protos.HistoryEvent, _a2 ...backend.OrchestrationIdReusePolicyOptions)) *Backend_CreateOrchestrationInstance_Call { - _c.Call.Run(func(args mock.Arguments) { - variadicArgs := make([]backend.OrchestrationIdReusePolicyOptions, len(args)-2) - for i, a := range args[2:] { - if a != nil { - variadicArgs[i] = a.(backend.OrchestrationIdReusePolicyOptions) - } - } - run(args[0].(context.Context), args[1].(*protos.HistoryEvent), variadicArgs...) - }) - return _c -} - -func (_c *Backend_CreateOrchestrationInstance_Call) Return(_a0 error) *Backend_CreateOrchestrationInstance_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_CreateOrchestrationInstance_Call) RunAndReturn(run func(context.Context, *protos.HistoryEvent, ...backend.OrchestrationIdReusePolicyOptions) error) *Backend_CreateOrchestrationInstance_Call { - _c.Call.Return(run) - return _c -} - -// CreateTaskHub provides a mock function with given fields: _a0 -func (_m *Backend) CreateTaskHub(_a0 context.Context) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for CreateTaskHub") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_CreateTaskHub_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateTaskHub' -type Backend_CreateTaskHub_Call struct { - *mock.Call -} - -// CreateTaskHub is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) CreateTaskHub(_a0 interface{}) *Backend_CreateTaskHub_Call { - return &Backend_CreateTaskHub_Call{Call: _e.mock.On("CreateTaskHub", _a0)} -} - -func (_c *Backend_CreateTaskHub_Call) Run(run func(_a0 context.Context)) *Backend_CreateTaskHub_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_CreateTaskHub_Call) Return(_a0 error) *Backend_CreateTaskHub_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_CreateTaskHub_Call) RunAndReturn(run func(context.Context) error) *Backend_CreateTaskHub_Call { - _c.Call.Return(run) - return _c -} - -// DeleteTaskHub provides a mock function with given fields: _a0 -func (_m *Backend) DeleteTaskHub(_a0 context.Context) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for DeleteTaskHub") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_DeleteTaskHub_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteTaskHub' -type Backend_DeleteTaskHub_Call struct { - *mock.Call -} - -// DeleteTaskHub is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) DeleteTaskHub(_a0 interface{}) *Backend_DeleteTaskHub_Call { - return &Backend_DeleteTaskHub_Call{Call: _e.mock.On("DeleteTaskHub", _a0)} -} - -func (_c *Backend_DeleteTaskHub_Call) Run(run func(_a0 context.Context)) *Backend_DeleteTaskHub_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_DeleteTaskHub_Call) Return(_a0 error) *Backend_DeleteTaskHub_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_DeleteTaskHub_Call) RunAndReturn(run func(context.Context) error) *Backend_DeleteTaskHub_Call { - _c.Call.Return(run) - return _c -} - -// GetActivityWorkItem provides a mock function with given fields: _a0 -func (_m *Backend) GetActivityWorkItem(_a0 context.Context) (*backend.ActivityWorkItem, error) { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for GetActivityWorkItem") - } - - var r0 *backend.ActivityWorkItem - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (*backend.ActivityWorkItem, error)); ok { - return rf(_a0) - } - if rf, ok := ret.Get(0).(func(context.Context) *backend.ActivityWorkItem); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*backend.ActivityWorkItem) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Backend_GetActivityWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetActivityWorkItem' -type Backend_GetActivityWorkItem_Call struct { - *mock.Call -} - -// GetActivityWorkItem is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) GetActivityWorkItem(_a0 interface{}) *Backend_GetActivityWorkItem_Call { - return &Backend_GetActivityWorkItem_Call{Call: _e.mock.On("GetActivityWorkItem", _a0)} -} - -func (_c *Backend_GetActivityWorkItem_Call) Run(run func(_a0 context.Context)) *Backend_GetActivityWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_GetActivityWorkItem_Call) Return(_a0 *backend.ActivityWorkItem, _a1 error) *Backend_GetActivityWorkItem_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Backend_GetActivityWorkItem_Call) RunAndReturn(run func(context.Context) (*backend.ActivityWorkItem, error)) *Backend_GetActivityWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// GetOrchestrationMetadata provides a mock function with given fields: _a0, _a1 -func (_m *Backend) GetOrchestrationMetadata(_a0 context.Context, _a1 api.InstanceID) (*api.OrchestrationMetadata, error) { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for GetOrchestrationMetadata") - } - - var r0 *api.OrchestrationMetadata - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID) (*api.OrchestrationMetadata, error)); ok { - return rf(_a0, _a1) - } - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID) *api.OrchestrationMetadata); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*api.OrchestrationMetadata) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, api.InstanceID) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Backend_GetOrchestrationMetadata_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetOrchestrationMetadata' -type Backend_GetOrchestrationMetadata_Call struct { - *mock.Call -} - -// GetOrchestrationMetadata is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 api.InstanceID -func (_e *Backend_Expecter) GetOrchestrationMetadata(_a0 interface{}, _a1 interface{}) *Backend_GetOrchestrationMetadata_Call { - return &Backend_GetOrchestrationMetadata_Call{Call: _e.mock.On("GetOrchestrationMetadata", _a0, _a1)} -} - -func (_c *Backend_GetOrchestrationMetadata_Call) Run(run func(_a0 context.Context, _a1 api.InstanceID)) *Backend_GetOrchestrationMetadata_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(api.InstanceID)) - }) - return _c -} - -func (_c *Backend_GetOrchestrationMetadata_Call) Return(_a0 *api.OrchestrationMetadata, _a1 error) *Backend_GetOrchestrationMetadata_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Backend_GetOrchestrationMetadata_Call) RunAndReturn(run func(context.Context, api.InstanceID) (*api.OrchestrationMetadata, error)) *Backend_GetOrchestrationMetadata_Call { - _c.Call.Return(run) - return _c -} - -// GetOrchestrationRuntimeState provides a mock function with given fields: _a0, _a1 -func (_m *Backend) GetOrchestrationRuntimeState(_a0 context.Context, _a1 *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, error) { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for GetOrchestrationRuntimeState") - } - - var r0 *backend.OrchestrationRuntimeState - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, error)); ok { - return rf(_a0, _a1) - } - if rf, ok := ret.Get(0).(func(context.Context, *backend.OrchestrationWorkItem) *backend.OrchestrationRuntimeState); ok { - r0 = rf(_a0, _a1) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*backend.OrchestrationRuntimeState) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, *backend.OrchestrationWorkItem) error); ok { - r1 = rf(_a0, _a1) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Backend_GetOrchestrationRuntimeState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetOrchestrationRuntimeState' -type Backend_GetOrchestrationRuntimeState_Call struct { - *mock.Call -} - -// GetOrchestrationRuntimeState is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *backend.OrchestrationWorkItem -func (_e *Backend_Expecter) GetOrchestrationRuntimeState(_a0 interface{}, _a1 interface{}) *Backend_GetOrchestrationRuntimeState_Call { - return &Backend_GetOrchestrationRuntimeState_Call{Call: _e.mock.On("GetOrchestrationRuntimeState", _a0, _a1)} -} - -func (_c *Backend_GetOrchestrationRuntimeState_Call) Run(run func(_a0 context.Context, _a1 *backend.OrchestrationWorkItem)) *Backend_GetOrchestrationRuntimeState_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(*backend.OrchestrationWorkItem)) - }) - return _c -} - -func (_c *Backend_GetOrchestrationRuntimeState_Call) Return(_a0 *backend.OrchestrationRuntimeState, _a1 error) *Backend_GetOrchestrationRuntimeState_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Backend_GetOrchestrationRuntimeState_Call) RunAndReturn(run func(context.Context, *backend.OrchestrationWorkItem) (*backend.OrchestrationRuntimeState, error)) *Backend_GetOrchestrationRuntimeState_Call { - _c.Call.Return(run) - return _c -} - -// GetOrchestrationWorkItem provides a mock function with given fields: _a0 -func (_m *Backend) GetOrchestrationWorkItem(_a0 context.Context) (*backend.OrchestrationWorkItem, error) { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for GetOrchestrationWorkItem") - } - - var r0 *backend.OrchestrationWorkItem - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (*backend.OrchestrationWorkItem, error)); ok { - return rf(_a0) - } - if rf, ok := ret.Get(0).(func(context.Context) *backend.OrchestrationWorkItem); ok { - r0 = rf(_a0) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*backend.OrchestrationWorkItem) - } - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Backend_GetOrchestrationWorkItem_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetOrchestrationWorkItem' -type Backend_GetOrchestrationWorkItem_Call struct { - *mock.Call -} - -// GetOrchestrationWorkItem is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) GetOrchestrationWorkItem(_a0 interface{}) *Backend_GetOrchestrationWorkItem_Call { - return &Backend_GetOrchestrationWorkItem_Call{Call: _e.mock.On("GetOrchestrationWorkItem", _a0)} -} - -func (_c *Backend_GetOrchestrationWorkItem_Call) Run(run func(_a0 context.Context)) *Backend_GetOrchestrationWorkItem_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_GetOrchestrationWorkItem_Call) Return(_a0 *backend.OrchestrationWorkItem, _a1 error) *Backend_GetOrchestrationWorkItem_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Backend_GetOrchestrationWorkItem_Call) RunAndReturn(run func(context.Context) (*backend.OrchestrationWorkItem, error)) *Backend_GetOrchestrationWorkItem_Call { - _c.Call.Return(run) - return _c -} - -// PurgeOrchestrationState provides a mock function with given fields: _a0, _a1 -func (_m *Backend) PurgeOrchestrationState(_a0 context.Context, _a1 api.InstanceID) error { - ret := _m.Called(_a0, _a1) - - if len(ret) == 0 { - panic("no return value specified for PurgeOrchestrationState") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID) error); ok { - r0 = rf(_a0, _a1) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_PurgeOrchestrationState_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'PurgeOrchestrationState' -type Backend_PurgeOrchestrationState_Call struct { - *mock.Call -} - -// PurgeOrchestrationState is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 api.InstanceID -func (_e *Backend_Expecter) PurgeOrchestrationState(_a0 interface{}, _a1 interface{}) *Backend_PurgeOrchestrationState_Call { - return &Backend_PurgeOrchestrationState_Call{Call: _e.mock.On("PurgeOrchestrationState", _a0, _a1)} -} - -func (_c *Backend_PurgeOrchestrationState_Call) Run(run func(_a0 context.Context, _a1 api.InstanceID)) *Backend_PurgeOrchestrationState_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(api.InstanceID)) - }) - return _c -} - -func (_c *Backend_PurgeOrchestrationState_Call) Return(_a0 error) *Backend_PurgeOrchestrationState_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_PurgeOrchestrationState_Call) RunAndReturn(run func(context.Context, api.InstanceID) error) *Backend_PurgeOrchestrationState_Call { - _c.Call.Return(run) - return _c -} - -// Start provides a mock function with given fields: _a0 -func (_m *Backend) Start(_a0 context.Context) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for Start") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type Backend_Start_Call struct { - *mock.Call -} - -// Start is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) Start(_a0 interface{}) *Backend_Start_Call { - return &Backend_Start_Call{Call: _e.mock.On("Start", _a0)} -} - -func (_c *Backend_Start_Call) Run(run func(_a0 context.Context)) *Backend_Start_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_Start_Call) Return(_a0 error) *Backend_Start_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_Start_Call) RunAndReturn(run func(context.Context) error) *Backend_Start_Call { - _c.Call.Return(run) - return _c -} - -// Stop provides a mock function with given fields: _a0 -func (_m *Backend) Stop(_a0 context.Context) error { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for Stop") - } - - var r0 error - if rf, ok := ret.Get(0).(func(context.Context) error); ok { - r0 = rf(_a0) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Backend_Stop_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Stop' -type Backend_Stop_Call struct { - *mock.Call -} - -// Stop is a helper method to define mock.On call -// - _a0 context.Context -func (_e *Backend_Expecter) Stop(_a0 interface{}) *Backend_Stop_Call { - return &Backend_Stop_Call{Call: _e.mock.On("Stop", _a0)} -} - -func (_c *Backend_Stop_Call) Run(run func(_a0 context.Context)) *Backend_Stop_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *Backend_Stop_Call) Return(_a0 error) *Backend_Stop_Call { - _c.Call.Return(_a0) - return _c -} - -func (_c *Backend_Stop_Call) RunAndReturn(run func(context.Context) error) *Backend_Stop_Call { - _c.Call.Return(run) - return _c -} - -// NewBackend creates a new instance of Backend. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewBackend(t interface { - mock.TestingT - Cleanup(func()) -}) *Backend { - mock := &Backend{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/tests/mocks/Executor.go b/tests/mocks/Executor.go deleted file mode 100644 index 659ac369..00000000 --- a/tests/mocks/Executor.go +++ /dev/null @@ -1,162 +0,0 @@ -// Code generated by mockery v2.38.0. DO NOT EDIT. - -package mocks - -import ( - api "github.com/microsoft/durabletask-go/api" - backend "github.com/microsoft/durabletask-go/backend" - - context "context" - - mock "github.com/stretchr/testify/mock" - - protos "github.com/microsoft/durabletask-go/internal/protos" -) - -// Executor is an autogenerated mock type for the Executor type -type Executor struct { - mock.Mock -} - -type Executor_Expecter struct { - mock *mock.Mock -} - -func (_m *Executor) EXPECT() *Executor_Expecter { - return &Executor_Expecter{mock: &_m.Mock} -} - -// ExecuteActivity provides a mock function with given fields: _a0, _a1, _a2 -func (_m *Executor) ExecuteActivity(_a0 context.Context, _a1 api.InstanceID, _a2 *protos.HistoryEvent) (*protos.HistoryEvent, error) { - ret := _m.Called(_a0, _a1, _a2) - - if len(ret) == 0 { - panic("no return value specified for ExecuteActivity") - } - - var r0 *protos.HistoryEvent - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error)); ok { - return rf(_a0, _a1, _a2) - } - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID, *protos.HistoryEvent) *protos.HistoryEvent); ok { - r0 = rf(_a0, _a1, _a2) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*protos.HistoryEvent) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, api.InstanceID, *protos.HistoryEvent) error); ok { - r1 = rf(_a0, _a1, _a2) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Executor_ExecuteActivity_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ExecuteActivity' -type Executor_ExecuteActivity_Call struct { - *mock.Call -} - -// ExecuteActivity is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 api.InstanceID -// - _a2 *protos.HistoryEvent -func (_e *Executor_Expecter) ExecuteActivity(_a0 interface{}, _a1 interface{}, _a2 interface{}) *Executor_ExecuteActivity_Call { - return &Executor_ExecuteActivity_Call{Call: _e.mock.On("ExecuteActivity", _a0, _a1, _a2)} -} - -func (_c *Executor_ExecuteActivity_Call) Run(run func(_a0 context.Context, _a1 api.InstanceID, _a2 *protos.HistoryEvent)) *Executor_ExecuteActivity_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(api.InstanceID), args[2].(*protos.HistoryEvent)) - }) - return _c -} - -func (_c *Executor_ExecuteActivity_Call) Return(_a0 *protos.HistoryEvent, _a1 error) *Executor_ExecuteActivity_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Executor_ExecuteActivity_Call) RunAndReturn(run func(context.Context, api.InstanceID, *protos.HistoryEvent) (*protos.HistoryEvent, error)) *Executor_ExecuteActivity_Call { - _c.Call.Return(run) - return _c -} - -// ExecuteOrchestrator provides a mock function with given fields: ctx, iid, oldEvents, newEvents -func (_m *Executor) ExecuteOrchestrator(ctx context.Context, iid api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent) (*backend.ExecutionResults, error) { - ret := _m.Called(ctx, iid, oldEvents, newEvents) - - if len(ret) == 0 { - panic("no return value specified for ExecuteOrchestrator") - } - - var r0 *backend.ExecutionResults - var r1 error - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID, []*protos.HistoryEvent, []*protos.HistoryEvent) (*backend.ExecutionResults, error)); ok { - return rf(ctx, iid, oldEvents, newEvents) - } - if rf, ok := ret.Get(0).(func(context.Context, api.InstanceID, []*protos.HistoryEvent, []*protos.HistoryEvent) *backend.ExecutionResults); ok { - r0 = rf(ctx, iid, oldEvents, newEvents) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*backend.ExecutionResults) - } - } - - if rf, ok := ret.Get(1).(func(context.Context, api.InstanceID, []*protos.HistoryEvent, []*protos.HistoryEvent) error); ok { - r1 = rf(ctx, iid, oldEvents, newEvents) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Executor_ExecuteOrchestrator_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ExecuteOrchestrator' -type Executor_ExecuteOrchestrator_Call struct { - *mock.Call -} - -// ExecuteOrchestrator is a helper method to define mock.On call -// - ctx context.Context -// - iid api.InstanceID -// - oldEvents []*protos.HistoryEvent -// - newEvents []*protos.HistoryEvent -func (_e *Executor_Expecter) ExecuteOrchestrator(ctx interface{}, iid interface{}, oldEvents interface{}, newEvents interface{}) *Executor_ExecuteOrchestrator_Call { - return &Executor_ExecuteOrchestrator_Call{Call: _e.mock.On("ExecuteOrchestrator", ctx, iid, oldEvents, newEvents)} -} - -func (_c *Executor_ExecuteOrchestrator_Call) Run(run func(ctx context.Context, iid api.InstanceID, oldEvents []*protos.HistoryEvent, newEvents []*protos.HistoryEvent)) *Executor_ExecuteOrchestrator_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context), args[1].(api.InstanceID), args[2].([]*protos.HistoryEvent), args[3].([]*protos.HistoryEvent)) - }) - return _c -} - -func (_c *Executor_ExecuteOrchestrator_Call) Return(_a0 *backend.ExecutionResults, _a1 error) *Executor_ExecuteOrchestrator_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *Executor_ExecuteOrchestrator_Call) RunAndReturn(run func(context.Context, api.InstanceID, []*protos.HistoryEvent, []*protos.HistoryEvent) (*backend.ExecutionResults, error)) *Executor_ExecuteOrchestrator_Call { - _c.Call.Return(run) - return _c -} - -// NewExecutor creates a new instance of Executor. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewExecutor(t interface { - mock.TestingT - Cleanup(func()) -}) *Executor { - mock := &Executor{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/tests/mocks/TaskWorker.go b/tests/mocks/TaskWorker.go deleted file mode 100644 index 7ee0f2cc..00000000 --- a/tests/mocks/TaskWorker.go +++ /dev/null @@ -1,157 +0,0 @@ -// Code generated by mockery v2.38.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - - mock "github.com/stretchr/testify/mock" -) - -// TaskWorker is an autogenerated mock type for the TaskWorker type -type TaskWorker struct { - mock.Mock -} - -type TaskWorker_Expecter struct { - mock *mock.Mock -} - -func (_m *TaskWorker) EXPECT() *TaskWorker_Expecter { - return &TaskWorker_Expecter{mock: &_m.Mock} -} - -// ProcessNext provides a mock function with given fields: _a0 -func (_m *TaskWorker) ProcessNext(_a0 context.Context) (bool, error) { - ret := _m.Called(_a0) - - if len(ret) == 0 { - panic("no return value specified for ProcessNext") - } - - var r0 bool - var r1 error - if rf, ok := ret.Get(0).(func(context.Context) (bool, error)); ok { - return rf(_a0) - } - if rf, ok := ret.Get(0).(func(context.Context) bool); ok { - r0 = rf(_a0) - } else { - r0 = ret.Get(0).(bool) - } - - if rf, ok := ret.Get(1).(func(context.Context) error); ok { - r1 = rf(_a0) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// TaskWorker_ProcessNext_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ProcessNext' -type TaskWorker_ProcessNext_Call struct { - *mock.Call -} - -// ProcessNext is a helper method to define mock.On call -// - _a0 context.Context -func (_e *TaskWorker_Expecter) ProcessNext(_a0 interface{}) *TaskWorker_ProcessNext_Call { - return &TaskWorker_ProcessNext_Call{Call: _e.mock.On("ProcessNext", _a0)} -} - -func (_c *TaskWorker_ProcessNext_Call) Run(run func(_a0 context.Context)) *TaskWorker_ProcessNext_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *TaskWorker_ProcessNext_Call) Return(_a0 bool, _a1 error) *TaskWorker_ProcessNext_Call { - _c.Call.Return(_a0, _a1) - return _c -} - -func (_c *TaskWorker_ProcessNext_Call) RunAndReturn(run func(context.Context) (bool, error)) *TaskWorker_ProcessNext_Call { - _c.Call.Return(run) - return _c -} - -// Start provides a mock function with given fields: _a0 -func (_m *TaskWorker) Start(_a0 context.Context) { - _m.Called(_a0) -} - -// TaskWorker_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' -type TaskWorker_Start_Call struct { - *mock.Call -} - -// Start is a helper method to define mock.On call -// - _a0 context.Context -func (_e *TaskWorker_Expecter) Start(_a0 interface{}) *TaskWorker_Start_Call { - return &TaskWorker_Start_Call{Call: _e.mock.On("Start", _a0)} -} - -func (_c *TaskWorker_Start_Call) Run(run func(_a0 context.Context)) *TaskWorker_Start_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(context.Context)) - }) - return _c -} - -func (_c *TaskWorker_Start_Call) Return() *TaskWorker_Start_Call { - _c.Call.Return() - return _c -} - -func (_c *TaskWorker_Start_Call) RunAndReturn(run func(context.Context)) *TaskWorker_Start_Call { - _c.Call.Return(run) - return _c -} - -// StopAndDrain provides a mock function with given fields: -func (_m *TaskWorker) StopAndDrain() { - _m.Called() -} - -// TaskWorker_StopAndDrain_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'StopAndDrain' -type TaskWorker_StopAndDrain_Call struct { - *mock.Call -} - -// StopAndDrain is a helper method to define mock.On call -func (_e *TaskWorker_Expecter) StopAndDrain() *TaskWorker_StopAndDrain_Call { - return &TaskWorker_StopAndDrain_Call{Call: _e.mock.On("StopAndDrain")} -} - -func (_c *TaskWorker_StopAndDrain_Call) Run(run func()) *TaskWorker_StopAndDrain_Call { - _c.Call.Run(func(args mock.Arguments) { - run() - }) - return _c -} - -func (_c *TaskWorker_StopAndDrain_Call) Return() *TaskWorker_StopAndDrain_Call { - _c.Call.Return() - return _c -} - -func (_c *TaskWorker_StopAndDrain_Call) RunAndReturn(run func()) *TaskWorker_StopAndDrain_Call { - _c.Call.Return(run) - return _c -} - -// NewTaskWorker creates a new instance of TaskWorker. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewTaskWorker(t interface { - mock.TestingT - Cleanup(func()) -}) *TaskWorker { - mock := &TaskWorker{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/tests/mocks/task.go b/tests/mocks/task.go deleted file mode 100644 index f31a410f..00000000 --- a/tests/mocks/task.go +++ /dev/null @@ -1,128 +0,0 @@ -package mocks - -import ( - context "context" - "errors" - "sync" - "sync/atomic" - "time" - - backend "github.com/microsoft/durabletask-go/backend" -) - -var _ backend.TaskProcessor = &TestTaskProcessor{} - -// TestTaskProcessor implements a dummy task processor useful for testing -type TestTaskProcessor struct { - name string - - processingBlocked atomic.Bool - - workItemMu sync.Mutex - workItems []backend.WorkItem - - abandonedWorkItemMu sync.Mutex - abandonedWorkItems []backend.WorkItem - - completedWorkItemMu sync.Mutex - completedWorkItems []backend.WorkItem -} - -func NewTestTaskPocessor(name string) *TestTaskProcessor { - return &TestTaskProcessor{ - name: name, - } -} - -func (t *TestTaskProcessor) BlockProcessing() { - t.processingBlocked.Store(true) -} - -func (t *TestTaskProcessor) UnblockProcessing() { - t.processingBlocked.Store(false) -} - -func (t *TestTaskProcessor) PendingWorkItems() []backend.WorkItem { - t.workItemMu.Lock() - defer t.workItemMu.Unlock() - - // copy array - return append([]backend.WorkItem{}, t.workItems...) -} - -func (t *TestTaskProcessor) AbandonedWorkItems() []backend.WorkItem { - t.abandonedWorkItemMu.Lock() - defer t.abandonedWorkItemMu.Unlock() - - // copy array - return append([]backend.WorkItem{}, t.abandonedWorkItems...) -} - -func (t *TestTaskProcessor) CompletedWorkItems() []backend.WorkItem { - t.completedWorkItemMu.Lock() - defer t.completedWorkItemMu.Unlock() - - // copy array - return append([]backend.WorkItem{}, t.completedWorkItems...) -} - -func (t *TestTaskProcessor) AddWorkItems(wis ...backend.WorkItem) { - t.workItemMu.Lock() - defer t.workItemMu.Unlock() - - t.workItems = append(t.workItems, wis...) -} - -func (t *TestTaskProcessor) Name() string { - return t.name -} - -func (t *TestTaskProcessor) FetchWorkItem(context.Context) (backend.WorkItem, error) { - t.workItemMu.Lock() - defer t.workItemMu.Unlock() - - if len(t.workItems) == 0 { - return nil, backend.ErrNoWorkItems - } - - // pop first item - i := 0 - wi := t.workItems[i] - t.workItems = append(t.workItems[:i], t.workItems[i+1:]...) - - return wi, nil -} - -func (t *TestTaskProcessor) ProcessWorkItem(ctx context.Context, wi backend.WorkItem) error { - if !t.processingBlocked.Load() { - return nil - } - // wait for context cancellation or until processing is unblocked - for { - select { - case <-ctx.Done(): - return errors.New("dummy error processing work item") - default: - if !t.processingBlocked.Load() { - return nil - } - time.Sleep(time.Millisecond) - } - } -} - -func (t *TestTaskProcessor) AbandonWorkItem(ctx context.Context, wi backend.WorkItem) error { - t.abandonedWorkItemMu.Lock() - defer t.abandonedWorkItemMu.Unlock() - - t.abandonedWorkItems = append(t.abandonedWorkItems, wi) - return nil -} - -func (t *TestTaskProcessor) CompleteWorkItem(ctx context.Context, wi backend.WorkItem) error { - t.completedWorkItemMu.Lock() - defer t.completedWorkItemMu.Unlock() - - t.completedWorkItems = append(t.completedWorkItems, wi) - return nil -} diff --git a/tests/orchestrations_test.go b/tests/orchestrations_test.go deleted file mode 100644 index f7b66c66..00000000 --- a/tests/orchestrations_test.go +++ /dev/null @@ -1,1492 +0,0 @@ -package tests - -import ( - "context" - "errors" - "fmt" - "sort" - "strconv" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/backend/sqlite" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/microsoft/durabletask-go/task" -) - -func Test_EmptyOrchestration(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("EmptyOrchestrator", func(ctx *task.OrchestrationContext) (any, error) { - return nil, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "EmptyOrchestrator") - require.NoError(t, err) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("EmptyOrchestrator", id), - assertOrchestratorExecuted("EmptyOrchestrator", id, "COMPLETED"), - ) -} - -func Test_SingleTimer(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleTimer", func(ctx *task.OrchestrationContext) (any, error) { - err := ctx.CreateTimer(time.Duration(0)).Await(nil) - return nil, err - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "SingleTimer") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.GreaterOrEqual(t, metadata.LastUpdatedAt, metadata.CreatedAt) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("SingleTimer", id), - assertTimer(id), - assertOrchestratorExecuted("SingleTimer", id, "COMPLETED"), - ) -} - -func Test_ConcurrentTimers(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("TimerFanOut", func(ctx *task.OrchestrationContext) (any, error) { - tasks := []task.Task{} - for i := 0; i < 3; i++ { - tasks = append(tasks, ctx.CreateTimer(1*time.Second)) - } - for _, t := range tasks { - if err := t.Await(nil); err != nil { - return nil, err - } - } - return nil, nil - })) - - // Initialization - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "TimerFanOut") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.GreaterOrEqual(t, metadata.LastUpdatedAt, metadata.CreatedAt) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("TimerFanOut", id), - assertTimer(id), - assertTimer(id), - assertTimer(id), - assertOrchestratorExecuted("TimerFanOut", id, "COMPLETED"), - ) -} - -func Test_IsReplaying(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("IsReplayingOrch", func(ctx *task.OrchestrationContext) (any, error) { - values := []bool{ctx.IsReplaying} - if err := ctx.CreateTimer(time.Duration(0)).Await(nil); err != nil { - return nil, err - } - values = append(values, ctx.IsReplaying) - if err := ctx.CreateTimer(time.Duration(0)).Await(nil); err != nil { - return nil, err - } - values = append(values, ctx.IsReplaying) - return values, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "IsReplayingOrch") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `[true,true,false]`, metadata.SerializedOutput) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("IsReplayingOrch", id), - assertTimer(id), - assertTimer(id), - assertOrchestratorExecuted("IsReplayingOrch", id, "COMPLETED"), - ) -} - -func Test_SingleActivity(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界")) - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `"Hello, 世界!"`, metadata.SerializedOutput) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("SingleActivity", id), - assertActivity("SayHello", id, 0), - assertOrchestratorExecuted("SingleActivity", id, "COMPLETED"), - ) -} - -func Test_ActivityChain(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ActivityChain", func(ctx *task.OrchestrationContext) (any, error) { - val := 0 - for i := 0; i < 10; i++ { - if err := ctx.CallActivity("PlusOne", task.WithActivityInput(val)).Await(&val); err != nil { - return nil, err - } - } - return val, nil - })) - require.NoError(t, r.AddActivityN("PlusOne", func(ctx task.ActivityContext) (any, error) { - var input int - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - return input + 1, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ActivityChain") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `10`, metadata.SerializedOutput) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("ActivityChain", id), - assertActivity("PlusOne", id, 0), assertActivity("PlusOne", id, 1), assertActivity("PlusOne", id, 2), - assertActivity("PlusOne", id, 3), assertActivity("PlusOne", id, 4), assertActivity("PlusOne", id, 5), - assertActivity("PlusOne", id, 6), assertActivity("PlusOne", id, 7), assertActivity("PlusOne", id, 8), - assertActivity("PlusOne", id, 9), assertOrchestratorExecuted("ActivityChain", id, "COMPLETED"), - ) -} - -func Test_ActivityRetries(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ActivityRetries", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity("FailActivity", task.WithActivityRetryPolicy(&task.RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 10 * time.Millisecond, - })).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddActivityN("FailActivity", func(ctx task.ActivityContext) (any, error) { - return nil, errors.New("activity failure") - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ActivityRetries") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - // With 3 max attempts there will be two retries with 10 millis delay before each - require.GreaterOrEqual(t, metadata.LastUpdatedAt, metadata.CreatedAt.Add(2*10*time.Millisecond)) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("ActivityRetries", id), - assertActivity("FailActivity", id, 0), - assertTimer(id, assertTaskID(1)), - assertActivity("FailActivity", id, 2), - assertTimer(id, assertTaskID(3)), - assertActivity("FailActivity", id, 4), - assertOrchestratorExecuted("ActivityRetries", id, "FAILED"), - ) -} - -func Test_ActivityFanOut(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ActivityFanOut", func(ctx *task.OrchestrationContext) (any, error) { - tasks := []task.Task{} - for i := 0; i < 10; i++ { - tasks = append(tasks, ctx.CallActivity("ToString", task.WithActivityInput(i))) - } - results := []string{} - for _, t := range tasks { - var result string - if err := t.Await(&result); err != nil { - return nil, err - } - results = append(results, result) - } - sort.Sort(sort.Reverse(sort.StringSlice(results))) - return results, nil - })) - require.NoError(t, r.AddActivityN("ToString", func(ctx task.ActivityContext) (any, error) { - var input int - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - time.Sleep(1 * time.Second) - return fmt.Sprintf("%d", input), nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r, backend.WithMaxParallelism(10)) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ActivityFanOut") - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `["9","8","7","6","5","4","3","2","1","0"]`, metadata.SerializedOutput) - - // Because all the activities run in parallel, they should complete very quickly - assert.Less(t, metadata.LastUpdatedAt.Sub(metadata.CreatedAt), 3*time.Second) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("ActivityFanOut", id), - // TODO: Find a way to assert an unordered sequence of traces since the order of activity traces is non-deterministic. - ) -} - -func Test_SingleSubOrchestrator_Completed(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Parent", func(ctx *task.OrchestrationContext) (any, error) { - var input any - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output any - err := ctx.CallSubOrchestrator( - "Child", - task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_child"), - task.WithSubOrchestratorInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddOrchestratorN("Child", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return "", err - } - return input, nil - })) - - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - id, err := client.ScheduleNewOrchestration(ctx, "Parent", api.WithInput("Hello, world!")) - require.NoError(t, err) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `"Hello, world!"`, metadata.SerializedOutput) - - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("Parent", id), - assertOrchestratorExecuted("Child", id+"_child", "COMPLETED"), - assertOrchestratorExecuted("Parent", id, "COMPLETED"), - ) -} - -func Test_SingleSubOrchestrator_Failed(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Parent", func(ctx *task.OrchestrationContext) (any, error) { - err := ctx.CallSubOrchestrator( - "Child", - task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_child")).Await(nil) - return nil, err - })) - require.NoError(t, r.AddOrchestratorN("Child", func(ctx *task.OrchestrationContext) (any, error) { - return nil, errors.New("Child failed") - })) - - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - id, err := client.ScheduleNewOrchestration(ctx, "Parent") - require.NoError(t, err) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - if assert.NotNil(t, metadata.FailureDetails) { - assert.Contains(t, metadata.FailureDetails.ErrorMessage, "Child failed") - } - - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("Parent", id), - assertOrchestratorExecuted("Child", id+"_child", "FAILED"), - assertOrchestratorExecuted("Parent", id, "FAILED"), - ) -} - -func Test_SingleSubOrchestrator_Failed_Retries(t *testing.T) { - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Parent", func(ctx *task.OrchestrationContext) (any, error) { - err := ctx.CallSubOrchestrator( - "Child", - task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_child"), - task.WithSubOrchestrationRetryPolicy(&task.RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 10 * time.Millisecond, - BackoffCoefficient: 2, - })).Await(nil) - return nil, err - })) - require.NoError(t, r.AddOrchestratorN("Child", func(ctx *task.OrchestrationContext) (any, error) { - return nil, errors.New("Child failed") - })) - - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - id, err := client.ScheduleNewOrchestration(ctx, "Parent") - require.NoError(t, err) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - if assert.NotNil(t, metadata.FailureDetails) { - assert.Contains(t, metadata.FailureDetails.ErrorMessage, "Child failed") - } - - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("Parent", id), - assertOrchestratorExecuted("Child", id+"_child", "FAILED"), - assertTimer(id, assertTaskID(1)), - assertOrchestratorExecuted("Child", id+"_child", "FAILED"), - assertTimer(id, assertTaskID(3)), - assertOrchestratorExecuted("Child", id+"_child", "FAILED"), - assertOrchestratorExecuted("Parent", id, "FAILED"), - ) -} - -func Test_ContinueAsNew(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ContinueAsNewTest", func(ctx *task.OrchestrationContext) (any, error) { - var input int32 - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - - if input < 10 { - if err := ctx.CreateTimer(0).Await(nil); err != nil { - return nil, err - } - ctx.ContinueAsNew(input + 1) - } - return input, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ContinueAsNewTest", api.WithInput(0)) - if assert.NoError(t, err) { - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - if assert.NoError(t, err) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `10`, metadata.SerializedOutput) - } - } - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("ContinueAsNewTest", id), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertTimer(id), assertOrchestratorExecuted("ContinueAsNewTest", id, "CONTINUED_AS_NEW"), - assertOrchestratorExecuted("ContinueAsNewTest", id, "COMPLETED"), - ) -} - -func Test_ContinueAsNew_Events(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ContinueAsNewTest", func(ctx *task.OrchestrationContext) (any, error) { - var input int32 - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var complete bool - if err := ctx.WaitForSingleEvent("MyEvent", -1).Await(&complete); err != nil { - return nil, err - } - if complete { - return input, nil - } - ctx.ContinueAsNew(input+1, task.WithKeepUnprocessedEvents()) - return nil, nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ContinueAsNewTest", api.WithInput(0)) - require.NoError(t, err) - for i := 0; i < 10; i++ { - require.NoError(t, client.RaiseEvent(ctx, id, "MyEvent", api.WithEventPayload(false))) - } - require.NoError(t, client.RaiseEvent(ctx, id, "MyEvent", api.WithEventPayload(true))) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `10`, metadata.SerializedOutput) -} - -func Test_ExternalEventContention(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ContinueAsNewTest", func(ctx *task.OrchestrationContext) (any, error) { - var data int32 - if err := ctx.WaitForSingleEvent("MyEventData", 1*time.Second).Await(&data); err != nil && !errors.Is(err, task.ErrTaskCanceled) { - return nil, err - } - - var complete bool - if err := ctx.WaitForSingleEvent("MyEventSignal", -1).Await(&complete); err != nil { - return nil, err - } - - if complete { - return data, nil - } - - ctx.ContinueAsNew(nil, task.WithKeepUnprocessedEvents()) - return nil, nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ContinueAsNewTest") - require.NoError(t, err) - - // Wait for the timer to elapse - timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - _, err = client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.ErrorIs(t, err, timeoutCtx.Err()) - - // Now raise the event, which should queue correctly for the next time - // around - require.NoError(t, client.RaiseEvent(ctx, id, "MyEventData", api.WithEventPayload(42))) - require.NoError(t, client.RaiseEvent(ctx, id, "MyEventSignal", api.WithEventPayload(false))) - require.NoError(t, client.RaiseEvent(ctx, id, "MyEventSignal", api.WithEventPayload(true))) - - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `42`, metadata.SerializedOutput) -} - -func Test_ExternalEventOrchestration(t *testing.T) { - const eventCount = 10 - - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ExternalEventOrchestration", func(ctx *task.OrchestrationContext) (any, error) { - for i := 0; i < eventCount; i++ { - var value int - if err := ctx.WaitForSingleEvent("MyEvent", 5*time.Second).Await(&value); err != nil { - return false, err - } - if value != i { - return false, errors.New("Unexpected value") - } - } - return true, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestration", api.WithInput(0)) - if assert.NoError(t, err) { - for i := 0; i < eventCount; i++ { - opts := api.WithEventPayload(i) - require.NoError(t, client.RaiseEvent(ctx, id, "MyEvent", opts)) - } - - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - metadata, err := client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - require.True(t, metadata.IsComplete()) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - } - - // Validate the exported OTel traces - eventSizeInBytes := 1 - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("ExternalEventOrchestration", id), - assertOrchestratorExecuted("ExternalEventOrchestration", id, "COMPLETED", assertSpanEvents( - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - )), - ) -} - -func Test_ExternalEventTimeout(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ExternalEventOrchestrationWithTimeout", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.WaitForSingleEvent("MyEvent", 2*time.Second).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run two variations, one where we raise the external event and one where we don't (timeout) - for _, raiseEvent := range []bool{true, false} { - t.Run(fmt.Sprintf("RaiseEvent:%v", raiseEvent), func(t *testing.T) { - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrationWithTimeout") - require.NoError(t, err) - if raiseEvent { - require.NoError(t, client.RaiseEvent(ctx, id, "MyEvent")) - } - - timeoutCtx, cancel := context.WithTimeout(ctx, 5*time.Second) - defer cancel() - - metadata, err := client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - if raiseEvent { - assert.True(t, metadata.IsComplete()) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - - assertSpanSequence(t, spans, - assertOrchestratorCreated("ExternalEventOrchestrationWithTimeout", id), - assertOrchestratorExecuted("ExternalEventOrchestrationWithTimeout", id, "COMPLETED", assertSpanEvents( - assertExternalEvent("MyEvent", 0), - )), - ) - } else { - require.True(t, metadata.IsComplete()) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, metadata.RuntimeStatus) - if assert.NotNil(t, metadata.FailureDetails) { - // The exact message is not important - just make sure it's something clear - // NOTE: In a future version, we might have a specifc ErrorType contract. For now, the - // caller shouldn't make any assumptions about this. - assert.Equal(t, "the task was canceled", metadata.FailureDetails.ErrorMessage) - } - - assertSpanSequence(t, spans, - assertOrchestratorCreated("ExternalEventOrchestrationWithTimeout", id), - // A timer is used to implement the event timeout - assertTimer(id), - assertOrchestratorExecuted("ExternalEventOrchestrationWithTimeout", id, "FAILED", assertSpanEvents()), - ) - } - }) - exporter.Reset() - } -} - -func Test_SuspendResumeOrchestration(t *testing.T) { - const eventCount = 10 - - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SuspendResumeOrchestration", func(ctx *task.OrchestrationContext) (any, error) { - for i := 0; i < eventCount; i++ { - var value int - if err := ctx.WaitForSingleEvent("MyEvent", 5*time.Second).Await(&value); err != nil { - return false, err - } - if value != i { - return false, errors.New("Unexpected value") - } - } - return true, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration, which will block waiting for external events - id, err := client.ScheduleNewOrchestration(ctx, "SuspendResumeOrchestration", api.WithInput(0)) - require.NoError(t, err) - - // Wait for the orchestration to finish starting - _, err = client.WaitForOrchestrationStart(ctx, id) - require.NoError(t, err) - - // Suspend the orchestration - require.NoError(t, client.SuspendOrchestration(ctx, id, "")) - - // Raise a bunch of events to the orchestration (they should get buffered but not consumed) - for i := 0; i < eventCount; i++ { - opts := api.WithEventPayload(i) - require.NoError(t, client.RaiseEvent(ctx, id, "MyEvent", opts)) - } - - // Make sure the orchestration *doesn't* complete - timeoutCtx, cancel := context.WithTimeout(ctx, 3*time.Second) - defer cancel() - _, err = client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.ErrorIs(t, err, timeoutCtx.Err()) - - var metadata *api.OrchestrationMetadata - metadata, err = client.FetchOrchestrationMetadata(ctx, id) - if assert.NoError(t, err) { - assert.True(t, metadata.IsRunning()) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_SUSPENDED, metadata.RuntimeStatus) - } - - // Resume the orchestration and wait for it to complete - require.NoError(t, client.ResumeOrchestration(ctx, id, "")) - timeoutCtx, cancel = context.WithTimeout(ctx, 3*time.Second) - defer cancel() - _, err = client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - - // Validate the exported OTel traces - eventSizeInBytes := 1 - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("SuspendResumeOrchestration", id), - assertOrchestratorExecuted("SuspendResumeOrchestration", id, "COMPLETED", assertSpanEvents( - assertSuspendedEvent(), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertExternalEvent("MyEvent", eventSizeInBytes), - assertResumedEvent(), - )), - ) -} - -func Test_TerminateOrchestration(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("MyOrchestrator", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CreateTimer(3 * time.Second).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration, which will block waiting for external events - id, err := client.ScheduleNewOrchestration(ctx, "MyOrchestrator") - require.NoError(t, err) - - // Terminate the orchestration before the timer expires - require.NoError(t, client.TerminateOrchestration(ctx, id, api.WithOutput("You got terminated!"))) - - // Wait for the orchestration to complete - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.True(t, metadata.IsComplete()) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, metadata.RuntimeStatus) - require.Equal(t, `"You got terminated!"`, metadata.SerializedOutput) - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("MyOrchestrator", id), - assertOrchestratorExecuted("MyOrchestrator", id, "TERMINATED"), - ) -} - -func Test_TerminateOrchestration_Recursive(t *testing.T) { - delayTime := 4 * time.Second - executedActivity := false - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Root", func(ctx *task.OrchestrationContext) (any, error) { - tasks := []task.Task{} - for i := 0; i < 5; i++ { - task := ctx.CallSubOrchestrator("L1", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L1_"+strconv.Itoa(i))) - tasks = append(tasks, task) - } - for _, task := range tasks { - if err := task.Await(nil); err != nil { - return nil, err - } - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L1", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallSubOrchestrator("L2", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L2")).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L2", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - if err := ctx.CallActivity("Fail").Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddActivityN("Fail", func(ctx task.ActivityContext) (any, error) { - executedActivity = true - return nil, errors.New("Failed: Should not have executed the activity") - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Test terminating with and without recursion - for _, recurse := range []bool{true, false} { - t.Run(fmt.Sprintf("Recurse = %v", recurse), func(t *testing.T) { - // Run the orchestration, which will block waiting for external events - id, err := client.ScheduleNewOrchestration(ctx, "Root") - require.NoError(t, err) - - // Wait long enough to ensure all orchestrations have started (but not longer than the timer delay) - assert.Eventually(t, func() bool { - // List of all orchestrations created - orchestrationIDs := []string{string(id)} - for i := 0; i < 5; i++ { - orchestrationIDs = append(orchestrationIDs, string(id)+"_L1_"+strconv.Itoa(i), string(id)+"_L1_"+strconv.Itoa(i)+"_L2") - } - for _, orchID := range orchestrationIDs { - metadata, err := client.FetchOrchestrationMetadata(ctx, api.InstanceID(orchID)) - require.NoError(t, err) - // All orchestrations should be running - if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING { - return false - } - } - return true - }, 2*time.Second, 100*time.Millisecond) - - // Terminate the root orchestration and mark whether a recursive termination - output := fmt.Sprintf("Recursive termination = %v", recurse) - opts := []api.TerminateOptions{api.WithOutput(output), api.WithRecursiveTerminate(recurse)} - require.NoError(t, client.TerminateOrchestration(ctx, id, opts...)) - - // Wait for the root orchestration to complete and verify its terminated status - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, metadata.RuntimeStatus) - require.Equal(t, fmt.Sprintf("\"%s\"", output), metadata.SerializedOutput) - - // Wait for all L2 suborchestrations to complete - orchIDs := []string{} - for i := 0; i < 5; i++ { - orchIDs = append(orchIDs, string(id)+"_L1_"+strconv.Itoa(i)+"_L2") - } - for _, orchID := range orchIDs { - _, err := client.WaitForOrchestrationCompletion(ctx, api.InstanceID(orchID)) - require.NoError(t, err) - } - // Verify tht none of the L2 suborchestrations executed the activity in case of recursive termination - assert.NotEqual(t, recurse, executedActivity) - }) - } -} - -func Test_TerminateOrchestration_Recursive_TerminateCompletedSubOrchestration(t *testing.T) { - delayTime := 4 * time.Second - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Root", func(ctx *task.OrchestrationContext) (any, error) { - // Create L1 sub-orchestration and wait for it to complete - if err := ctx.CallSubOrchestrator("L1", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L1")).Await(nil); err != nil { - return nil, err - } - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L1", func(ctx *task.OrchestrationContext) (any, error) { - // Create L2 sub-orchestration but don't wait for it to complete - ctx.CallSubOrchestrator("L2", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L2")) - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L2", func(ctx *task.OrchestrationContext) (any, error) { - // Wait for `delayTime` - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Test terminating with and without recursion - for _, recurse := range []bool{true, false} { - t.Run(fmt.Sprintf("Recurse = %v", recurse), func(t *testing.T) { - // Run the orchestration, which will block waiting for external events - id, err := client.ScheduleNewOrchestration(ctx, "Root") - require.NoError(t, err) - - // Wait long enough to ensure that all L1 orchestrations have completed but Root and L2 are still running - assert.Eventually(t, func() bool { - // List of all orchestrations created - orchestrationIDs := []string{string(id), string(id) + "_L1", string(id) + "_L1_L2"} - for _, orchID := range orchestrationIDs { - // Fetch orchestration metadata - metadata, err := client.FetchOrchestrationMetadata(ctx, api.InstanceID(orchID)) - require.NoError(t, err) - if orchID == string(id)+"_L1" { - // L1 orchestration should have completed - if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED { - return false - } - } else { - // Root and L2 orchestrations should still be running - if metadata.RuntimeStatus != protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING { - return false - } - } - } - return true - }, 2*time.Second, 100*time.Millisecond) - - // Terminate the root orchestration and mark whether a recursive termination - output := fmt.Sprintf("Recursive termination = %v", recurse) - opts := []api.TerminateOptions{api.WithOutput(output), api.WithRecursiveTerminate(recurse)} - require.NoError(t, client.TerminateOrchestration(ctx, id, opts...)) - - // Wait for the root orchestration to complete and verify its terminated status - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED, metadata.RuntimeStatus) - require.Equal(t, fmt.Sprintf("\"%s\"", output), metadata.SerializedOutput) - - // Verify that the L1 and L2 orchestrations have completed with the appropriate status - L1_OrchestrationStatus := protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED - L2_OrchestrationStatus := protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED - L2_Output := "" - L1_Output := "" - if recurse { - L2_OrchestrationStatus = protos.OrchestrationStatus_ORCHESTRATION_STATUS_TERMINATED - L2_Output = fmt.Sprintf("\"%s\"", output) - } - // In recursive case, L1 orchestration is not terminated because it was already completed when the root orchestration was terminated - metadata, err = client.WaitForOrchestrationCompletion(ctx, id+"_L1") - require.NoError(t, err) - require.Equal(t, L1_OrchestrationStatus, metadata.RuntimeStatus) - require.Equal(t, L1_Output, metadata.SerializedOutput) - - // In recursive case, L2 is terminated because it was still running when the root orchestration was terminated - metadata, err = client.WaitForOrchestrationCompletion(ctx, id+"_L1_L2") - require.NoError(t, err) - require.Equal(t, L2_OrchestrationStatus, metadata.RuntimeStatus) - require.Equal(t, L2_Output, metadata.SerializedOutput) - }) - } -} - -func Test_PurgeCompletedOrchestration(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("ExternalEventOrchestration", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.WaitForSingleEvent("MyEvent", 30*time.Second).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestration") - if !assert.NoError(t, err) { - return - } - if _, err = client.WaitForOrchestrationStart(ctx, id); !assert.NoError(t, err) { - return - } - - // Try to purge the orchestration state before it completes and verify that it fails with ErrNotCompleted - if err = client.PurgeOrchestrationState(ctx, id); !assert.ErrorIs(t, err, api.ErrNotCompleted) { - return - } - - // Raise an event to the orchestration so that it can complete - if err = client.RaiseEvent(ctx, id, "MyEvent"); !assert.NoError(t, err) { - return - } - if _, err = client.WaitForOrchestrationCompletion(ctx, id); !assert.NoError(t, err) { - return - } - - // Try to purge the orchestration state again and verify that it succeeds - if err = client.PurgeOrchestrationState(ctx, id); !assert.NoError(t, err) { - return - } - - // Try to fetch the orchestration metadata and verify that it fails with ErrInstanceNotFound - if _, err = client.FetchOrchestrationMetadata(ctx, id); !assert.ErrorIs(t, err, api.ErrInstanceNotFound) { - return - } - - // Try to purge again and verify that it also fails with ErrInstanceNotFound - if err = client.PurgeOrchestrationState(ctx, id); !assert.ErrorIs(t, err, api.ErrInstanceNotFound) { - return - } -} - -func Test_PurgeOrchestration_Recursive(t *testing.T) { - delayTime := 4 * time.Second - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("Root", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallSubOrchestrator("L1", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L1")).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L1", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallSubOrchestrator("L2", task.WithSubOrchestrationInstanceID(string(ctx.ID)+"_L2")).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - require.NoError(t, r.AddOrchestratorN("L2", func(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CreateTimer(delayTime).Await(nil); err != nil { - return nil, err - } - return nil, nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Test terminating with and without recursion - for _, recurse := range []bool{true, false} { - t.Run(fmt.Sprintf("Recurse = %v", recurse), func(t *testing.T) { - // Run the orchestration, which will block waiting for external events - id, err := client.ScheduleNewOrchestration(ctx, "Root") - require.NoError(t, err) - - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - - // Purge the root orchestration - opts := []api.PurgeOptions{api.WithRecursivePurge(recurse)} - err = client.PurgeOrchestrationState(ctx, id, opts...) - assert.NoError(t, err) - - // Verify that root Orchestration has been purged - _, err = client.FetchOrchestrationMetadata(ctx, id) - assert.ErrorIs(t, err, api.ErrInstanceNotFound) - - if recurse { - // Verify that L1 and L2 orchestrations have been purged - _, err = client.FetchOrchestrationMetadata(ctx, id+"_L1") - assert.ErrorIs(t, err, api.ErrInstanceNotFound) - - _, err = client.FetchOrchestrationMetadata(ctx, id+"_L1_L2") - assert.ErrorIs(t, err, api.ErrInstanceNotFound) - } else { - // Verify that L1 and L2 orchestrations are not purged - metadata, err = client.FetchOrchestrationMetadata(ctx, id+"_L1") - assert.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - - _, err = client.FetchOrchestrationMetadata(ctx, id+"_L1_L2") - assert.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - } - }) - } -} - -func Test_RecreateCompletedOrchestration(t *testing.T) { - t.Skip("Not yet supported. Needs https://github.com/microsoft/durabletask-go/issues/42") - - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("HelloOrchestration", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - // Initialization - ctx := context.Background() - exporter := initTracing() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - // Run the first orchestration - id, err := client.ScheduleNewOrchestration(ctx, "HelloOrchestration", api.WithInput("世界")) - require.NoError(t, err) - metadata, err := client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `"Hello, 世界!"`, metadata.SerializedOutput) - - // Run the second orchestration with the same ID as the first - var newID api.InstanceID - newID, err = client.ScheduleNewOrchestration(ctx, "HelloOrchestration", api.WithInstanceID(id), api.WithInput("World")) - require.NoError(t, err) - require.Equal(t, id, newID) - metadata, err = client.WaitForOrchestrationCompletion(ctx, id) - require.NoError(t, err) - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, metadata.RuntimeStatus) - assert.Equal(t, `"Hello, World!"`, metadata.SerializedOutput) - - // Validate the exported OTel traces - spans := exporter.GetSpans().Snapshots() - assertSpanSequence(t, spans, - assertOrchestratorCreated("SingleActivity", id), - assertActivity("SayHello", id, 0), - assertOrchestratorExecuted("SingleActivity", id, "COMPLETED"), - assertOrchestratorCreated("SingleActivity", id), - assertActivity("SayHello", id, 0), - assertOrchestratorExecuted("SingleActivity", id, "COMPLETED"), - ) -} - -func Test_SingleActivity_ReuseInstanceIDIgnore(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - instanceID := api.InstanceID("IGNORE_IF_RUNNING_OR_COMPLETED") - reuseIdPolicy := &api.OrchestrationIdReusePolicy{ - Action: api.REUSE_ID_ACTION_IGNORE, - OperationStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_COMPLETED, api.RUNTIME_STATUS_PENDING}, - } - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - // wait orchestration to start - _, err = client.WaitForOrchestrationStart(ctx, id) - require.NoError(t, err) - pivotTime := time.Now() - // schedule again, it should ignore creating the new orchestration - id, err = client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(id), api.WithOrchestrationIdReusePolicy(reuseIdPolicy)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - // the first orchestration should complete as the second one is ignored - assert.Equal(t, `"Hello, 世界!"`, metadata.SerializedOutput) - // assert the orchestration created timestamp - assert.True(t, pivotTime.After(metadata.CreatedAt)) -} - -func Test_SingleActivity_ReuseInstanceIDTerminate(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - instanceID := api.InstanceID("TERMINATE_IF_RUNNING_OR_COMPLETED") - reuseIdPolicy := &api.OrchestrationIdReusePolicy{ - Action: api.REUSE_ID_ACTION_TERMINATE, - OperationStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_COMPLETED, api.RUNTIME_STATUS_PENDING}, - } - - // Run the orchestration - id, err := client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - // wait orchestration to start - _, err = client.WaitForOrchestrationStart(ctx, id) - require.NoError(t, err) - pivotTime := time.Now() - // schedule again, it should terminate the first orchestration and start a new one - id, err = client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(id), api.WithOrchestrationIdReusePolicy(reuseIdPolicy)) - require.NoError(t, err) - timeoutCtx, cancelTimeout := context.WithTimeout(ctx, 30*time.Second) - defer cancelTimeout() - metadata, err := client.WaitForOrchestrationCompletion(timeoutCtx, id) - require.NoError(t, err) - assert.Equal(t, true, metadata.IsComplete()) - // the second orchestration should complete. - assert.Equal(t, `"Hello, World!"`, metadata.SerializedOutput) - // assert the orchestration created timestamp - assert.True(t, pivotTime.Before(metadata.CreatedAt)) -} - -func Test_SingleActivity_ReuseInstanceIDError(t *testing.T) { - // Registration - r := task.NewTaskRegistry() - require.NoError(t, r.AddOrchestratorN("SingleActivity", func(ctx *task.OrchestrationContext) (any, error) { - var input string - if err := ctx.GetInput(&input); err != nil { - return nil, err - } - var output string - err := ctx.CallActivity("SayHello", task.WithActivityInput(input)).Await(&output) - return output, err - })) - require.NoError(t, r.AddActivityN("SayHello", func(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err - } - return fmt.Sprintf("Hello, %s!", name), nil - })) - - // Initialization - ctx := context.Background() - client, worker := initTaskHubWorker(ctx, r) - defer func() { - if err := worker.Shutdown(ctx); err != nil { - t.Logf("shutdown: %v", err) - } - }() - - instanceID := api.InstanceID("ERROR_IF_RUNNING_OR_COMPLETED") - - // Run the orchestration - _, err := client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("世界"), api.WithInstanceID(instanceID)) - require.NoError(t, err) - _, err = client.ScheduleNewOrchestration(ctx, "SingleActivity", api.WithInput("World"), api.WithInstanceID(instanceID)) - if assert.Error(t, err) { - assert.Contains(t, err.Error(), "orchestration instance already exists") - } -} - -func initTaskHubWorker(ctx context.Context, r *task.TaskRegistry, opts ...backend.NewTaskWorkerOptions) (backend.TaskHubClient, backend.TaskHubWorker) { - // TODO: Switch to options pattern - logger := backend.DefaultLogger() - be := sqlite.NewSqliteBackend(sqlite.NewSqliteOptions(""), logger) - executor := task.NewTaskExecutor(r) - orchestrationWorker := backend.NewOrchestrationWorker(be, executor, logger, opts...) - activityWorker := backend.NewActivityTaskWorker(be, executor, logger, opts...) - taskHubWorker := backend.NewTaskHubWorker(be, orchestrationWorker, activityWorker, logger) - if err := taskHubWorker.Start(ctx); err != nil { - panic(err) - } - taskHubClient := backend.NewTaskHubClient(be) - return taskHubClient, taskHubWorker -} diff --git a/tests/runtimestate_test.go b/tests/runtimestate_test.go deleted file mode 100644 index b14608ba..00000000 --- a/tests/runtimestate_test.go +++ /dev/null @@ -1,410 +0,0 @@ -package tests - -import ( - "testing" - "time" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/stretchr/testify/assert" - "google.golang.org/protobuf/types/known/timestamppb" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -// Verifies runtime state created from an ExecutionStarted event -func Test_NewOrchestration(t *testing.T) { - const iid = "abc" - const expectedName = "myorchestration" - createdAt := time.Now().UTC() - - e := &protos.HistoryEvent{ - EventId: -1, - Timestamp: timestamppb.New(createdAt), - EventType: &protos.HistoryEvent_ExecutionStarted{ - ExecutionStarted: &protos.ExecutionStartedEvent{ - OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: iid}, - Name: expectedName, - }, - }, - } - - s := backend.NewOrchestrationRuntimeState(iid, []*protos.HistoryEvent{e}) - assert.Equal(t, api.InstanceID(iid), s.InstanceID()) - - actualName, err := s.Name() - if assert.NoError(t, err) { - assert.Equal(t, expectedName, actualName) - } - - actualTime, err := s.CreatedTime() - if assert.NoError(t, err) { - assert.WithinDuration(t, createdAt, actualTime, 0) - } - - _, err = s.CompletedTime() - if assert.Error(t, err) { - assert.Equal(t, api.ErrNotCompleted, err) - } - - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING, s.RuntimeStatus()) - - oldEvents := s.OldEvents() - if assert.Equal(t, 1, len(oldEvents)) { - assert.Equal(t, e, oldEvents[0]) - } - - assert.Equal(t, 0, len(s.NewEvents())) -} - -func Test_CompletedOrchestration(t *testing.T) { - const iid = "abc" - const expectedName = "myorchestration" - createdAt := time.Now().UTC() - completedAt := createdAt.Add(10 * time.Second) - - events := []*protos.HistoryEvent{{ - EventId: -1, - Timestamp: timestamppb.New(createdAt), - EventType: &protos.HistoryEvent_ExecutionStarted{ - ExecutionStarted: &protos.ExecutionStartedEvent{ - OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: iid}, - Name: expectedName, - }, - }, - }, { - EventId: -1, - Timestamp: timestamppb.New(completedAt), - EventType: &protos.HistoryEvent_ExecutionCompleted{ - ExecutionCompleted: &protos.ExecutionCompletedEvent{ - OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, - }, - }, - }} - - s := backend.NewOrchestrationRuntimeState(iid, events) - assert.Equal(t, api.InstanceID(iid), s.InstanceID()) - - actualName, err := s.Name() - if assert.NoError(t, err) { - assert.Equal(t, expectedName, actualName) - } - - actualCreatedTime, err := s.CreatedTime() - if assert.NoError(t, err) { - assert.WithinDuration(t, createdAt, actualCreatedTime, 0) - } - - actualCompletedTime, err := s.CompletedTime() - if assert.NoError(t, err) { - assert.WithinDuration(t, completedAt, actualCompletedTime, 0) - } - - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, s.RuntimeStatus()) - - assert.Equal(t, events, s.OldEvents()) - assert.Equal(t, 0, len(s.NewEvents())) -} - -func Test_CompletedSubOrchestration(t *testing.T) { - expectedOutput := "\"done!\"" - expectedTaskID := int32(3) - - // TODO: Loop through different completion status values - status := protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED - - parentInfo := helpers.NewParentInfo(expectedTaskID, "Parent", "parent_id") - s := backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("Child", "child_id", nil, parentInfo, nil, nil), - }) - - actions := []*protos.OrchestratorAction{ - helpers.NewCompleteOrchestrationAction( - expectedTaskID, - status, - wrapperspb.String(expectedOutput), - []*protos.HistoryEvent{}, - nil), - } - - continuedAsNew, err := s.ApplyActions(actions, nil) - if assert.NoError(t, err) && assert.False(t, continuedAsNew) { - if assert.Len(t, s.NewEvents(), 1) { - e := s.NewEvents()[0] - assert.NotNil(t, e.Timestamp) - if ec := e.GetExecutionCompleted(); assert.NotNil(t, ec) { - assert.Equal(t, expectedTaskID, e.EventId) - assert.Equal(t, status, ec.OrchestrationStatus) - assert.Equal(t, expectedOutput, ec.Result.GetValue()) - assert.Nil(t, ec.FailureDetails) - } - } - if assert.Len(t, s.PendingMessages(), 1) { - e := s.PendingMessages()[0] - assert.NotNil(t, e.HistoryEvent.Timestamp) - if soc := e.HistoryEvent.GetSubOrchestrationInstanceCompleted(); assert.NotNil(t, soc) { - assert.Equal(t, expectedTaskID, soc.TaskScheduledId) - assert.Equal(t, expectedOutput, soc.Result.GetValue()) - } - } - } -} - -func Test_RuntimeState_ContinueAsNew(t *testing.T) { - iid := "abc" - expectedName := "MyOrchestration" - continueAsNewInput := "\"done!\"" - expectedTaskID := int32(3) - eventName := "MyRaisedEvent" - eventPayload := "MyEventPayload" - - state := backend.NewOrchestrationRuntimeState(api.InstanceID(iid), []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent(expectedName, iid, nil, nil, nil, nil), - }) - - carryoverEvents := []*protos.HistoryEvent{helpers.NewEventRaisedEvent(eventName, wrapperspb.String(eventPayload))} - actions := []*protos.OrchestratorAction{ - helpers.NewCompleteOrchestrationAction( - expectedTaskID, - protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, - wrapperspb.String(continueAsNewInput), - carryoverEvents, - nil), - } - - continuedAsNew, err := state.ApplyActions(actions, nil) - if assert.NoError(t, err) && assert.True(t, continuedAsNew) { - if assert.Len(t, state.NewEvents(), 3) { - assert.NotNil(t, state.NewEvents()[0].Timestamp) - assert.NotNil(t, state.NewEvents()[0].GetOrchestratorStarted()) - assert.NotNil(t, state.NewEvents()[1].Timestamp) - if ec := state.NewEvents()[1].GetExecutionStarted(); assert.NotNil(t, ec) { - assert.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_RUNNING, state.RuntimeStatus()) - assert.Equal(t, string(state.InstanceID()), ec.OrchestrationInstance.InstanceId) - if name, err := state.Name(); assert.NoError(t, err) { - assert.Equal(t, expectedName, name) - assert.Equal(t, expectedName, ec.Name) - } - if input, err := state.Input(); assert.NoError(t, err) { - assert.Equal(t, continueAsNewInput, input) - } - } - assert.NotNil(t, state.NewEvents()[2].Timestamp) - if er := state.NewEvents()[2].GetEventRaised(); assert.NotNil(t, er) { - assert.Equal(t, eventName, er.Name) - assert.Equal(t, eventPayload, er.Input.GetValue()) - } - } - assert.Empty(t, state.PendingMessages()) - assert.Empty(t, state.PendingTasks()) - assert.Empty(t, state.PendingTimers()) - } -} - -func Test_CreateTimer(t *testing.T) { - const iid = "abc" - expectedFireAt := time.Now().UTC().Add(72 * time.Hour) - - s := backend.NewOrchestrationRuntimeState(iid, []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("MyOrchestration", iid, nil, nil, nil, nil), - }) - - var actions []*protos.OrchestratorAction - timerCount := 3 - for i := 1; i <= timerCount; i++ { - actions = append(actions, helpers.NewCreateTimerAction(int32(i), expectedFireAt)) - } - - continuedAsNew, err := s.ApplyActions(actions, nil) - if assert.NoError(t, err) && assert.False(t, continuedAsNew) { - if assert.Len(t, s.NewEvents(), timerCount) { - for _, e := range s.NewEvents() { - assert.NotNil(t, e.Timestamp) - if timerCreated := e.GetTimerCreated(); assert.NotNil(t, timerCreated) { - assert.WithinDuration(t, expectedFireAt, timerCreated.FireAt.AsTime(), 0) - } - } - } - if assert.Len(t, s.PendingTimers(), timerCount) { - for i, e := range s.PendingTimers() { - assert.NotNil(t, e.Timestamp) - if timerFired := e.GetTimerFired(); assert.NotNil(t, timerFired) { - expectedTimerID := int32(i + 1) - assert.WithinDuration(t, expectedFireAt, timerFired.FireAt.AsTime(), 0) - assert.Equal(t, expectedTimerID, timerFired.TimerId) - } - } - } - } -} - -func Test_ScheduleTask(t *testing.T) { - const iid = "abc" - expectedTaskID := int32(1) - expectedName := "MyActivity" - expectedInput := "{\"Foo\":5}" - - state := backend.NewOrchestrationRuntimeState(iid, []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("MyOrchestration", iid, wrapperspb.String(expectedInput), nil, nil, nil), - }) - - actions := []*protos.OrchestratorAction{ - helpers.NewScheduleTaskAction(expectedTaskID, expectedName, wrapperspb.String(expectedInput)), - } - - tc := &protos.TraceContext{TraceParent: "trace", TraceState: wrapperspb.String("state")} - continuedAsNew, err := state.ApplyActions(actions, tc) - if assert.NoError(t, err) && assert.False(t, continuedAsNew) { - if assert.Len(t, state.NewEvents(), 1) { - e := state.NewEvents()[0] - if taskScheduled := e.GetTaskScheduled(); assert.NotNil(t, taskScheduled) { - assert.Equal(t, expectedTaskID, e.EventId) - assert.Equal(t, expectedName, taskScheduled.Name) - assert.Equal(t, expectedInput, taskScheduled.Input.GetValue()) - if assert.NotNil(t, taskScheduled.ParentTraceContext) { - assert.Equal(t, "trace", taskScheduled.ParentTraceContext.TraceParent) - assert.Equal(t, "state", taskScheduled.ParentTraceContext.TraceState.GetValue()) - } - } - } - if assert.Len(t, state.PendingTasks(), 1) { - e := state.PendingTasks()[0] - if taskScheduled := e.GetTaskScheduled(); assert.NotNil(t, taskScheduled) { - assert.Equal(t, expectedTaskID, e.EventId) - assert.Equal(t, expectedName, taskScheduled.Name) - assert.Equal(t, expectedInput, taskScheduled.Input.GetValue()) - if assert.NotNil(t, taskScheduled.ParentTraceContext) { - assert.Equal(t, "trace", taskScheduled.ParentTraceContext.TraceParent) - assert.Equal(t, "state", taskScheduled.ParentTraceContext.TraceState.GetValue()) - } - } - } - } -} - -func Test_CreateSubOrchestration(t *testing.T) { - iid := "abc" - expectedTaskID := int32(4) - expectedInstanceID := "xyz" - expectedName := "MySubOrchestration" - expectedInput := wrapperspb.String("{\"Foo\":5}") - expectedTraceParent := "trace" - expectedTraceState := "trace_state" - - state := backend.NewOrchestrationRuntimeState(api.InstanceID(iid), []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("Parent", iid, nil, nil, nil, nil), - }) - - actions := []*protos.OrchestratorAction{ - helpers.NewCreateSubOrchestrationAction(expectedTaskID, expectedName, expectedInstanceID, expectedInput), - } - - tc := &protos.TraceContext{ - TraceParent: expectedTraceParent, - TraceState: wrapperspb.String(expectedTraceState), - } - continuedAsNew, err := state.ApplyActions(actions, tc) - if assert.NoError(t, err) && assert.False(t, continuedAsNew) { - if assert.Len(t, state.NewEvents(), 1) { - e := state.NewEvents()[0] - if orchCreated := e.GetSubOrchestrationInstanceCreated(); assert.NotNil(t, orchCreated) { - assert.Equal(t, expectedTaskID, e.EventId) - assert.Equal(t, expectedInstanceID, orchCreated.InstanceId) - assert.Equal(t, expectedName, orchCreated.Name) - assert.Equal(t, expectedInput.GetValue(), orchCreated.Input.GetValue()) - if assert.NotNil(t, orchCreated.ParentTraceContext) { - assert.Equal(t, expectedTraceParent, orchCreated.ParentTraceContext.TraceParent) - assert.Equal(t, expectedTraceState, orchCreated.ParentTraceContext.TraceState.GetValue()) - } - } - } - if assert.Len(t, state.PendingMessages(), 1) { - msg := state.PendingMessages()[0] - if executionStarted := msg.HistoryEvent.GetExecutionStarted(); assert.NotNil(t, executionStarted) { - assert.Equal(t, int32(-1), msg.HistoryEvent.EventId) - assert.Equal(t, expectedInstanceID, executionStarted.OrchestrationInstance.InstanceId) - assert.NotEmpty(t, executionStarted.OrchestrationInstance.ExecutionId) - assert.Equal(t, expectedName, executionStarted.Name) - assert.Equal(t, expectedInput.GetValue(), executionStarted.Input.GetValue()) - if assert.NotNil(t, executionStarted.ParentInstance) { - assert.Equal(t, "Parent", executionStarted.ParentInstance.Name.GetValue()) - assert.Equal(t, expectedTaskID, executionStarted.ParentInstance.TaskScheduledId) - if assert.NotNil(t, executionStarted.ParentInstance.OrchestrationInstance) { - assert.Equal(t, iid, executionStarted.ParentInstance.OrchestrationInstance.InstanceId) - } - } - if assert.NotNil(t, executionStarted.ParentTraceContext) { - assert.Equal(t, expectedTraceParent, executionStarted.ParentTraceContext.TraceParent) - assert.Equal(t, expectedTraceState, executionStarted.ParentTraceContext.TraceState.GetValue()) - } - } - } - } -} - -func Test_SendEvent(t *testing.T) { - expectedInstanceID := "xyz" - expectedEventName := "MyEvent" - expectedInput := "foo" - - s := backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("MyOrchestration", "abc", wrapperspb.String(expectedInput), nil, nil, nil), - }) - - actions := []*protos.OrchestratorAction{ - helpers.NewSendEventAction(expectedInstanceID, expectedEventName, wrapperspb.String(expectedInput)), - } - - continuedAsNew, err := s.ApplyActions(actions, nil) - if assert.NoError(t, err) && assert.False(t, continuedAsNew) { - if assert.Len(t, s.NewEvents(), 1) { - e := s.NewEvents()[0] - if sendEvent := e.GetEventSent(); assert.NotNil(t, sendEvent) { - assert.Equal(t, expectedEventName, sendEvent.Name) - assert.Equal(t, expectedInput, sendEvent.Input.GetValue()) - assert.Equal(t, expectedInstanceID, sendEvent.InstanceId) - } - } - if assert.Len(t, s.PendingMessages(), 1) { - msg := s.PendingMessages()[0] - if sendEvent := msg.HistoryEvent.GetEventSent(); assert.NotNil(t, sendEvent) { - assert.Equal(t, expectedEventName, sendEvent.Name) - assert.Equal(t, expectedInput, sendEvent.Input.GetValue()) - assert.Equal(t, expectedInstanceID, sendEvent.InstanceId) - } - } - } -} - -func Test_StateIsValid(t *testing.T) { - s := backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{}) - assert.True(t, s.IsValid()) - s = backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{ - helpers.NewExecutionStartedEvent("MyOrchestration", "abc", nil, nil, nil, nil), - }) - assert.True(t, s.IsValid()) - s = backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{ - helpers.NewTaskCompletedEvent(1, nil), - }) - assert.False(t, s.IsValid()) -} - -func Test_DuplicateEvents(t *testing.T) { - s := backend.NewOrchestrationRuntimeState("abc", []*protos.HistoryEvent{}) - if err := s.AddEvent(helpers.NewExecutionStartedEvent("MyOrchestration", "abc", nil, nil, nil, nil)); assert.NoError(t, err) { - err = s.AddEvent(helpers.NewExecutionStartedEvent("MyOrchestration", "abc", nil, nil, nil, nil)) - assert.ErrorIs(t, err, backend.ErrDuplicateEvent) - } else { - return - } - - // TODO: Add other types of duplicate events (task completion, external events, sub-orchestration, etc.) - - if err := s.AddEvent(helpers.NewExecutionCompletedEvent(-1, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, nil, nil)); assert.NoError(t, err) { - err = s.AddEvent(helpers.NewExecutionCompletedEvent(-1, protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, nil, nil)) - assert.ErrorIs(t, err, backend.ErrDuplicateEvent) - } else { - return - } -} diff --git a/tests/shared_test.go b/tests/shared_test.go new file mode 100644 index 00000000..e63034e7 --- /dev/null +++ b/tests/shared_test.go @@ -0,0 +1,12 @@ +// Package tests holds cross-package behavior tests that exercise the +// transport-neutral runtime — the task executor, entity batch execution, and +// orchestration metadata — without a live task hub service. +// +// Every test here is deterministic: histories are hand-built rather than +// fetched from a service. End-to-end coverage against the only supported +// runtime lives in tests/durabletaskscheduler. +package tests + +import "context" + +var ctx = context.Background() diff --git a/tests/task_executor_test.go b/tests/task_executor_test.go index 9e2d1d2e..a4a78c22 100644 --- a/tests/task_executor_test.go +++ b/tests/task_executor_test.go @@ -34,7 +34,7 @@ func Test_Executor_WaitForEventSchedulesTimer(t *testing.T) { // Execute the orchestrator function and expect to get back a single timer action executor := task.NewTaskExecutor(r) - results, err := executor.ExecuteOrchestrator(ctx, iid, oldEvents, newEvents) + results, err := executor.ExecuteOrchestrator(ctx, iid, oldEvents, newEvents, nil) require.NoError(t, err) require.Equal(t, 1, len(results.Response.Actions), "Expected a single action to be scheduled") createTimerAction := results.Response.Actions[0].GetCreateTimer() @@ -66,7 +66,7 @@ func Test_Executor_SuspendStopsAllActions(t *testing.T) { } // Execute the orchestrator function and expect to get back no actions - results, err := executor.ExecuteOrchestrator(ctx, iid, oldEvents, newEvents) + results, err := executor.ExecuteOrchestrator(ctx, iid, oldEvents, newEvents, nil) require.NoError(t, err) require.Empty(t, results.Response.Actions, "Suspended orchestrations should not have any actions") } diff --git a/tests/taskhub_test.go b/tests/taskhub_test.go deleted file mode 100644 index 936d93ed..00000000 --- a/tests/taskhub_test.go +++ /dev/null @@ -1,43 +0,0 @@ -package tests - -import ( - "context" - "testing" - - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/tests/mocks" - "github.com/stretchr/testify/assert" -) - -func Test_TaskHubWorkerStartsDependencies(t *testing.T) { - ctx := context.Background() - - be := mocks.NewBackend(t) - orchWorker := mocks.NewTaskWorker(t) - actWorker := mocks.NewTaskWorker(t) - - be.EXPECT().CreateTaskHub(ctx).Return(nil).Once() - be.EXPECT().Start(ctx).Return(nil).Once() - orchWorker.EXPECT().Start(ctx).Return().Once() - actWorker.EXPECT().Start(ctx).Return().Once() - - w := backend.NewTaskHubWorker(be, orchWorker, actWorker, logger) - err := w.Start(ctx) - assert.NoError(t, err) -} - -func Test_TaskHubWorkerStopsDependencies(t *testing.T) { - ctx := context.Background() - - be := mocks.NewBackend(t) - orchWorker := mocks.NewTaskWorker(t) - actWorker := mocks.NewTaskWorker(t) - - be.EXPECT().Stop(ctx).Return(nil).Once() - orchWorker.EXPECT().StopAndDrain().Return().Once() - actWorker.EXPECT().StopAndDrain().Return().Once() - - w := backend.NewTaskHubWorker(be, orchWorker, actWorker, logger) - err := w.Shutdown(ctx) - assert.NoError(t, err) -} diff --git a/tests/tracing_test.go b/tests/tracing_test.go deleted file mode 100644 index 1fe62406..00000000 --- a/tests/tracing_test.go +++ /dev/null @@ -1,237 +0,0 @@ -package tests - -import ( - "fmt" - "sync" - "time" - - "github.com/stretchr/testify/assert" - "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/sdk/trace" - "go.opentelemetry.io/otel/sdk/trace/tracetest" - - "github.com/microsoft/durabletask-go/api" -) - -type ( - spanValidator func(t assert.TestingT, spans []trace.ReadOnlySpan, index int) - spanAttributeValidator func(t assert.TestingT, span trace.ReadOnlySpan) bool - spanEventValidator func(t assert.TestingT, span trace.ReadOnlySpan, eventIndex int) bool -) - -var ( - initTracingOnce sync.Once - sharedTraceExporter = tracetest.NewInMemoryExporter() -) - -func assertSpanSequence(t assert.TestingT, spans []trace.ReadOnlySpan, spanAsserts ...spanValidator) { - for i, f := range spanAsserts { - f(t, spans, i) - } -} - -// assertOrchestratorCreated validates a create_orchestration span -func assertOrchestratorCreated(name string, id api.InstanceID, optionalAsserts ...spanAttributeValidator) spanValidator { - spanName := fmt.Sprintf("create_orchestration||%s", name) - opts := []spanAttributeValidator{ - assertTaskType("orchestration"), - assertTaskName(name), - assertInstanceID(id), - } - opts = append(opts, optionalAsserts...) - return assertSpan(spanName, opts...) -} - -// assertOrchestratorCreated validates an orchestration span -func assertOrchestratorExecuted(name string, id api.InstanceID, status string, optionalAsserts ...spanAttributeValidator) spanValidator { - spanName := fmt.Sprintf("orchestration||%s", name) - opts := []spanAttributeValidator{ - assertTaskType("orchestration"), - assertTaskName(name), - assertInstanceID(id), - assertStatus(status), - } - opts = append(opts, optionalAsserts...) - return assertSpan(spanName, opts...) -} - -func assertActivity(name string, id api.InstanceID, taskID int64, optionalAsserts ...spanAttributeValidator) spanValidator { - spanName := fmt.Sprintf("activity||%s", name) - opts := []spanAttributeValidator{ - assertTaskType("activity"), - assertTaskName(name), - assertInstanceID(id), - assertTaskID(taskID), - } - opts = append(opts, optionalAsserts...) - return assertSpan(spanName, opts...) -} - -func assertTimer(id api.InstanceID, optionalAsserts ...spanAttributeValidator) spanValidator { - opts := []spanAttributeValidator{ - assertInstanceID(id), - assertTimerFired(), - } - opts = append(opts, optionalAsserts...) - return assertSpan("timer", opts...) -} - -func assertSpanEvents(eventAsserts ...spanEventValidator) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - if assert.Equal(t, len(eventAsserts), len(span.Events()), "unexpected number of span events") { - for i, f := range eventAsserts { - if !f(t, span, i) { - return false - } - } - } - return true - } -} - -func assertExternalEvent(eventName string, payloadSize int) spanEventValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan, eventIndex int) bool { - event := span.Events()[eventIndex] - hasMessage := assert.Equal(t, "Received external event", event.Name) - hasNameAttribute := assert.Contains(t, event.Attributes, attribute.KeyValue{ - Key: "name", - Value: attribute.StringValue(eventName), - }) - hasSizeAttribute := assert.Contains(t, event.Attributes, attribute.KeyValue{ - Key: "size", - Value: attribute.IntValue(payloadSize), - }) - return hasMessage && hasNameAttribute && hasSizeAttribute - } -} - -func assertSuspendedEvent() spanEventValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan, eventIndex int) bool { - event := span.Events()[eventIndex] - return assert.Equal(t, "Execution suspended", event.Name) - } -} - -func assertResumedEvent() spanEventValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan, eventIndex int) bool { - event := span.Events()[eventIndex] - return assert.Equal(t, "Execution resumed", event.Name) - } -} - -func assertSpan(name string, optionalAsserts ...spanAttributeValidator) spanValidator { - return func(t assert.TestingT, spans []trace.ReadOnlySpan, index int) { - if !doAssertSpan(t, spans, index, name, optionalAsserts...) { - fmt.Printf("span assertion for %s (index=%d) failed\n", name, index) - } - } -} - -func doAssertSpan(t assert.TestingT, spans []trace.ReadOnlySpan, index int, name string, optionalAsserts ...spanAttributeValidator) bool { - // array bounds check - if !assert.Lessf(t, index, len(spans), "%d spans were exported, but more were expected by the test", len(spans)) { - return false - } - - span := spans[index] - - // All spans have a name that we must validate - success := assert.Equal(t, name, span.Name()) - - // Optional validations that are span-specific - for _, optionalAssert := range optionalAsserts { - if !optionalAssert(t, span) { - success = false - } - } - - return success -} - -func assertTaskType(expectedTaskType string) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - return assert.Contains(t, span.Attributes(), attribute.KeyValue{ - Key: "durabletask.type", - Value: attribute.StringValue(expectedTaskType), - }) - } -} - -func assertTaskName(expectedTaskName string) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - return assert.Contains(t, span.Attributes(), attribute.KeyValue{ - Key: "durabletask.task.name", - Value: attribute.StringValue(expectedTaskName), - }) - } -} - -func assertTaskID(expectedTaskID int64) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - return assert.Contains(t, span.Attributes(), attribute.KeyValue{ - Key: "durabletask.task.task_id", - Value: attribute.Int64Value(expectedTaskID), - }) - } -} - -func assertInstanceID(expectedID api.InstanceID) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - return assert.Contains(t, span.Attributes(), attribute.KeyValue{ - Key: "durabletask.task.instance_id", - Value: attribute.StringValue(string(expectedID)), - }) - } -} - -func assertStatus(expectedStatus string) spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - return assert.Contains(t, span.Attributes(), attribute.KeyValue{ - Key: "durabletask.runtime_status", - Value: attribute.StringValue(expectedStatus), - }) - } -} - -func assertTimerFired() spanAttributeValidator { - return func(t assert.TestingT, span trace.ReadOnlySpan) bool { - var firedAtStr string - for _, a := range span.Attributes() { - if a.Key == "durabletask.fire_at" { - firedAtStr = a.Value.AsString() - break - } - } - - if assert.NotEmptyf(t, firedAtStr, "couldn't find the durabletask.fire_at attribute") { - // Ensure we can parse the value and that the value fits into a general range. - // Note that we're not attempting to validate a specific time. - firedAt, err := time.Parse(time.RFC3339, firedAtStr) - now := time.Now().UTC() - return assert.NoError(t, err) && - assert.Less(t, firedAt, now) && - assert.Greater(t, firedAt, now.Add(-1*time.Hour)) - } - - return false - } -} - -// initTracing configures in-memory OTel tracing and returns an exporter which can be used -// to examine the exported traces. We only want to look at exported traces because we do -// tricks to mark certain spans as non-exported (i.e. orchestration replays), and want -// to ensure that those spans are never actually exported. -func initTracing() *tracetest.InMemoryExporter { - // The global tracer provider can only be initialized once. - // Subsequent initializations will silently fail. - initTracingOnce.Do(func() { - processor := trace.NewSimpleSpanProcessor(sharedTraceExporter) - provider := trace.NewTracerProvider(trace.WithSpanProcessor(processor)) - otel.SetTracerProvider(provider) - }) - - // Reset the shared exporter so that new tests don't see traces from previous tests. - sharedTraceExporter.Reset() - return sharedTraceExporter -} diff --git a/tests/tracingtree/tracingtree.go b/tests/tracingtree/tracingtree.go new file mode 100644 index 00000000..4e112ed9 --- /dev/null +++ b/tests/tracingtree/tracingtree.go @@ -0,0 +1,76 @@ +// Package tracingtree provides the application-process OpenTelemetry plumbing +// used by the Durable Task Scheduler tracing tests: a process-wide in-memory +// exporter and the caller span whose W3C context the SDK propagates to DTS. +// Durable operation spans are emitted by DTS service-side, not by the Go worker. +package tracingtree + +import ( + "context" + "fmt" + "strings" + "sync" + + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + "go.opentelemetry.io/otel/trace" +) + +// attributeInstanceID is the instance-ID attribute DTS stamps on its +// service-side durable operation spans. It is used only in diagnostic output. +const attributeInstanceID = "durabletask.task.instance_id" + +var ( + initOnce sync.Once + sharedExporter = tracetest.NewInMemoryExporter() +) + +// Init installs an in-memory tracer provider as the global OTel provider and +// resets the previously collected spans. The global provider can only be +// installed once per process, so the exporter is shared and reset per test. +func Init() *tracetest.InMemoryExporter { + initOnce.Do(func() { + processor := sdktrace.NewSimpleSpanProcessor(sharedExporter) + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(processor))) + }) + sharedExporter.Reset() + return sharedExporter +} + +// StartCallerSpan starts the client span that stands in for the application +// code scheduling an orchestration. The orchestration's whole span tree must +// hang off it, so the span is asserted to be sampled: an unsampled caller span +// would silently stop propagating trace context. +func StartCallerSpan( + t require.TestingT, + tracerName string, + parent context.Context, + name string, +) (context.Context, trace.Span) { + if h, ok := t.(interface{ Helper() }); ok { + h.Helper() + } + callerCtx, span := otel.Tracer(tracerName).Start(parent, name, trace.WithSpanKind(trace.SpanKindClient)) + require.True(t, span.SpanContext().IsSampled(), "the caller span must be sampled for trace propagation") + return callerCtx, span +} + +// Describe renders exported spans for assertion failure messages. +func Describe(spans []sdktrace.ReadOnlySpan) string { + if len(spans) == 0 { + return "(none)" + } + descriptions := make([]string, 0, len(spans)) + for _, span := range spans { + instanceID := "" + for _, kv := range span.Attributes() { + if string(kv.Key) == attributeInstanceID { + instanceID = kv.Value.AsString() + break + } + } + descriptions = append(descriptions, fmt.Sprintf("%s(instance=%s)", span.Name(), instanceID)) + } + return strings.Join(descriptions, ", ") +} diff --git a/tests/worker_test.go b/tests/worker_test.go deleted file mode 100644 index c4a5a74c..00000000 --- a/tests/worker_test.go +++ /dev/null @@ -1,227 +0,0 @@ -package tests - -import ( - "context" - "sync/atomic" - "testing" - "time" - - "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/backend" - "github.com/microsoft/durabletask-go/internal/helpers" - "github.com/microsoft/durabletask-go/internal/protos" - "github.com/microsoft/durabletask-go/tests/mocks" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -// https://github.com/stretchr/testify/issues/519 -var ( - anyContext = mock.Anything -) - -func Test_TryProcessSingleOrchestrationWorkItem_BasicFlow(t *testing.T) { - ctx := context.Background() - wi := &backend.OrchestrationWorkItem{ - InstanceID: "test123", - NewEvents: []*protos.HistoryEvent{helpers.NewExecutionStartedEvent("MyOrch", "test123", nil, nil, nil, nil)}, - } - state := &backend.OrchestrationRuntimeState{} - result := &backend.ExecutionResults{Response: &protos.OrchestratorResponse{}} - - completed := atomic.Bool{} - be := mocks.NewBackend(t) - be.EXPECT().GetOrchestrationWorkItem(anyContext).Return(wi, nil).Once() - be.EXPECT().GetOrchestrationRuntimeState(anyContext, wi).Return(state, nil).Once() - be.EXPECT().CompleteOrchestrationWorkItem(anyContext, wi).RunAndReturn(func(ctx context.Context, owi *backend.OrchestrationWorkItem) error { - completed.Store(true) - return nil - }).Once() - - ex := mocks.NewExecutor(t) - ex.EXPECT().ExecuteOrchestrator(anyContext, wi.InstanceID, state.OldEvents(), mock.Anything).Return(result, nil).Once() - - worker := backend.NewOrchestrationWorker(be, ex, logger) - ok, err := worker.ProcessNext(ctx) - // Successfully processing a work-item should result in a nil error - assert.Nil(t, err) - assert.True(t, ok) - - require.EventuallyWithT(t, func(collect *assert.CollectT) { - if !completed.Load() { - collect.Errorf("process next not called CompleteOrchestrationWorkItem yet") - } - }, 1*time.Second, 100*time.Millisecond) - - worker.StopAndDrain() -} - -func Test_TryProcessSingleOrchestrationWorkItem_NoWorkItems(t *testing.T) { - ctx := context.Background() - be := mocks.NewBackend(t) - be.EXPECT().GetOrchestrationWorkItem(anyContext).Return(nil, backend.ErrNoWorkItems).Once() - - w := backend.NewOrchestrationWorker(be, nil, logger) - ok, err := w.ProcessNext(ctx) - assert.Nil(t, err) - assert.False(t, ok) -} - -func Test_TryProcessSingleOrchestrationWorkItem_ExecutionStartedAndCompleted(t *testing.T) { - ctx := context.Background() - iid := api.InstanceID("test123") - - // Simulate getting an ExecutionStarted message from the orchestration queue - startEvent := helpers.NewExecutionStartedEvent("MyOrchestration", string(iid), nil, nil, nil, nil) - wi := &backend.OrchestrationWorkItem{ - InstanceID: iid, - NewEvents: []*protos.HistoryEvent{startEvent}, - } - - // Empty orchestration runtime state since we're starting a new execution from scratch - state := backend.NewOrchestrationRuntimeState(iid, []*protos.HistoryEvent{}) - - be := mocks.NewBackend(t) - be.EXPECT().GetOrchestrationWorkItem(anyContext).Return(wi, nil).Once() - be.EXPECT().GetOrchestrationRuntimeState(anyContext, wi).Return(state, nil).Once() - - ex := mocks.NewExecutor(t) - - // Return an execution completed action to simulate the completion of the orchestration (a no-op) - resultValue := "done" - result := &backend.ExecutionResults{ - Response: &protos.OrchestratorResponse{ - Actions: []*protos.OrchestratorAction{ - helpers.NewCompleteOrchestrationAction( - -1, - protos.OrchestrationStatus_ORCHESTRATION_STATUS_COMPLETED, - wrapperspb.String(resultValue), - nil, - nil), - }, - }, - } - - // Execute should be called with an empty oldEvents list. NewEvents should contain two items, - // but there doesn't seem to be a good way to assert this. - ex.EXPECT().ExecuteOrchestrator(anyContext, iid, []*protos.HistoryEvent{}, mock.Anything).Return(result, nil).Once() - - // After execution, the Complete action should be called - completed := atomic.Bool{} - be.EXPECT().CompleteOrchestrationWorkItem(anyContext, wi).RunAndReturn(func(ctx context.Context, owi *backend.OrchestrationWorkItem) error { - completed.Store(true) - return nil - }).Once() - - // Set up and run the test - worker := backend.NewOrchestrationWorker(be, ex, logger) - ok, err := worker.ProcessNext(ctx) - // Successfully processing a work-item should result in a nil error - assert.Nil(t, err) - assert.True(t, ok) - - require.EventuallyWithT(t, func(collect *assert.CollectT) { - if !completed.Load() { - collect.Errorf("process next not called CompleteOrchestrationWorkItem yet") - } - }, 1*time.Second, 100*time.Millisecond) - - worker.StopAndDrain() -} - -func Test_TaskWorker(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tp := mocks.NewTestTaskPocessor("test") - tp.UnblockProcessing() - - first := backend.ActivityWorkItem{ - SequenceNumber: 1, - } - second := backend.ActivityWorkItem{ - SequenceNumber: 2, - } - tp.AddWorkItems(first, second) - - worker := backend.NewTaskWorker(tp, logger) - - worker.Start(ctx) - - require.EventuallyWithT(t, func(collect *assert.CollectT) { - if len(tp.PendingWorkItems()) == 0 { - return - } - collect.Errorf("work items not consumed yet") - }, 500*time.Millisecond, 100*time.Millisecond) - - require.Len(t, tp.PendingWorkItems(), 0) - require.Len(t, tp.AbandonedWorkItems(), 0) - require.Len(t, tp.CompletedWorkItems(), 2) - require.Equal(t, first, tp.CompletedWorkItems()[0]) - require.Equal(t, second, tp.CompletedWorkItems()[1]) - - drainFinished := make(chan bool) - go func() { - worker.StopAndDrain() - drainFinished <- true - }() - - select { - case <-drainFinished: - return - case <-time.After(1 * time.Second): - t.Fatalf("worker stop and drain not finished within timeout") - } - -} - -func Test_StartAndStop(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - tp := mocks.NewTestTaskPocessor("test") - tp.BlockProcessing() - - first := backend.ActivityWorkItem{ - SequenceNumber: 1, - } - second := backend.ActivityWorkItem{ - SequenceNumber: 2, - } - tp.AddWorkItems(first, second) - - worker := backend.NewTaskWorker(tp, logger) - - worker.Start(ctx) - - require.EventuallyWithT(t, func(collect *assert.CollectT) { - if len(tp.PendingWorkItems()) == 1 { - return - } - collect.Errorf("first work item not consumed yet") - }, 500*time.Millisecond, 100*time.Millisecond) - - // due to the configuration of the TestTaskProcessor, now the work item is blocked on ProcessWorkItem until the context is cancelled - - drainFinished := make(chan bool) - go func() { - worker.StopAndDrain() - drainFinished <- true - }() - - select { - case <-drainFinished: - return - case <-time.After(1 * time.Second): - t.Fatalf("worker stop and drain not finished within timeout") - } - - require.Len(t, tp.PendingWorkItems(), 1) - require.Equal(t, second, tp.PendingWorkItems()[0]) - require.Len(t, tp.AbandonedWorkItems(), 1) - require.Equal(t, first, tp.AbandonedWorkItems()[0]) - require.Len(t, tp.CompletedWorkItems(), 0) -} diff --git a/vendored/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH b/vendored/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH index 049687a0..21e3e6f9 100644 --- a/vendored/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH +++ b/vendored/durabletask-protobuf/PROTO_SOURCE_COMMIT_HASH @@ -1,4 +1,4 @@ Source: https://github.com/microsoft/durabletask-protobuf Branch: main -Commit: 4207e1dbd14cedc268f69c3befee60fcaad19367 -URL: https://github.com/microsoft/durabletask-protobuf/blob/4207e1dbd14cedc268f69c3befee60fcaad19367/protos/orchestrator_service.proto \ No newline at end of file +Commit: 3145f9337fca9de57d2f89a6ff6f07150d34f1c2 +URL: https://github.com/microsoft/durabletask-protobuf/blob/3145f9337fca9de57d2f89a6ff6f07150d34f1c2/protos/orchestrator_service.proto diff --git a/vendored/durabletask-protobuf/protos/orchestrator_service.proto b/vendored/durabletask-protobuf/protos/orchestrator_service.proto index 2ca37553..3d9194ac 100644 --- a/vendored/durabletask-protobuf/protos/orchestrator_service.proto +++ b/vendored/durabletask-protobuf/protos/orchestrator_service.proto @@ -11,6 +11,7 @@ import "google/protobuf/timestamp.proto"; import "google/protobuf/duration.proto"; import "google/protobuf/wrappers.proto"; import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; message OrchestrationInstance { string instanceId = 1; @@ -23,6 +24,8 @@ message ActivityRequest { google.protobuf.StringValue input = 3; OrchestrationInstance orchestrationInstance = 4; int32 taskId = 5; + TraceContext parentTraceContext = 6; + map tags = 7; } message ActivityResponse { @@ -30,6 +33,7 @@ message ActivityResponse { int32 taskId = 2; google.protobuf.StringValue result = 3; TaskFailureDetails failureDetails = 4; + string completionToken = 5; } message TaskFailureDetails { @@ -38,6 +42,7 @@ message TaskFailureDetails { google.protobuf.StringValue stackTrace = 3; TaskFailureDetails innerFailure = 4; bool isNonRetriable = 5; + map properties = 6; } enum OrchestrationStatus { @@ -73,6 +78,7 @@ message ExecutionStartedEvent { google.protobuf.Timestamp scheduledStartTimestamp = 6; TraceContext parentTraceContext = 7; google.protobuf.StringValue orchestrationSpanID = 8; + map tags = 9; } message ExecutionCompletedEvent { @@ -91,6 +97,7 @@ message TaskScheduledEvent { google.protobuf.StringValue version = 2; google.protobuf.StringValue input = 3; TraceContext parentTraceContext = 4; + map tags = 5; } message TaskCompletedEvent { @@ -109,6 +116,7 @@ message SubOrchestrationInstanceCreatedEvent { google.protobuf.StringValue version = 3; google.protobuf.StringValue input = 4; TraceContext parentTraceContext = 5; + map tags = 6; } message SubOrchestrationInstanceCompletedEvent { @@ -169,6 +177,63 @@ message ExecutionResumedEvent { google.protobuf.StringValue input = 1; } +message EntityOperationSignaledEvent { + string requestId = 1; + string operation = 2; + google.protobuf.Timestamp scheduledTime = 3; + google.protobuf.StringValue input = 4; + google.protobuf.StringValue targetInstanceId = 5; // used only within histories, null in messages +} + +message EntityOperationCalledEvent { + string requestId = 1; + string operation = 2; + google.protobuf.Timestamp scheduledTime = 3; + google.protobuf.StringValue input = 4; + google.protobuf.StringValue parentInstanceId = 5; // used only within messages, null in histories + google.protobuf.StringValue parentExecutionId = 6; // used only within messages, null in histories + google.protobuf.StringValue targetInstanceId = 7; // used only within histories, null in messages +} + +message EntityLockRequestedEvent { + string criticalSectionId = 1; + repeated string lockSet = 2; + int32 position = 3; + google.protobuf.StringValue parentInstanceId = 4; // used only within messages, null in histories +} + +message EntityOperationCompletedEvent { + string requestId = 1; + google.protobuf.StringValue output = 2; +} + +message EntityOperationFailedEvent { + string requestId = 1; + TaskFailureDetails failureDetails = 2; +} + +message EntityUnlockSentEvent { + string criticalSectionId = 1; + google.protobuf.StringValue parentInstanceId = 2; // used only within messages, null in histories + google.protobuf.StringValue targetInstanceId = 3; // used only within histories, null in messages +} + +message EntityLockGrantedEvent { + string criticalSectionId = 1; +} + +message ExecutionRewoundEvent { + google.protobuf.StringValue reason = 1; + google.protobuf.StringValue parentExecutionId = 2; // used only for rewinding suborchestrations, null otherwise + google.protobuf.StringValue instanceId = 3; // used only for rewinding suborchestrations, null otherwise + TraceContext parentTraceContext = 4; // used only for rewinding suborchestrations, null otherwise + google.protobuf.StringValue name = 5; // used by DTS backend only + google.protobuf.StringValue version = 6; // used by DTS backend only + google.protobuf.StringValue input = 7; // used by DTS backend only + ParentInstanceInfo parentInstance = 8; // used by DTS backend only + map tags = 9; // used by DTS backend only +} + message HistoryEvent { int32 eventId = 1; google.protobuf.Timestamp timestamp = 2; @@ -193,6 +258,14 @@ message HistoryEvent { ContinueAsNewEvent continueAsNew = 20; ExecutionSuspendedEvent executionSuspended = 21; ExecutionResumedEvent executionResumed = 22; + EntityOperationSignaledEvent entityOperationSignaled = 23; + EntityOperationCalledEvent entityOperationCalled = 24; + EntityOperationCompletedEvent entityOperationCompleted = 25; + EntityOperationFailedEvent entityOperationFailed = 26; + EntityLockRequestedEvent entityLockRequested = 27; + EntityLockGrantedEvent entityLockGranted = 28; + EntityUnlockSentEvent entityUnlockSent = 29; + ExecutionRewoundEvent executionRewound = 30; } } @@ -200,6 +273,8 @@ message ScheduleTaskAction { string name = 1; google.protobuf.StringValue version = 2; google.protobuf.StringValue input = 3; + map tags = 4; + TraceContext parentTraceContext = 5; } message CreateSubOrchestrationAction { @@ -207,6 +282,8 @@ message CreateSubOrchestrationAction { string name = 2; google.protobuf.StringValue version = 3; google.protobuf.StringValue input = 4; + TraceContext parentTraceContext = 5; + map tags = 6; } message CreateTimerAction { @@ -226,6 +303,7 @@ message CompleteOrchestrationAction { google.protobuf.StringValue newVersion = 4; repeated HistoryEvent carryoverEvents = 5; TaskFailureDetails failureDetails = 6; + map tags = 7; } message TerminateOrchestrationAction { @@ -234,6 +312,19 @@ message TerminateOrchestrationAction { bool recurse = 3; } +message SendEntityMessageAction { + oneof EntityMessageType { + EntityOperationSignaledEvent entityOperationSignaled = 1; + EntityOperationCalledEvent entityOperationCalled = 2; + EntityLockRequestedEvent entityLockRequested = 3; + EntityUnlockSentEvent entityUnlockSent = 4; + } +} + +message RewindOrchestrationAction { + repeated HistoryEvent newHistory = 1; +} + message OrchestratorAction { int32 id = 1; oneof orchestratorActionType { @@ -243,21 +334,50 @@ message OrchestratorAction { SendEventAction sendEvent = 5; CompleteOrchestrationAction completeOrchestration = 6; TerminateOrchestrationAction terminateOrchestration = 7; + SendEntityMessageAction sendEntityMessage = 8; + RewindOrchestrationAction rewindOrchestration = 9; } } +message OrchestrationTraceContext { + google.protobuf.StringValue spanID = 1; + google.protobuf.Timestamp spanStartTime = 2; +} + message OrchestratorRequest { string instanceId = 1; google.protobuf.StringValue executionId = 2; repeated HistoryEvent pastEvents = 3; repeated HistoryEvent newEvents = 4; OrchestratorEntityParameters entityParameters = 5; + bool requiresHistoryStreaming = 6; + map properties = 7; + + OrchestrationTraceContext orchestrationTraceContext = 8; } message OrchestratorResponse { string instanceId = 1; repeated OrchestratorAction actions = 2; google.protobuf.StringValue customStatus = 3; + string completionToken = 4; + + // The number of work item events that were processed by the orchestrator. + // This field is optional. If not set, the service should assume that the orchestrator processed all events. + google.protobuf.Int32Value numEventsProcessed = 5; + OrchestrationTraceContext orchestrationTraceContext = 6; + + // Whether or not a history is required to complete the original OrchestratorRequest and none was provided. + bool requiresHistory = 7; + + /* Chunking logic has since been deprecated and fields related to it are marked as such */ + + // True if this is a partial (chunked) completion. The backend must keep the work item open until the final chunk (isPartial=false). + bool isPartial = 8 [deprecated=true]; + + // Zero-based position of the current chunk within a chunked completion sequence. + // This field is omitted for non-chunked completions. + google.protobuf.Int32Value chunkIndex = 9 [deprecated=true]; } message CreateInstanceRequest { @@ -267,17 +387,15 @@ message CreateInstanceRequest { google.protobuf.StringValue input = 4; google.protobuf.Timestamp scheduledStartTimestamp = 5; OrchestrationIdReusePolicy orchestrationIdReusePolicy = 6; + google.protobuf.StringValue executionId = 7; + map tags = 8; + TraceContext parentTraceContext = 9; + google.protobuf.Timestamp requestTime = 10; } message OrchestrationIdReusePolicy { - repeated OrchestrationStatus operationStatus = 1; - CreateOrchestrationAction action = 2; -} - -enum CreateOrchestrationAction { - ERROR = 0; - IGNORE = 1; - TERMINATE = 2; + repeated OrchestrationStatus replaceableStatus = 1; + reserved 2; } message CreateInstanceResponse { @@ -315,6 +433,10 @@ message OrchestrationState { google.protobuf.StringValue output = 9; google.protobuf.StringValue customStatus = 10; TaskFailureDetails failureDetails = 11; + google.protobuf.StringValue executionId = 12; + google.protobuf.Timestamp completedTimestamp = 13; + google.protobuf.StringValue parentInstanceId = 14; + map tags = 15; } message RaiseEventRequest { @@ -375,22 +497,49 @@ message QueryInstancesResponse { google.protobuf.StringValue continuationToken = 2; } +message ListInstanceIdsRequest { + repeated OrchestrationStatus runtimeStatus = 1; + google.protobuf.Timestamp completedTimeFrom = 2; + google.protobuf.Timestamp completedTimeTo = 3; + int32 pageSize = 4; + google.protobuf.StringValue lastInstanceKey = 5; +} + +message ListInstanceIdsResponse { + repeated string instanceIds = 1; + google.protobuf.StringValue lastInstanceKey = 2; +} + message PurgeInstancesRequest { oneof request { string instanceId = 1; PurgeInstanceFilter purgeInstanceFilter = 2; + InstanceBatch instanceBatch = 4; } bool recursive = 3; + // used in the case when an instanceId is specified to determine if the purge request is for an orchestration (as opposed to an entity) + bool isOrchestration = 5; } message PurgeInstanceFilter { google.protobuf.Timestamp createdTimeFrom = 1; google.protobuf.Timestamp createdTimeTo = 2; repeated OrchestrationStatus runtimeStatus = 3; + google.protobuf.Duration timeout = 4; } message PurgeInstancesResponse { int32 deletedInstanceCount = 1; + google.protobuf.BoolValue isComplete = 2; +} + +message RestartInstanceRequest { + string instanceId = 1; + bool restartWithNewInstanceId = 2; +} + +message RestartInstanceResponse { + string instanceId = 1; } message CreateTaskHubRequest { @@ -415,26 +564,25 @@ message SignalEntityRequest { google.protobuf.StringValue input = 3; string requestId = 4; google.protobuf.Timestamp scheduledTime = 5; + TraceContext parentTraceContext = 6; + google.protobuf.Timestamp requestTime = 7; } message SignalEntityResponse { - // no payload + // no payload } -message GetEntityRequest -{ +message GetEntityRequest { string instanceId = 1; bool includeState = 2; } -message GetEntityResponse -{ +message GetEntityResponse { bool exists = 1; EntityMetadata entity = 2; } -message EntityQuery -{ +message EntityQuery { google.protobuf.StringValue instanceIdStartsWith = 1; google.protobuf.Timestamp lastModifiedFrom = 2; google.protobuf.Timestamp lastModifiedTo = 3; @@ -444,19 +592,16 @@ message EntityQuery google.protobuf.StringValue continuationToken = 7; } -message QueryEntitiesRequest -{ +message QueryEntitiesRequest { EntityQuery query = 1; } -message QueryEntitiesResponse -{ +message QueryEntitiesResponse { repeated EntityMetadata entities = 1; google.protobuf.StringValue continuationToken = 2; } -message EntityMetadata -{ +message EntityMetadata { string instanceId = 1; google.protobuf.Timestamp lastModifiedTime = 2; int32 backlogQueueSize = 3; @@ -464,22 +609,19 @@ message EntityMetadata google.protobuf.StringValue serializedState = 5; } -message CleanEntityStorageRequest -{ +message CleanEntityStorageRequest { google.protobuf.StringValue continuationToken = 1; bool removeEmptyEntities = 2; bool releaseOrphanedLocks = 3; } -message CleanEntityStorageResponse -{ +message CleanEntityStorageResponse { google.protobuf.StringValue continuationToken = 1; int32 emptyEntitiesRemoved = 2; int32 orphanedLocksReleased = 3; } -message OrchestratorEntityParameters -{ +message OrchestratorEntityParameters { google.protobuf.Duration entityMessageReorderWindow = 1; } @@ -487,6 +629,7 @@ message EntityBatchRequest { string instanceId = 1; google.protobuf.StringValue entityState = 2; repeated OperationRequest operations = 3; + map properties = 4; } message EntityBatchResult { @@ -494,12 +637,24 @@ message EntityBatchResult { repeated OperationAction actions = 2; google.protobuf.StringValue entityState = 3; TaskFailureDetails failureDetails = 4; + string completionToken = 5; + repeated OperationInfo operationInfos = 6; // used only with DTS + // Whether or not an entity state is required to complete the original EntityBatchRequest and none was provided. + bool requiresState = 7; +} + +message EntityRequest { + string instanceId = 1; + string executionId = 2; + google.protobuf.StringValue entityState = 3; // null if entity does not exist + repeated HistoryEvent operationRequests = 4; } message OperationRequest { string operation = 1; string requestId = 2; google.protobuf.StringValue input = 3; + TraceContext traceContext = 4; } message OperationResult { @@ -509,12 +664,21 @@ message OperationResult { } } +message OperationInfo { + string requestId = 1; + OrchestrationInstance responseDestination = 2; // null for signals +} + message OperationResultSuccess { google.protobuf.StringValue result = 1; + google.protobuf.Timestamp startTimeUtc = 2; + google.protobuf.Timestamp endTimeUtc = 3; } message OperationResultFailure { TaskFailureDetails failureDetails = 1; + google.protobuf.Timestamp startTimeUtc = 2; + google.protobuf.Timestamp endTimeUtc = 3; } message OperationAction { @@ -530,6 +694,8 @@ message SendSignalAction { string name = 2; google.protobuf.StringValue input = 3; google.protobuf.Timestamp scheduledTime = 4; + google.protobuf.Timestamp requestTime = 5; + TraceContext parentTraceContext = 6; } message StartNewOrchestrationAction { @@ -538,6 +704,43 @@ message StartNewOrchestrationAction { google.protobuf.StringValue version = 3; google.protobuf.StringValue input = 4; google.protobuf.Timestamp scheduledTime = 5; + google.protobuf.Timestamp requestTime = 6; + TraceContext parentTraceContext = 7; +} + +message AbandonActivityTaskRequest { + string completionToken = 1; +} + +message AbandonActivityTaskResponse { + // Empty. +} + +message AbandonOrchestrationTaskRequest { + string completionToken = 1; +} + +message AbandonOrchestrationTaskResponse { + // Empty. +} + +message AbandonEntityTaskRequest { + string completionToken = 1; +} + +message AbandonEntityTaskResponse { + // Empty. +} + +message SkipGracefulOrchestrationTerminationsRequest { + InstanceBatch instanceBatch = 1; + google.protobuf.StringValue reason = 2; +} + +message SkipGracefulOrchestrationTerminationsResponse { + // Those instances which could not be terminated because they had locked entities at the time of this termination call, + // are already in a terminal state (completed, failed, terminated, etc.), are not orchestrations, or do not exist (i.e. have been purged) + repeated string unterminatedInstanceIds = 1; } service TaskHubSidecarService { @@ -553,18 +756,21 @@ service TaskHubSidecarService { // Rewinds an orchestration instance to last known good state and replays from there. rpc RewindInstance(RewindInstanceRequest) returns (RewindInstanceResponse); + // Restarts an orchestration instance. + rpc RestartInstance(RestartInstanceRequest) returns (RestartInstanceResponse); + // Waits for an orchestration instance to reach a running or completion state. rpc WaitForInstanceStart(GetInstanceRequest) returns (GetInstanceResponse); - + // Waits for an orchestration instance to reach a completion state (completed, failed, terminated, etc.). rpc WaitForInstanceCompletion(GetInstanceRequest) returns (GetInstanceResponse); // Raises an event to a running orchestration instance. rpc RaiseEvent(RaiseEventRequest) returns (RaiseEventResponse); - + // Terminates a running orchestration instance. rpc TerminateInstance(TerminateRequest) returns (TerminateResponse); - + // Suspends a running orchestration instance. rpc SuspendInstance(SuspendRequest) returns (SuspendResponse); @@ -574,6 +780,9 @@ service TaskHubSidecarService { // rpc DeleteInstance(DeleteInstanceRequest) returns (DeleteInstanceResponse); rpc QueryInstances(QueryInstancesRequest) returns (QueryInstancesResponse); + + rpc ListInstanceIds(ListInstanceIdsRequest) returns (ListInstanceIdsResponse); + rpc PurgeInstances(PurgeInstancesRequest) returns (PurgeInstancesResponse); rpc GetWorkItems(GetWorkItemsRequest) returns (stream WorkItem); @@ -581,6 +790,9 @@ service TaskHubSidecarService { rpc CompleteOrchestratorTask(OrchestratorResponse) returns (CompleteTaskResponse); rpc CompleteEntityTask(EntityBatchResult) returns (CompleteTaskResponse); + // Gets the history of an orchestration instance as a stream of events. + rpc StreamInstanceHistory(StreamInstanceHistoryRequest) returns (stream HistoryChunk); + // Deletes and Creates the necessary resources for the orchestration service and the instance store rpc CreateTaskHub(CreateTaskHubRequest) returns (CreateTaskHubResponse); @@ -598,20 +810,102 @@ service TaskHubSidecarService { // clean entity storage rpc CleanEntityStorage(CleanEntityStorageRequest) returns (CleanEntityStorageResponse); + + // Abandons a single work item + rpc AbandonTaskActivityWorkItem(AbandonActivityTaskRequest) returns (AbandonActivityTaskResponse); + + // Abandon an orchestration work item + rpc AbandonTaskOrchestratorWorkItem(AbandonOrchestrationTaskRequest) returns (AbandonOrchestrationTaskResponse); + + // Abandon an entity work item + rpc AbandonTaskEntityWorkItem(AbandonEntityTaskRequest) returns (AbandonEntityTaskResponse); + + // "Skip" graceful termination of orchestrations by immediately changing their status in storage to "terminated". + // Note that a maximum of 500 orchestrations can be terminated at a time using this method. + rpc SkipGracefulOrchestrationTerminations(SkipGracefulOrchestrationTerminationsRequest) returns (SkipGracefulOrchestrationTerminationsResponse); } message GetWorkItemsRequest { - // No parameters currently + int32 maxConcurrentOrchestrationWorkItems = 1; + int32 maxConcurrentActivityWorkItems = 2; + int32 maxConcurrentEntityWorkItems = 3; + + repeated WorkerCapability capabilities = 10; + WorkItemFilters workItemFilters = 11; +} + +enum WorkerCapability { + WORKER_CAPABILITY_UNSPECIFIED = 0; + + // Indicates that the worker is capable of streaming instance history as a more optimized + // alternative to receiving the full history embedded in the orchestrator work-item. + // When set, the service may return work items without any history events as an optimization. + // It is strongly recommended that all SDKs support this capability. + WORKER_CAPABILITY_HISTORY_STREAMING = 1; + + // Indicates that the worker supports scheduled tasks. + // The service may send schedule-triggered orchestration work items, + // and the worker must handle them, including the scheduledTime field. + WORKER_CAPABILITY_SCHEDULED_TASKS = 2; + + // Signals that the worker can handle large payloads stored externally (e.g., Blob Storage). + // Work items may contain URI references instead of inline data, and the worker must fetch them. + // This avoids message size limits and reduces network overhead. + WORKER_CAPABILITY_LARGE_PAYLOADS = 3; +} + +message WorkItemFilters { + repeated OrchestrationFilter orchestrations = 1; + repeated ActivityFilter activities = 2; + repeated EntityFilter entities = 3; +} + +message OrchestrationFilter { + string name = 1; + repeated string versions = 2; +} + +message ActivityFilter { + string name = 1; + repeated string versions = 2; +} + +message EntityFilter { + string name = 1; } message WorkItem { oneof request { OrchestratorRequest orchestratorRequest = 1; ActivityRequest activityRequest = 2; - EntityBatchRequest entityRequest = 3; + EntityBatchRequest entityRequest = 3; // (older) used by orchestration services implementations + HealthPing healthPing = 4; + EntityRequest entityRequestV2 = 5; // (newer) used by backend service implementations } + string completionToken = 10; } message CompleteTaskResponse { // No payload -} \ No newline at end of file +} + +message HealthPing { + // No payload +} + +message StreamInstanceHistoryRequest { + string instanceId = 1; + google.protobuf.StringValue executionId = 2; + + // When set to true, the service may return a more optimized response suitable for workers. + bool forWorkItemProcessing = 3; +} + +message HistoryChunk { + repeated HistoryEvent events = 1; +} + +message InstanceBatch { + // A maximum of 500 instance IDs can be provided in this list. + repeated string instanceIds = 1; +} diff --git a/vendored/durabletask-protobuf/update-proto.sh b/vendored/durabletask-protobuf/update-proto.sh index b5b22ab5..ee92ea36 100755 --- a/vendored/durabletask-protobuf/update-proto.sh +++ b/vendored/durabletask-protobuf/update-proto.sh @@ -23,7 +23,7 @@ HASH_FILE="${SCRIPT_DIR}/PROTO_SOURCE_COMMIT_HASH" mkdir -p "${PROTO_DIR}" -CURL_AUTH=() +CURL_AUTH=(-H "Accept: application/vnd.github.v3+json") if [[ -n "${GITHUB_TOKEN:-}" ]]; then CURL_AUTH=(-H "Authorization: Bearer ${GITHUB_TOKEN}") fi From 2b8601ee231aed709dea03d78b6b2a0c0c95c1bc Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 10:47:23 -0700 Subject: [PATCH 02/26] refactor: remove redundant runtime and test complexity Apply the 14 approved complexity cuts while preserving replay, wire formats, credential errors, and export retry behavior. Add focused equivalence coverage for failure properties and schedule JSON. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../analysis/orchestratorgo/registration.go | 35 +---- durabletaskscheduler/connection.go | 48 ++----- durabletaskscheduler/scheduled_tasks.go | 78 +++------- durabletaskscheduler/scheduled_tasks_test.go | 67 +++++++++ exporthistory/activities.go | 26 +--- exporthistory/activities_test.go | 39 ++--- exporthistory/orchestrations.go | 30 +--- exporthistory/storage.go | 18 +-- internal/failure/failure.go | 28 +--- internal/failure/failure_test.go | 25 ++++ internal/helpers/entity.go | 14 -- task/entity_dispatch_test.go | 51 ------- task/orchestrator.go | 56 ++------ task/orchestrator_test.go | 133 +++--------------- .../durabletaskscheduler/dts_emulator_test.go | 107 ++++---------- .../dts_tracing_tree_test.go | 28 +--- tests/failurechain/failurechain.go | 57 +------- 17 files changed, 220 insertions(+), 620 deletions(-) diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/registration.go b/cmd/orchestratorvet/analysis/orchestratorgo/registration.go index f0405708..bb2ea74d 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/registration.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/registration.go @@ -47,21 +47,6 @@ var registrationShapes = map[string]registrationShape{ "AddActivityNVersion": {kind: activityKind, nameIndex: 0, handlerIndex: 2, versionIndex: 1, arity: 3}, } -// registration is a single proven call to a task.TaskRegistry Add* method. -type registration struct { - // name is the registered task name when it can be proven statically. - name string - // nameKnown reports whether name was proven rather than guessed. - nameKnown bool - // version is the registered version when it is a string literal. - version string - // versionKnown reports whether the registration's version was proven. A - // version computed at runtime leaves it false, which keeps the registration - // out of duplicate detection: two calls with different computed versions - // are not a conflict. - versionKnown bool -} - // registrySet is the whole-package view of task.TaskRegistry registrations. type registrySet struct { // names holds the lowercased proven names per namespace. @@ -145,30 +130,24 @@ func collectRegistrations(pass *analysis.Pass, index *packageIndex) *registrySet // name to the registry and cannot conflict with another registration. version, versionKnown, versionRejected := registrationVersion(pass, call, shape) name, nameKnown, nameRejected := registrationName(pass, call, shape, handlerNode) - entry := registration{ - name: name, - nameKnown: nameKnown, - version: version, - versionKnown: versionKnown, - } rejected := versionRejected || nameRejected - if entry.nameKnown { + if nameKnown { // A rejected call registers nothing, but its name is still recorded // so the unresolved-name check does not pile a second diagnostic // onto a call site whose registration was already reported here. - set.names[shape.kind][strings.ToLower(entry.name)] = struct{}{} + set.names[shape.kind][strings.ToLower(name)] = struct{}{} // A registration whose version is computed at runtime may land on // any key, so it neither proves nor disproves a conflict. scope := index.registrationScopes[call] if scope != nil && isRegistryConstructor(pass, index.singleValue(receiver)) && - !index.unstableRegistries[receiver] && entry.versionKnown && !rejected { + !index.unstableRegistries[receiver] && versionKnown && !rejected { key := duplicateKey{ registry: receiver, scope: scope, kind: shape.kind, - name: strings.ToLower(entry.name), - version: strings.ToLower(entry.version), + name: strings.ToLower(name), + version: strings.ToLower(version), } if duplicates[key] { pass.Reportf( @@ -176,7 +155,7 @@ func collectRegistrations(pass *analysis.Pass, index *packageIndex) *registrySet "%s %q is registered more than once on the same task.TaskRegistry; "+ "the duplicate registration returns an error", shape.kind, - entry.name, + name, ) } duplicates[key] = true @@ -186,7 +165,7 @@ func collectRegistrations(pass *analysis.Pass, index *packageIndex) *registrySet } if handlerObj != nil && shape.kind == orchestratorKind { - set.orchestratorObjects[handlerObj] = entry.name + set.orchestratorObjects[handlerObj] = name } if shape.kind == orchestratorKind && handlerNode != nil && !seenRoots[handlerNode] { diff --git a/durabletaskscheduler/connection.go b/durabletaskscheduler/connection.go index 081c7e5d..d487884c 100644 --- a/durabletaskscheduler/connection.go +++ b/durabletaskscheduler/connection.go @@ -313,74 +313,52 @@ func normalizeAdditionallyAllowedTenants(tenants []string) []string { } func newAzureIdentityCredential(spec credentialSpec) (azcore.TokenCredential, error) { + var credential azcore.TokenCredential + var err error switch spec.authentication { case AuthenticationDefaultAzure: - credential, err := azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ + credential, err = azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ TenantID: spec.tenantID, AdditionallyAllowedTenants: spec.additionallyAllowedTenants, }) - if err != nil { - return nil, fmt.Errorf("failed to create DefaultAzureCredential: %w", err) - } - return credential, nil case AuthenticationManagedIdentity: credentialOptions := &azidentity.ManagedIdentityCredentialOptions{} if spec.clientID != "" { credentialOptions.ID = azidentity.ClientID(spec.clientID) } - credential, err := azidentity.NewManagedIdentityCredential(credentialOptions) - if err != nil { - return nil, fmt.Errorf("failed to create ManagedIdentityCredential: %w", err) - } - return credential, nil + credential, err = azidentity.NewManagedIdentityCredential(credentialOptions) case AuthenticationWorkloadIdentity: - credential, err := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + credential, err = azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ ClientID: spec.clientID, TenantID: spec.tenantID, TokenFilePath: spec.tokenFilePath, AdditionallyAllowedTenants: spec.additionallyAllowedTenants, }) - if err != nil { - return nil, fmt.Errorf("failed to create WorkloadIdentityCredential: %w", err) - } - return credential, nil case AuthenticationEnvironment: - credential, err := azidentity.NewEnvironmentCredential(nil) - if err != nil { - return nil, fmt.Errorf("failed to create EnvironmentCredential: %w", err) - } - return credential, nil + credential, err = azidentity.NewEnvironmentCredential(nil) case AuthenticationAzureCLI: - credential, err := azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{ + credential, err = azidentity.NewAzureCLICredential(&azidentity.AzureCLICredentialOptions{ TenantID: spec.tenantID, AdditionallyAllowedTenants: spec.additionallyAllowedTenants, }) - if err != nil { - return nil, fmt.Errorf("failed to create AzureCLICredential: %w", err) - } - return credential, nil case AuthenticationAzurePowerShell: - credential, err := azidentity.NewAzurePowerShellCredential(&azidentity.AzurePowerShellCredentialOptions{ + credential, err = azidentity.NewAzurePowerShellCredential(&azidentity.AzurePowerShellCredentialOptions{ TenantID: spec.tenantID, AdditionallyAllowedTenants: spec.additionallyAllowedTenants, }) - if err != nil { - return nil, fmt.Errorf("failed to create AzurePowerShellCredential: %w", err) - } - return credential, nil case AuthenticationInteractiveBrowser: - credential, err := azidentity.NewInteractiveBrowserCredential(&azidentity.InteractiveBrowserCredentialOptions{ + credential, err = azidentity.NewInteractiveBrowserCredential(&azidentity.InteractiveBrowserCredentialOptions{ ClientID: spec.clientID, TenantID: spec.tenantID, AdditionallyAllowedTenants: spec.additionallyAllowedTenants, }) - if err != nil { - return nil, fmt.Errorf("failed to create InteractiveBrowserCredential: %w", err) - } - return credential, nil default: return nil, fmt.Errorf("unsupported DTS authentication type %q", spec.authentication) } + if err != nil { + return nil, fmt.Errorf("failed to create %sCredential: %w", spec.authentication, err) + } + return credential, nil } func resolveCredential(options *Options, factory credentialFactory) (azcore.TokenCredential, error) { diff --git a/durabletaskscheduler/scheduled_tasks.go b/durabletaskscheduler/scheduled_tasks.go index a8ca3ec0..65529b6b 100644 --- a/durabletaskscheduler/scheduled_tasks.go +++ b/durabletaskscheduler/scheduled_tasks.go @@ -548,33 +548,15 @@ type scheduleConfiguration struct { } func (c scheduleConfiguration) MarshalJSON() ([]byte, error) { - type configJSON struct { - OrchestrationName string `json:"OrchestrationName"` - ScheduleID string `json:"ScheduleId"` - OrchestrationInput string `json:"OrchestrationInput,omitempty"` - OrchestrationInstanceID string `json:"OrchestrationInstanceId,omitempty"` - OrchestrationVersion string `json:"OrchestrationVersion,omitempty"` - StartAt *time.Time `json:"StartAt"` - EndAt *time.Time `json:"EndAt"` - Interval dotNetSpan `json:"Interval"` - StartImmediatelyIfLate bool `json:"StartImmediatelyIfLate"` - Tags map[string]string `json:"Tags,omitempty"` - ContextFields api.ContextFields `json:"ContextFields,omitempty"` - RetryPolicy *scheduleRetryPolicy `json:"RetryPolicy,omitempty"` - } - return json.Marshal(configJSON{ - OrchestrationName: c.OrchestrationName, - ScheduleID: c.ScheduleID, - OrchestrationInput: c.OrchestrationInput, - OrchestrationInstanceID: c.OrchestrationInstanceID, - OrchestrationVersion: c.OrchestrationVersion, - StartAt: optionalTime(c.StartAt), - EndAt: optionalTime(c.EndAt), - Interval: c.Interval, - StartImmediatelyIfLate: c.StartImmediatelyIfLate, - Tags: c.Tags, - ContextFields: c.ContextFields, - RetryPolicy: c.RetryPolicy, + type configJSON scheduleConfiguration + return json.Marshal(struct { + configJSON + StartAt *time.Time `json:"StartAt"` + EndAt *time.Time `json:"EndAt"` + }{ + configJSON: configJSON(c), + StartAt: optionalTime(c.StartAt), + EndAt: optionalTime(c.EndAt), }) } @@ -1100,14 +1082,14 @@ func applyScheduleUpdate(ctx *task.EntityContext, config *scheduleConfiguration, changed, resetNext = true, true } if options.Tags != nil && !maps.Equal(options.Tags, config.Tags) { - if err := validateScheduleTags(options.Tags); err != nil { + if err := validateScheduleKeys("tag", options.Tags); err != nil { return false, false, err } config.Tags = cloneStrings(options.Tags) changed = true } if options.ContextFields != nil && !maps.Equal(options.ContextFields, config.ContextFields) { - if err := validateScheduleContextFields(options.ContextFields); err != nil { + if err := validateScheduleKeys("context field", options.ContextFields); err != nil { return false, false, err } config.ContextFields = api.ContextFields(cloneStrings(options.ContextFields)) @@ -1186,10 +1168,10 @@ func validateCreation(options ScheduleCreationOptions) error { return &ScheduleValidationError{Message: err.Error()} } } - if err := validateScheduleTags(options.Tags); err != nil { + if err := validateScheduleKeys("tag", options.Tags); err != nil { return err } - if err := validateScheduleContextFields(options.ContextFields); err != nil { + if err := validateScheduleKeys("context field", options.ContextFields); err != nil { return err } if _, err := scheduleRetryPolicyFromPublic(options.RetryPolicy); err != nil { @@ -1244,36 +1226,16 @@ func newExecutionToken() string { return strings.ReplaceAll(uuid.NewString(), "-", "") } -func validateScheduleTags(tags map[string]string) error { - for key := range tags { +// validateScheduleKeys rejects empty keys and reserved wire prefixes. +func validateScheduleKeys(kind string, values map[string]string) error { + for key := range values { if key == "" { - return &ScheduleValidationError{Message: "tag key cannot be empty"} + return &ScheduleValidationError{Message: kind + " key cannot be empty"} } - if err := checkUnreservedScheduleKey("tag", key); err != nil { - return err - } - } - return nil -} - -func validateScheduleContextFields(fields api.ContextFields) error { - for key := range fields { - if key == "" { - return &ScheduleValidationError{Message: "context field key cannot be empty"} + if strings.HasPrefix(key, api.ReservedContextFieldPrefix) || + strings.HasPrefix(key, tagcodec.UserTagPrefix) { + return &ScheduleValidationError{Message: fmt.Sprintf("%s %q uses a reserved prefix", kind, key)} } - if err := checkUnreservedScheduleKey("context field", key); err != nil { - return err - } - } - return nil -} - -// checkUnreservedScheduleKey rejects keys that collide with the reserved -// prefixes used to carry orchestration context on the wire. -func checkUnreservedScheduleKey(kind, key string) error { - if strings.HasPrefix(key, api.ReservedContextFieldPrefix) || - strings.HasPrefix(key, tagcodec.UserTagPrefix) { - return &ScheduleValidationError{Message: fmt.Sprintf("%s %q uses a reserved prefix", kind, key)} } return nil } diff --git a/durabletaskscheduler/scheduled_tasks_test.go b/durabletaskscheduler/scheduled_tasks_test.go index 21f5b81f..359305a7 100644 --- a/durabletaskscheduler/scheduled_tasks_test.go +++ b/durabletaskscheduler/scheduled_tasks_test.go @@ -68,6 +68,73 @@ func TestScheduleStateUsesDotNetCompatibleJSON(t *testing.T) { require.Equal(t, `"1.02:03:04.5000000"`, string(span)) } +func TestScheduleConfigurationJSONShape(t *testing.T) { + tests := []struct { + name string + config scheduleConfiguration + want string + }{ + { + name: "zero values", + want: `{"OrchestrationName":"","ScheduleId":"","StartAt":null,"EndAt":null,"Interval":"00:00:00","StartImmediatelyIfLate":false}`, + }, + { + name: "all fields", + config: scheduleConfiguration{ + OrchestrationName: "Backup", + ScheduleID: "daily", + OrchestrationInput: `{"count":1}`, + OrchestrationInstanceID: "backup-instance", + OrchestrationVersion: "1.0", + StartAt: time.Date(2026, time.September, 1, 12, 0, 0, 0, time.UTC), + EndAt: time.Date(2026, time.September, 2, 12, 0, 0, 0, time.UTC), + Interval: dotNetSpan(26*time.Hour + 500*time.Millisecond), + StartImmediatelyIfLate: true, + Tags: map[string]string{"team": "storage"}, + ContextFields: api.ContextFields{"tenant": "one"}, + RetryPolicy: &scheduleRetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: dotNetSpan(1500 * time.Millisecond), + BackoffCoefficient: 2, + MaxRetryInterval: dotNetSpan(2 * time.Minute), + RetryTimeout: dotNetSpan(time.Hour), + }, + }, + want: `{ + "OrchestrationName":"Backup", + "ScheduleId":"daily", + "OrchestrationInput":"{\"count\":1}", + "OrchestrationInstanceId":"backup-instance", + "OrchestrationVersion":"1.0", + "StartAt":"2026-09-01T12:00:00Z", + "EndAt":"2026-09-02T12:00:00Z", + "Interval":"1.02:00:00.5000000", + "StartImmediatelyIfLate":true, + "Tags":{"team":"storage"}, + "ContextFields":{"tenant":"one"}, + "RetryPolicy":{ + "MaxAttempts":3, + "InitialRetryInterval":"00:00:01.5000000", + "BackoffCoefficient":2, + "MaxRetryInterval":"00:02:00", + "RetryTimeout":"01:00:00" + } + }`, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + payload, err := json.Marshal(test.config) + require.NoError(t, err) + require.JSONEq(t, test.want, string(payload)) + + var decoded scheduleConfiguration + require.NoError(t, json.Unmarshal(payload, &decoded)) + require.Equal(t, test.config, decoded) + }) + } +} + func TestScheduleOptionsUseDotNetCompatibleWireJSON(t *testing.T) { payload, err := json.Marshal(ScheduleCreationOptions{ ScheduleID: "daily", diff --git a/exporthistory/activities.go b/exporthistory/activities.go index 131f467e..d180a343 100644 --- a/exporthistory/activities.go +++ b/exporthistory/activities.go @@ -94,18 +94,6 @@ func (r *exportRuntime) listTerminalInstancesActivity(ctx task.ActivityContext) return result, nil } -// permanentExportError marks a per-instance condition that retrying cannot fix, -// such as an instance that does not exist or has not reached a terminal state. -// Transient failures are returned as activity errors instead, so the activity's -// retry policy can recover before the instance is recorded as failed. -type permanentExportError struct{ message string } - -func (e *permanentExportError) Error() string { return e.message } - -func permanentExportFailure(format string, args ...any) error { - return &permanentExportError{message: fmt.Sprintf(format, args...)} -} - // exportInstanceHistoryActivity exports one instance's history to the // destination. // @@ -140,10 +128,6 @@ func (r *exportRuntime) exportInstanceHistoryActivity(ctx task.ActivityContext) result, err := r.exportInstance(ctx.Context(), input) if err != nil { - var permanent *permanentExportError - if errors.As(err, &permanent) { - return ExportResult{InstanceID: input.InstanceID, Success: false, Error: permanent.Error()}, nil - } return nil, err } return result, nil @@ -154,16 +138,18 @@ func (r *exportRuntime) exportInstance(ctx context.Context, input ExportRequest) metadata, err := r.source.FetchOrchestrationMetadata(ctx, instanceID) if err != nil { if errors.Is(err, api.ErrInstanceNotFound) { - return ExportResult{}, permanentExportFailure("instance %s not found", input.InstanceID) + return ExportResult{InstanceID: input.InstanceID, Error: fmt.Sprintf("instance %s not found", input.InstanceID)}, nil } return ExportResult{}, fmt.Errorf("failed to read instance %s metadata: %w", input.InstanceID, err) } if metadata == nil { - return ExportResult{}, permanentExportFailure("instance %s not found", input.InstanceID) + return ExportResult{InstanceID: input.InstanceID, Error: fmt.Sprintf("instance %s not found", input.InstanceID)}, nil } if !isTerminalStatus(metadata.RuntimeStatus) { - return ExportResult{}, permanentExportFailure( - "instance %s is not in a completed state", input.InstanceID) + return ExportResult{ + InstanceID: input.InstanceID, + Error: fmt.Sprintf("instance %s is not in a completed state", input.InstanceID), + }, nil } if metadata.ExecutionID == "" { return ExportResult{}, fmt.Errorf("instance %s metadata is missing an execution ID", input.InstanceID) diff --git a/exporthistory/activities_test.go b/exporthistory/activities_test.go index c85141d4..7d2730fa 100644 --- a/exporthistory/activities_test.go +++ b/exporthistory/activities_test.go @@ -355,14 +355,18 @@ func TestExportInstanceHistoryActivity(t *testing.T) { t.Run("collects permanent per-instance failures without retrying", func(t *testing.T) { source := newFakeSource() source.addInstance("running", api.RUNTIME_STATUS_RUNNING, 1) + source.metadata["nil-metadata"] = nil + source.metadataErr["wrapped-missing"] = fmt.Errorf("lookup: %w", api.ErrInstanceNotFound) runtime := newTestRuntime(source, newMemoryStore()) tests := []struct { instanceID string message string }{ - {"missing", "not found"}, - {"running", "not in a completed state"}, + {"missing", "instance missing not found"}, + {"nil-metadata", "instance nil-metadata not found"}, + {"wrapped-missing", "instance wrapped-missing not found"}, + {"running", "instance running is not in a completed state"}, } for _, test := range tests { result, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ @@ -374,7 +378,7 @@ func TestExportInstanceHistoryActivity(t *testing.T) { exported := result.(ExportResult) assert.False(t, exported.Success, test.instanceID) assert.Equal(t, test.instanceID, exported.InstanceID) - assert.Contains(t, exported.Error, test.message) + assert.Equal(t, test.message, exported.Error) } }) @@ -399,12 +403,13 @@ func TestExportInstanceHistoryActivity(t *testing.T) { {"store-error", "upload rejected"}, } for _, test := range tests { - _, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ + result, err := runtime.exportInstanceHistoryActivity(newActivityContext(t, ExportRequest{ InstanceID: test.instanceID, Destination: destination, Format: DefaultExportFormat(), })) require.Error(t, err, test.instanceID) + assert.Nil(t, result, test.instanceID) assert.Contains(t, err.Error(), test.message) } }) @@ -503,32 +508,6 @@ func TestBlobObjectName(t *testing.T) { "the digest must not depend on the format") } -// TestBatchRetryBackoff pins the delays the whole-page retry ladder actually -// schedules. Only attempts 1 and 2 create a timer, because the third and final -// attempt fails the page instead of waiting again. -func TestBatchRetryBackoff(t *testing.T) { - assert.Equal(t, minBatchRetryBackoff, batchRetryBackoff(1)) - assert.Equal(t, 2*time.Minute, batchRetryBackoff(2)) - // The reachable schedule ends here: maxBatchRetryAttempts is 3, and the - // final attempt returns the collected failures without a backoff. - assert.Equal(t, 3, maxBatchRetryAttempts) -} - -// TestProcessBatchWithRetryHoldsTheCursorWhenNoAttemptRuns covers the defensive -// exit from the retry loop: a batch that was never attempted must be reported as -// failed so the caller keeps the cursor on the page instead of committing a -// checkpoint that skips it. -func TestProcessBatchWithRetryHoldsTheCursorWhenNoAttemptRuns(t *testing.T) { - for _, attempts := range []int{0, -1} { - result, err := processBatchWithRetry( - &task.OrchestrationContext{}, "job-1", []string{"i1", "i2"}, ExportJobConfiguration{}, attempts) - require.NoError(t, err) - assert.Zero(t, result.exportedCount) - require.Len(t, result.failures, 1) - assert.Contains(t, result.failures[0].Reason, "never attempted") - } -} - func TestExportActivityRetryPolicy(t *testing.T) { policy := exportActivityRetryPolicy() require.NoError(t, policy.Validate()) diff --git a/exporthistory/orchestrations.go b/exporthistory/orchestrations.go index 6ce637a7..bb2c5c96 100644 --- a/exporthistory/orchestrations.go +++ b/exporthistory/orchestrations.go @@ -186,7 +186,7 @@ func runExportJobOrchestration(ctx *task.OrchestrationContext, input ExportJobRu continue } } else { - batch, err := processBatchWithRetry(ctx, jobID, page.InstanceIDs, config, maxBatchRetryAttempts) + batch, err := processBatchWithRetry(ctx, jobID, page.InstanceIDs, config) if err != nil { return nil, err } @@ -249,10 +249,10 @@ func processBatchWithRetry( jobID string, instanceIDs []string, config ExportJobConfiguration, - maxAttempts int, ) (batchExportResult, error) { logger := ctx.Logger() - for attempt := 1; attempt <= maxAttempts; attempt++ { + backoff := minBatchRetryBackoff + for attempt := 1; ; attempt++ { results, err := exportBatch(ctx, instanceIDs, config) if err != nil { return batchExportResult{}, err @@ -273,7 +273,7 @@ func processBatchWithRetry( } logger.Warn("export batch failed", "jobId", jobID, "attempt", attempt, "failures", len(failed), "instances", len(instanceIDs)) - if attempt == maxAttempts { + if attempt == maxBatchRetryAttempts { failures := make([]ExportFailure, 0, len(failed)) for _, result := range failed { reason := result.Error @@ -289,31 +289,11 @@ func processBatchWithRetry( } return batchExportResult{exportedCount: succeeded, failures: failures}, nil } - if err := ctx.CreateTimer(batchRetryBackoff(attempt)).Await(nil); err != nil { + if err := ctx.CreateTimer(backoff).Await(nil); err != nil { return batchExportResult{}, err } - } - // Reached only when maxAttempts is not positive, which means the page was - // never attempted. Reporting it as failed keeps the cursor on the page - // instead of committing a checkpoint that would silently skip it. - return batchExportResult{ - failures: []ExportFailure{{ - InstanceID: strings.Join(instanceIDs, ","), - Reason: fmt.Sprintf("the export batch was never attempted: %d retry attempts configured", maxAttempts), - LastAttempt: ctx.CurrentTimeUtc, - }}, - }, nil -} - -// batchRetryBackoff is the delay before retry number attempt+1, doubling from -// [minBatchRetryBackoff]. Only attempts 1 and 2 schedule a retry, because the -// final attempt fails the page, so the reachable schedule is 1 then 2 minutes. -func batchRetryBackoff(attempt int) time.Duration { - backoff := minBatchRetryBackoff - for i := 1; i < attempt; i++ { backoff *= 2 } - return backoff } // exportBatch fans out per-instance exports in deterministic windows bounded by diff --git a/exporthistory/storage.go b/exporthistory/storage.go index a8741ade..f24e0bb9 100644 --- a/exporthistory/storage.go +++ b/exporthistory/storage.go @@ -348,27 +348,15 @@ func gzipContent(content []byte) ([]byte, error) { // validBlobContainerName mirrors the Azure Blob container naming rules. func validBlobContainerName(name string) bool { - if len(name) < 3 || len(name) > 63 { + if len(name) < 3 || len(name) > 63 || name[0] == '-' || name[len(name)-1] == '-' { return false } - if strings.HasPrefix(name, "-") || strings.HasSuffix(name, "-") { - return false - } - previousDash := false for _, r := range name { - switch { - case r >= 'a' && r <= 'z', r >= '0' && r <= '9': - previousDash = false - case r == '-': - if previousDash { - return false - } - previousDash = true - default: + if (r < 'a' || r > 'z') && (r < '0' || r > '9') && r != '-' { return false } } - return true + return !strings.Contains(name, "--") } // validateBlobPrefix rejects prefixes that would escape the destination or diff --git a/internal/failure/failure.go b/internal/failure/failure.go index e72e9828..1579e0d1 100644 --- a/internal/failure/failure.go +++ b/internal/failure/failure.go @@ -62,7 +62,7 @@ func fromProto(details *protos.TaskFailureDetails, depth int, budget *propertyBu truncated = true break } - result.Properties[key] = valueFromProto(details.GetProperties()[key]) + result.Properties[key] = details.GetProperties()[key].AsInterface() budget.count++ budget.bytes += size } @@ -284,7 +284,7 @@ func failureDetailsProperty(details *protos.TaskFailureDetails, depth int) map[s if len(details.GetProperties()) > 0 { properties := make(map[string]any, len(details.GetProperties())) for key, value := range details.GetProperties() { - properties[key] = valueFromProto(value) + properties[key] = value.AsInterface() } result["properties"] = properties } @@ -362,30 +362,6 @@ func valueToProto(value any, depth int) *structpb.Value { } } -func valueFromProto(value *structpb.Value) any { - if value == nil { - return nil - } - switch typed := value.Kind.(type) { - case *structpb.Value_StringValue: - return typed.StringValue - case *structpb.Value_StructValue: - result := make(map[string]any, len(typed.StructValue.Fields)) - for key, field := range typed.StructValue.Fields { - result[key] = valueFromProto(field) - } - return result - case *structpb.Value_ListValue: - result := make([]any, len(typed.ListValue.Values)) - for i, item := range typed.ListValue.Values { - result[i] = valueFromProto(item) - } - return result - default: - return value.AsInterface() - } -} - func truncateUTF8(value string, limit int) string { if len(value) <= limit { return value diff --git a/internal/failure/failure_test.go b/internal/failure/failure_test.go index 3357d0aa..b9daad00 100644 --- a/internal/failure/failure_test.go +++ b/internal/failure/failure_test.go @@ -4,12 +4,15 @@ import ( "bytes" "errors" "fmt" + "reflect" "testing" "time" "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/internal/failure" + "github.com/microsoft/durabletask-go/internal/protos" "github.com/microsoft/durabletask-go/task" + "google.golang.org/protobuf/types/known/structpb" ) func TestTaskFailedErrorConversionPreservesInnerFailure(t *testing.T) { @@ -98,6 +101,28 @@ func TestJoinedErrorsUseOneCauseAndAdditionalProperties(t *testing.T) { } } +func TestFailurePropertiesRoundTripNilAndNestedValues(t *testing.T) { + want := map[string]any{ + "nil": nil, + "nested": map[string]any{"values": []any{nil, "a", true, float64(4)}}, + } + nested, err := structpb.NewValue(want["nested"]) + if err != nil { + t.Fatal(err) + } + wire := &protos.TaskFailureDetails{ + Properties: map[string]*structpb.Value{"nil": nil, "nested": nested}, + } + decoded := failure.FromProto(wire) + if !reflect.DeepEqual(decoded.Properties, want) { + t.Fatalf("decoded properties = %#v, want %#v", decoded.Properties, want) + } + roundTrip := failure.FromProto(failure.ToProto(decoded)) + if !reflect.DeepEqual(roundTrip.Properties, want) { + t.Fatalf("round-trip properties = %#v, want %#v", roundTrip.Properties, want) + } +} + func TestJoinedErrorsPromoteDurableFailure(t *testing.T) { taskErr := &task.TaskFailedError{ TaskName: "activity", diff --git a/internal/helpers/entity.go b/internal/helpers/entity.go index cb10763d..3295d1d2 100644 --- a/internal/helpers/entity.go +++ b/internal/helpers/entity.go @@ -1,21 +1,7 @@ package helpers -// EntityRequestEventName is the event name used for all entity operation requests. -// This matches the .NET DTFx EntityMessageEventNames.RequestMessageEventName constant. -const EntityRequestEventName = "op" - const entitySignalPropertyPrefix = "__durabletask.entity.signal." func EntitySignalProperty(requestID string) string { return entitySignalPropertyPrefix + requestID } - -// EntityRequestMessage is the payload sent to an entity for operation requests. -// This matches the .NET DTFx RequestMessage format for wire compatibility. -type EntityRequestMessage struct { - ID string `json:"id"` - ParentInstanceID string `json:"parentInstanceId,omitempty"` - IsSignal bool `json:"isSignal"` - Operation string `json:"operation"` - Input string `json:"input,omitempty"` -} diff --git a/task/entity_dispatch_test.go b/task/entity_dispatch_test.go index 94eafe59..23307a6b 100644 --- a/task/entity_dispatch_test.go +++ b/task/entity_dispatch_test.go @@ -75,19 +75,6 @@ func Test_EntityDispatcher_BasicOperations(t *testing.T) { assert.Equal(t, 5, state.Value) } -func Test_EntityDispatcher_CaseInsensitive(t *testing.T) { - entity := NewEntityFor[testCounter]() - - ctx := &EntityContext{ - ID: api.NewEntityID("counter", "test"), - Operation: "add", // lowercase - rawInput: presentEntityPayload("10"), - } - result, err := entity(ctx) - require.NoError(t, err) - assert.Equal(t, 10, result) -} - func Test_EntityDispatcher_WithExistingState(t *testing.T) { entity := NewEntityFor[testCounter]() @@ -102,19 +89,6 @@ func Test_EntityDispatcher_WithExistingState(t *testing.T) { assert.Equal(t, 10, result) } -func Test_EntityDispatcher_Get(t *testing.T) { - entity := NewEntityFor[testCounter]() - - ctx := &EntityContext{ - ID: api.NewEntityID("counter", "test"), - Operation: "Get", - state: entityState{value: []byte(`{"value":42}`), hasValue: true}, - } - result, err := entity(ctx) - require.NoError(t, err) - assert.Equal(t, 42, result) -} - func Test_EntityDispatcher_Reset(t *testing.T) { entity := NewEntityFor[testCounter]() @@ -131,31 +105,6 @@ func Test_EntityDispatcher_Reset(t *testing.T) { assert.Equal(t, 0, state.Value) } -func Test_EntityDispatcher_ImplicitDelete(t *testing.T) { - entity := NewEntityFor[testCounter]() - - ctx := &EntityContext{ - ID: api.NewEntityID("counter", "test"), - Operation: "delete", - state: entityState{value: []byte(`{"value":42}`), hasValue: true}, - } - _, err := entity(ctx) - require.NoError(t, err) - assert.False(t, ctx.HasState()) -} - -func Test_EntityDispatcher_UnknownOperation(t *testing.T) { - entity := NewEntityFor[testCounter]() - - ctx := &EntityContext{ - ID: api.NewEntityID("counter", "test"), - Operation: "unknown", - } - _, err := entity(ctx) - require.Error(t, err) - assert.Contains(t, err.Error(), "does not support operation") -} - // Test with EntityContext parameter type contextAwareEntity struct { LastOp string `json:"lastOp"` diff --git a/task/orchestrator.go b/task/orchestrator.go index 656c9aef..f240488f 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -532,15 +532,15 @@ func (ctx *OrchestrationContext) processEvent(e *protos.HistoryEvent) error { } else if ts := e.GetTaskScheduled(); ts != nil { err = ctx.onTaskScheduled(e.EventId, ts) } else if tc := e.GetTaskCompleted(); tc != nil { - err = ctx.onTaskCompleted(tc) + err = ctx.onTaskCompleted(tc.TaskScheduledId, tc.Result) } else if tf := e.GetTaskFailed(); tf != nil { - err = ctx.onTaskFailed(tf) + err = ctx.onTaskFailed(tf.TaskScheduledId, tf.FailureDetails) } else if ts := e.GetSubOrchestrationInstanceCreated(); ts != nil { err = ctx.onSubOrchestrationScheduled(e.EventId, ts) } else if sc := e.GetSubOrchestrationInstanceCompleted(); sc != nil { - err = ctx.onSubOrchestrationCompleted(sc) + err = ctx.onTaskCompleted(sc.TaskScheduledId, sc.Result) } else if sf := e.GetSubOrchestrationInstanceFailed(); sf != nil { - err = ctx.onSubOrchestrationFailed(sf) + err = ctx.onTaskFailed(sf.TaskScheduledId, sf.FailureDetails) } else if tc := e.GetTimerCreated(); tc != nil { err = ctx.onTimerCreated(e) } else if tf := e.GetTimerFired(); tf != nil { @@ -1263,8 +1263,7 @@ func (ctx *OrchestrationContext) onTaskScheduled(taskID int32, ts *protos.TaskSc return nil } -func (ctx *OrchestrationContext) onTaskCompleted(tc *protos.TaskCompletedEvent) error { - taskID := tc.TaskScheduledId +func (ctx *OrchestrationContext) onTaskCompleted(taskID int32, result *wrapperspb.StringValue) error { task, ok := ctx.pendingTasks[taskID] if !ok { // TODO: This could be a duplicate event or it could be a non-deterministic orchestration. @@ -1274,16 +1273,15 @@ func (ctx *OrchestrationContext) onTaskCompleted(tc *protos.TaskCompletedEvent) } delete(ctx.pendingTasks, taskID) - if tc.Result != nil { - task.complete([]byte(tc.Result.Value)) + if result != nil { + task.complete([]byte(result.Value)) } else { task.complete(nil) } return nil } -func (ctx *OrchestrationContext) onTaskFailed(tf *protos.TaskFailedEvent) error { - taskID := tf.TaskScheduledId +func (ctx *OrchestrationContext) onTaskFailed(taskID int32, details *protos.TaskFailureDetails) error { task, ok := ctx.pendingTasks[taskID] if !ok { // TODO: This could be a duplicate event or it could be a non-deterministic orchestration. @@ -1294,7 +1292,7 @@ func (ctx *OrchestrationContext) onTaskFailed(tf *protos.TaskFailedEvent) error delete(ctx.pendingTasks, taskID) // completing a task will resume the corresponding Await() call - task.fail(tf.FailureDetails) + task.fail(details) return nil } @@ -1330,42 +1328,6 @@ func versionsMatchReplayHistory( strings.EqualFold(scheduled.GetValue(), legacyDefault) } -func (ctx *OrchestrationContext) onSubOrchestrationCompleted(soc *protos.SubOrchestrationInstanceCompletedEvent) error { - taskID := soc.TaskScheduledId - task, ok := ctx.pendingTasks[taskID] - if !ok { - // TODO: This could be a duplicate event or it could be a non-deterministic orchestration. - // Duplicate events should be handled gracefully with a warning. Otherwise, the - // orchestration should probably fail with an error. - return nil - } - delete(ctx.pendingTasks, taskID) - - // completing a task will resume the corresponding Await() call - if soc.Result != nil { - task.complete([]byte(soc.Result.Value)) - } else { - task.complete(nil) - } - return nil -} - -func (ctx *OrchestrationContext) onSubOrchestrationFailed(sof *protos.SubOrchestrationInstanceFailedEvent) error { - taskID := sof.TaskScheduledId - task, ok := ctx.pendingTasks[taskID] - if !ok { - // TODO: This could be a duplicate event or it could be a non-deterministic orchestration. - // Duplicate events should be handled gracefully with a warning. Otherwise, the - // orchestration should probably fail with an error. - return nil - } - delete(ctx.pendingTasks, taskID) - - // completing a task will resume the corresponding Await() call - task.fail(sof.FailureDetails) - return nil -} - func (ctx *OrchestrationContext) onTimerCreated(e *protos.HistoryEvent) error { if a, ok := ctx.pendingActions[e.EventId]; !ok || a.GetCreateTimer() == nil { return fmt.Errorf( diff --git a/task/orchestrator_test.go b/task/orchestrator_test.go index 7462196d..3bdfa9ff 100644 --- a/task/orchestrator_test.go +++ b/task/orchestrator_test.go @@ -40,127 +40,38 @@ func newTestOrchestrationContext( func Test_computeNextDelay(t *testing.T) { time1 := time.Now() time2 := time.Now().Add(1 * time.Minute) - type args struct { - currentTimeUtc time.Time - policy RetryPolicy - attempt int - firstAttempt time.Time + basePolicy := RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 2 * time.Second, + BackoffCoefficient: 2, + MaxRetryInterval: 10 * time.Second, + Handle: func(RetryContext) bool { return true }, + RetryTimeout: 2 * time.Minute, } tests := []struct { - name string - args args - want time.Duration + name string + attempt int + coefficient float64 + timeout time.Duration + want time.Duration }{ - { - name: "first attempt", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 2, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 2 * time.Minute, - }, - attempt: 0, - firstAttempt: time1, - }, - want: 2 * time.Second, - }, - { - name: "second attempt", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 2, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 2 * time.Minute, - }, - attempt: 1, - firstAttempt: time1, - }, - want: 4 * time.Second, - }, - { - name: "third attempt", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 2, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 2 * time.Minute, - }, - attempt: 2, - firstAttempt: time1, - }, - want: 8 * time.Second, - }, - { - name: "fourth attempt", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 2, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 2 * time.Minute, - }, - attempt: 3, - firstAttempt: time1, - }, - want: 10 * time.Second, - }, - { - name: "expired", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 2, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 30 * time.Second, - }, - attempt: 3, - firstAttempt: time1, - }, - want: 0, - }, - { - name: "fourth attempt backoff 1", - args: args{ - currentTimeUtc: time2, - policy: RetryPolicy{ - MaxAttempts: 3, - InitialRetryInterval: 2 * time.Second, - BackoffCoefficient: 1, - MaxRetryInterval: 10 * time.Second, - Handle: func(RetryContext) bool { return true }, - RetryTimeout: 2 * time.Minute, - }, - attempt: 3, - firstAttempt: time1, - }, - want: 2 * time.Second, - }, + {"first attempt", 0, 2, 2 * time.Minute, 2 * time.Second}, + {"second attempt", 1, 2, 2 * time.Minute, 4 * time.Second}, + {"third attempt", 2, 2, 2 * time.Minute, 8 * time.Second}, + {"fourth attempt", 3, 2, 2 * time.Minute, 10 * time.Second}, + {"expired", 3, 2, 30 * time.Second, 0}, + {"fourth attempt backoff 1", 3, 1, 2 * time.Minute, 2 * time.Second}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + policy := basePolicy + policy.BackoffCoefficient = tt.coefficient + policy.RetryTimeout = tt.timeout err := &TaskFailedError{ TaskName: "activity", FailureDetails: &api.FailureDetails{ErrorType: "TestError", ErrorMessage: "failed"}, } - if got := computeNextDelay(tt.args.currentTimeUtc, tt.args.policy, tt.args.attempt, tt.args.firstAttempt, err); got != tt.want { + if got := computeNextDelay(time2, policy, tt.attempt, time1, err); got != tt.want { t.Errorf("computeNextDelay() = %v, want %v", got, tt.want) } }) diff --git a/tests/durabletaskscheduler/dts_emulator_test.go b/tests/durabletaskscheduler/dts_emulator_test.go index b7fe833d..9f61ffd6 100644 --- a/tests/durabletaskscheduler/dts_emulator_test.go +++ b/tests/durabletaskscheduler/dts_emulator_test.go @@ -55,10 +55,6 @@ func startEmulatorClientAndWorker( ) (*durabletaskscheduler.Client, *durabletaskclient.TaskHubGrpcWorker, *durabletaskscheduler.Options) { t.Helper() options := emulatorOptions(t) - logger := api.DefaultLogger() - - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) workerOptions := []durabletaskclient.TaskHubGrpcWorkerOption{ durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), durabletaskclient.WithMaxConcurrentActivityWorkItems(8), @@ -66,6 +62,21 @@ func startEmulatorClientAndWorker( durabletaskclient.WithWorkerSilentDisconnectTimeout(15 * time.Second), } workerOptions = append(workerOptions, additionalWorkerOptions...) + managementClient, worker := startEmulatorWithOptions(t, options, registry, workerOptions...) + return managementClient, worker, options +} + +// startEmulatorWithOptions adds no defaults to the caller's worker options. +func startEmulatorWithOptions( + t *testing.T, + options *durabletaskscheduler.Options, + registry *task.TaskRegistry, + workerOptions ...durabletaskclient.TaskHubGrpcWorkerOption, +) (*durabletaskscheduler.Client, *durabletaskclient.TaskHubGrpcWorker) { + t.Helper() + logger := api.DefaultLogger() + managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) + require.NoError(t, err) worker, err := durabletaskscheduler.NewWorker(options, registry, logger, workerOptions...) require.NoError(t, err) require.NoError(t, worker.Start(context.Background())) @@ -76,7 +87,7 @@ func startEmulatorClientAndWorker( require.NoError(t, worker.Shutdown(shutdownCtx)) require.NoError(t, managementClient.Close()) }) - return managementClient, worker, options + return managementClient, worker } type dtsPayload struct { @@ -147,23 +158,12 @@ func TestDTSEmulatorCustomConverterAndVersionMigration(t *testing.T) { return input, nil })) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker( + managementClient, _ := startEmulatorWithOptions( + t, options, registry, - logger, durabletaskclient.WithAutoWorkItemFilters(), ) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) testCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -537,27 +537,16 @@ func TestDTSEmulatorScheduledFilteredLargePayloadWorker(t *testing.T) { return output, nil })) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker( + managementClient, _ := startEmulatorWithOptions( + t, options, registry, - logger, durabletaskclient.WithScheduledTaskCapability(true), durabletaskclient.WithWorkItemFilters(&durabletaskclient.WorkItemFilters{ Orchestrations: []durabletaskclient.WorkItemFilter{{Name: "DTSLargePayload"}}, Activities: []durabletaskclient.WorkItemFilter{{Name: "DTSLargePayloadEcho"}}, }), ) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -1116,18 +1105,7 @@ func TestDTSEmulatorLongTimerSplitting(t *testing.T) { })) options := emulatorOptions(t) options.MaximumTimerInterval = maximumInterval - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker(options, registry, logger) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) + managementClient, _ := startEmulatorWithOptions(t, options, registry) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() @@ -1201,18 +1179,7 @@ func TestDTSEmulatorCompletionRespectsConfiguredSendLimit(t *testing.T) { options := emulatorOptions(t) options.MaxSendMessageSize = 64 * 1024 - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker(options, registry, logger) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) + managementClient, _ := startEmulatorWithOptions(t, options, registry) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() @@ -1440,24 +1407,13 @@ func TestDTSEmulatorScheduledTasksAndHistory(t *testing.T) { registry, options.Versioning.DefaultVersion, )) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker( + managementClient, _ := startEmulatorWithOptions( + t, options, registry, - logger, durabletaskscheduler.WithScheduledTasks(), durabletaskclient.WithAutoWorkItemFilters(), ) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() @@ -1597,23 +1553,12 @@ func TestDTSEmulatorAzuriteBlobV2RoundTrip(t *testing.T) { } return firstResult, nil })) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker( + managementClient, _ := startEmulatorWithOptions( + t, options, registry, - logger, durabletaskclient.WithAutoWorkItemFilters(), ) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) defer cancel() diff --git a/tests/durabletaskscheduler/dts_tracing_tree_test.go b/tests/durabletaskscheduler/dts_tracing_tree_test.go index ff39eaa9..628669df 100644 --- a/tests/durabletaskscheduler/dts_tracing_tree_test.go +++ b/tests/durabletaskscheduler/dts_tracing_tree_test.go @@ -325,18 +325,7 @@ func TestDTSEmulatorTracingTreeVersionMigration(t *testing.T) { return input + "+v2", nil })) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker(options, registry, logger) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) + managementClient, _ := startEmulatorWithOptions(t, options, registry) ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) defer cancel() @@ -407,24 +396,13 @@ func TestDTSEmulatorTracingTreeScheduledTask(t *testing.T) { })) require.NoError(t, durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion(registry, "1.0")) - logger := api.DefaultLogger() - managementClient, err := durabletaskscheduler.NewClient(context.Background(), options, logger) - require.NoError(t, err) - worker, err := durabletaskscheduler.NewWorker( + managementClient, _ := startEmulatorWithOptions( + t, options, registry, - logger, durabletaskscheduler.WithScheduledTasks(), durabletaskclient.WithAutoWorkItemFilters(), ) - require.NoError(t, err) - require.NoError(t, worker.Start(context.Background())) - t.Cleanup(func() { - shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - require.NoError(t, worker.Shutdown(shutdownCtx)) - require.NoError(t, managementClient.Close()) - }) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) defer cancel() diff --git a/tests/failurechain/failurechain.go b/tests/failurechain/failurechain.go index c33ec0f2..d33c6be8 100644 --- a/tests/failurechain/failurechain.go +++ b/tests/failurechain/failurechain.go @@ -1,14 +1,8 @@ -// Package failurechain provides a shared, table-driven assertion for durable -// failure chains, so the Durable Task Scheduler tests validate the same -// cross-language contract the other language SDKs do. +// Package failurechain supplies an enriched application error for Durable Task +// Scheduler failure-chain tests. package failurechain -import ( - "testing" - - "github.com/microsoft/durabletask-go/api" - "github.com/stretchr/testify/require" -) +import "github.com/microsoft/durabletask-go/api" // LeafError is an application error that opts into every durable failure // enrichment hook: a stable cross-language error type, a stack trace, custom @@ -41,48 +35,3 @@ var ( _ api.DurableTaskErrorPropertiesProvider = (*LeafError)(nil) _ api.NonRetriable = (*LeafError)(nil) ) - -// Frame describes one expected frame of a durable failure chain. -type Frame struct { - ErrorType api.ErrorType - MessageContains []string - // StackContains is only checked when ExpectStack is true. - StackContains string - ExpectStack bool - NonRetriable bool - // Properties are checked by key. A nil map asserts the frame carries no - // properties at all. - Properties map[string]any -} - -// Assert walks a failure chain and asserts it matches frames exactly, including -// that the chain is neither shorter nor deeper than expected. -func Assert(t *testing.T, details *api.FailureDetails, frames []Frame) { - t.Helper() - current := details - for i, frame := range frames { - require.NotNilf(t, current, "failure chain ended before frame %d", i) - require.Equalf(t, frame.ErrorType, current.ErrorType, "frame %d error type", i) - for _, fragment := range frame.MessageContains { - require.Containsf(t, current.ErrorMessage, fragment, "frame %d message", i) - } - require.Equalf(t, frame.NonRetriable, current.IsNonRetriable, "frame %d non-retriable flag", i) - if frame.ExpectStack { - require.NotEmptyf(t, current.StackTrace, "frame %d stack trace", i) - if frame.StackContains != "" { - require.Containsf(t, current.StackTrace, frame.StackContains, "frame %d stack trace", i) - } - } else { - require.Emptyf(t, current.StackTrace, "frame %d stack trace", i) - } - if frame.Properties == nil { - require.Emptyf(t, current.Properties, "frame %d properties", i) - } else { - for key, want := range frame.Properties { - require.Equalf(t, want, current.Properties[key], "frame %d property %q", i, key) - } - } - current = current.InnerFailure - } - require.Nil(t, current, "failure chain is deeper than expected") -} From 05f43efb5f3ed799db683af25483ef5b349e10b4 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 12:12:04 -0700 Subject: [PATCH 03/26] fix: address orchestration and analyzer review feedback Reject foreign Select cases before observing their state, preserve generic event type names, skip uninvoked closures during replay analysis, and propagate asynchronous sample failures without changing its timeouts. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- cmd/orchestratorvet/README.md | 14 +- .../analysis/orchestratorgo/analyzer.go | 14 +- .../analysis/orchestratorgo/analyzer_test.go | 2 +- .../analysis/orchestratorgo/checks.go | 12 +- .../analysis/orchestratorgo/index.go | 16 ++- .../analysis/orchestratorgo/reach.go | 90 +++++++----- .../fixesunusedliteral/fixesunusedliteral.go | 28 ++++ .../fixesunusedliteral.go.golden | 28 ++++ .../src/unusedliterals/unusedliterals.go | 124 +++++++++++++++++ samples/externalevents/externalevents.go | 42 ++++-- samples/externalevents/externalevents_test.go | 60 ++++++++ task/eventchannel.go | 6 +- task/select.go | 14 +- task/select_test.go | 131 ++++++++++++++++++ 14 files changed, 513 insertions(+), 68 deletions(-) create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go.golden create mode 100644 cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/unusedliterals/unusedliterals.go create mode 100644 samples/externalevents/externalevents_test.go diff --git a/cmd/orchestratorvet/README.md b/cmd/orchestratorvet/README.md index 5c5121bc..a4abd519 100644 --- a/cmd/orchestratorvet/README.md +++ b/cmd/orchestratorvet/README.md @@ -40,10 +40,16 @@ the package under analysis passes it to one of From each of those roots it follows the whole-package call graph through same-package named functions, methods, function variables whose target can be -proven, and nested function literals. Recursion terminates because each function -is visited once, and a helper reached from several orchestrators is reported -once. Activity bodies, entity bodies, and any function not reachable from a -registered orchestrator are never reported. +proven, and the nested function literals whose bodies are proven to execute -- +through a direct call, a call through a single-assignment function variable, +a raw `go` or `defer` call, +or the callback argument of an explicitly modeled invoker such as +`(*task.OrchestrationContext).Go`. A literal passed to any other helper, or +assigned and never invoked, is not followed: its body never contributes a +diagnostic, and helpers reached only through it are not reported. Recursion +terminates because each function is visited once, and a helper reached from +several orchestrators is reported once. Activity bodies, entity bodies, and any +function not reachable from a registered orchestrator are never reported. ## Checks diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go index 3b131c91..4696234a 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer.go @@ -3,9 +3,14 @@ // // The analyzer starts from every proven registration in the package under // analysis, follows the call graph through same-package named functions, -// methods, resolvable function variables, and nested function literals, and then -// reports only constructs whose nondeterminism is provable from that syntax and -// type information alone. +// methods, resolvable function variables, and the nested function literals +// whose bodies are proven to execute -- through a direct call whose callee +// resolves to the literal, a call through a single-assignment variable, a raw go or +// defer call, or the callback argument of an explicitly modeled invoker such +// as (*task.OrchestrationContext).Go. It then reports only constructs whose +// nondeterminism is provable from that syntax and type information alone. +// Literals passed to unmodeled helpers, or assigned and never invoked, are not +// followed: their bodies never contribute a diagnostic. // // It reports: // @@ -71,9 +76,6 @@ func run(pass *analysis.Pass) (any, error) { reported: make(map[reportKey]bool), } for _, node := range reach.order { - if reach.lexicallyCovered(index, node) { - continue - } check.checkFunction(node) } check.reportClockDiagnostics() diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go index fb8de818..112a341d 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/analyzer_test.go @@ -23,7 +23,7 @@ import ( var fixPackages = []string{ "fixes", "fixesimport", "fixesstandalone", "fixesstandalonealias", "fixesbatch", "fixesbatchstandalone", "fixesbatchalias", "fixespartial", "fixesinline", - "fixbindings", + "fixbindings", "fixesunusedliteral", } // stubPackageRoot is the fixture tree holding the stand-in dependencies the diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/checks.go b/cmd/orchestratorvet/analysis/orchestratorgo/checks.go index f3c09601..ec7861ee 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/checks.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/checks.go @@ -66,19 +66,27 @@ func (c *checker) report( } // checkFunction walks one reachable function and reports every replay hazard it -// contains, including hazards inside nested function literals. +// contains. Nested function literals are skipped here: reach records only the +// literals whose bodies are proven to execute and walks each one separately, so +// descending into a literal here would double-check ones that run and audit +// ones that never do. Enclosing function nodes tracked by the package index +// prime the ancestor stack so a fix generated inside a nested literal can still +// see the surrounding orchestration context binding. func (c *checker) checkFunction(node ast.Node) { body := funcBody(node) if body == nil { return } file := c.index.fileOf[node] - stack := []ast.Node{node} + stack := c.index.enclosingFunctionStack(node) ast.Inspect(body, func(current ast.Node) bool { if current == nil { stack = stack[:len(stack)-1] return true } + if _, ok := current.(*ast.FuncLit); ok { + return false + } stack = append(stack, current) c.inspect(current, stack, file) return true diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/index.go b/cmd/orchestratorvet/analysis/orchestratorgo/index.go index e63f441b..3cf7eb07 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/index.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/index.go @@ -4,6 +4,7 @@ import ( "go/ast" "go/token" "go/types" + "slices" "strings" "golang.org/x/tools/go/analysis" @@ -33,8 +34,8 @@ type packageIndex struct { // enclosingFunc maps every function literal to the nearest function // declaration or literal that lexically contains it, and is absent for a // literal written outside any function, such as a package-level variable - // initializer. Walking a function body already visits the literals nested - // in it, so this is what lets the checker walk each one exactly once. + // initializer. The checker uses this chain to recover captured context + // bindings when it walks a reachable literal separately. enclosingFunc map[ast.Node]ast.Node // registrationCandidates holds, in source order, every call whose selector @@ -302,6 +303,17 @@ func (index *packageIndex) singleValue(object types.Object) ast.Expr { return values[0] } +// enclosingFunctionStack preserves captured context bindings when a reached +// literal is checked separately from its enclosing functions. +func (index *packageIndex) enclosingFunctionStack(node ast.Node) []ast.Node { + var stack []ast.Node + for current := node; current != nil; current = index.enclosingFunc[current] { + stack = append(stack, current) + } + slices.Reverse(stack) + return stack +} + // resolveFunction returns the function declaration or literal that expression // evaluates to, or nil when the target cannot be proven statically. func (index *packageIndex) resolveFunction(expression ast.Expr, seen map[types.Object]struct{}) ast.Node { diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/reach.go b/cmd/orchestratorvet/analysis/orchestratorgo/reach.go index 16dd9775..d78cb939 100644 --- a/cmd/orchestratorvet/analysis/orchestratorgo/reach.go +++ b/cmd/orchestratorvet/analysis/orchestratorgo/reach.go @@ -2,70 +2,84 @@ package orchestratorgo import ( "go/ast" + + "golang.org/x/tools/go/analysis" ) // reachSet is the deterministic set of package functions reachable from the -// registered orchestrators. +// registered orchestrators. Every entry is analyzed on its own -- the checker +// walks only that node's own body -- so a nested literal that never executes +// stays out of the set and is not audited. type reachSet struct { order []ast.Node set map[ast.Node]bool } -// lexicallyCovered reports whether node is a function literal nested inside -// another reachable function. Walking that outer function already visits the -// literal's body, so checking the literal separately would walk it a second -// time. It would also lose context: a fix rewritten from the outer walk can see -// the enclosing orchestration context parameter, which is out of scope when the -// literal is walked on its own. -// -// A literal written outside any function, such as a package-level function -// variable, has no enclosing function and is therefore never covered. -func (reach *reachSet) lexicallyCovered(index *packageIndex, node ast.Node) bool { - for parent := index.enclosingFunc[node]; parent != nil; parent = index.enclosingFunc[parent] { - if reach.set[parent] { - return true - } - } - return false -} - // reachableFunctions walks the whole-package call graph from roots, following -// named helpers, methods, resolvable function variables, and nested literals. -// Recursion terminates because each function node is added at most once. +// named helpers, methods, resolvable function variables, and the nested literals +// that are provably invoked. Recursion terminates because each function node is +// added at most once. +// +// A nested function literal is added only when its body is proven to execute: +// through a direct call, a call through a single-assignment function variable, +// a raw go or defer call, or the +// callback argument of an explicitly modeled invoker such as +// (*task.OrchestrationContext).Go. Arbitrary function arguments are not assumed +// to run, so a literal passed to a helper that never invokes it stays out of the +// reachable set. func reachableFunctions(index *packageIndex, roots []ast.Node) *reachSet { reach := &reachSet{set: make(map[ast.Node]bool)} - for _, root := range roots { - if root == nil || reach.set[root] { - continue + add := func(node ast.Node) { + if node == nil || reach.set[node] { + return } - reach.set[root] = true - reach.order = append(reach.order, root) + reach.set[node] = true + reach.order = append(reach.order, node) + } + for _, root := range roots { + add(root) } for next := 0; next < len(reach.order); next++ { current := reach.order[next] - forEachCall(current, func(call *ast.CallExpr) { - callee := index.callee(call) - if callee == nil || reach.set[callee] { - return - } - reach.set[callee] = true - reach.order = append(reach.order, callee) + forEachOwnedCall(current, func(call *ast.CallExpr) { + add(index.callee(call)) + add(index.resolveFunction(orchestrationGoCallback(index.pass, call), nil)) }) } return reach } -// forEachCall visits every call expression lexically contained in a function, -// including calls inside nested function literals, go statements, and defers. -func forEachCall(node ast.Node, visit func(*ast.CallExpr)) { +// forEachOwnedCall visits every call expression lexically contained in the +// current function's own body, including calls in raw go and defer statements +// but NOT calls written inside a nested function literal. Those literals are +// separate reachable units: reach adds only the ones actually invoked and walks +// them in their own turn, so descending here would follow calls that never run. +func forEachOwnedCall(node ast.Node, visit func(*ast.CallExpr)) { body := funcBody(node) if body == nil { return } - ast.Inspect(body, func(node ast.Node) bool { - if call, ok := node.(*ast.CallExpr); ok { + ast.Inspect(body, func(current ast.Node) bool { + if _, ok := current.(*ast.FuncLit); ok { + return false + } + if call, ok := current.(*ast.CallExpr); ok { visit(call) } return true }) } + +// orchestrationGoCallback models Go without assuming arbitrary callbacks execute. +func orchestrationGoCallback(pass *analysis.Pass, call *ast.CallExpr) ast.Expr { + function := staticFunc(pass, call.Fun) + if function == nil || function.Name() != "Go" || len(call.Args) == 0 { + return nil + } + ownerPath, ownerName, ok := methodOwner(function) + if !ok || ownerPath != taskPackagePath || ownerName != "OrchestrationContext" { + return nil + } + // The callback is last for both ctx.Go(fn) and (*OrchestrationContext).Go(ctx, fn). + return call.Args[len(call.Args)-1] +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go new file mode 100644 index 00000000..85b05a5d --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go @@ -0,0 +1,28 @@ +// Package fixesunusedliteral proves the atomic clock fix keeps the `time` +// import when it is used only in a nested literal that never executes: the +// analyzer no longer diagnoses that literal, so the import cannot be dropped. +package fixesunusedliteral + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func fixMe(ctx *task.OrchestrationContext) (any, error) { + return time.Now(), nil // want `time\.Now is not deterministic in an orchestrator` +} + +// dormant holds a literal that references time but is never invoked, so no +// diagnostic fires here. The atomic clock fix must NOT strip the time import. +func dormant(ctx *task.OrchestrationContext) (any, error) { + dead := func() time.Time { return time.Now() } + _ = dead + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(fixMe) + _ = registry.AddOrchestrator(dormant) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go.golden b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go.golden new file mode 100644 index 00000000..1f35352b --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/fixesunusedliteral/fixesunusedliteral.go.golden @@ -0,0 +1,28 @@ +// Package fixesunusedliteral proves the atomic clock fix keeps the `time` +// import when it is used only in a nested literal that never executes: the +// analyzer no longer diagnoses that literal, so the import cannot be dropped. +package fixesunusedliteral + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +func fixMe(ctx *task.OrchestrationContext) (any, error) { + return ctx.CurrentTimeUtc, nil // want `time\.Now is not deterministic in an orchestrator` +} + +// dormant holds a literal that references time but is never invoked, so no +// diagnostic fires here. The atomic clock fix must NOT strip the time import. +func dormant(ctx *task.OrchestrationContext) (any, error) { + dead := func() time.Time { return time.Now() } + _ = dead + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(fixMe) + _ = registry.AddOrchestrator(dormant) +} diff --git a/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/unusedliterals/unusedliterals.go b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/unusedliterals/unusedliterals.go new file mode 100644 index 00000000..c9cc4cb2 --- /dev/null +++ b/cmd/orchestratorvet/analysis/orchestratorgo/testdata/src/unusedliterals/unusedliterals.go @@ -0,0 +1,124 @@ +// Package unusedliterals verifies that function literals declared but never +// statically invoked are not analyzed as executable orchestrator code, while +// literals reached through statically resolvable calls, raw go/defer, or the +// (*task.OrchestrationContext).Go callback still report their hazards. +package unusedliterals + +import ( + "time" + + "github.com/microsoft/durabletask-go/task" +) + +// unusedNestedLiteral assigns hazards to a literal that never runs, and +// references a helper only through that literal, so neither must be reported. +func unusedNestedLiteral(ctx *task.OrchestrationContext) (any, error) { + dormant := func() { + _ = time.Now() + dormantHelper() + } + _ = dormant + return nil, nil +} + +// dormantHelper is only reachable through the unused literal above and must +// therefore stay silent. +func dormantHelper() { + time.Sleep(time.Second) +} + +// invokedNestedLiteral is the same shape but the literal actually runs, so +// hazards inside it -- and inside helpers reached only through it -- report. +func invokedNestedLiteral(ctx *task.OrchestrationContext) (any, error) { + live := func() { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + liveHelper() + } + live() + return nil, nil +} + +func liveHelper() { + time.Sleep(time.Second) // want `time\.Sleep is not deterministic in an orchestrator` +} + +// contextGoCallback verifies (*task.OrchestrationContext).Go arguments still +// count as executed. +func contextGoCallback(ctx *task.OrchestrationContext) (any, error) { + ctx.Go(func(child *task.OrchestrationContext) { + go func() {}() // want `raw go statement is not deterministic in an orchestrator` + }) + return nil, nil +} + +func contextGoMethodExpression(ctx *task.OrchestrationContext) (any, error) { + (*task.OrchestrationContext).Go(ctx, func(child *task.OrchestrationContext) { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + }) + return nil, nil +} + +func contextGoNamedCallback(ctx *task.OrchestrationContext) (any, error) { + callback := func(child *task.OrchestrationContext) { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + } + ctx.Go(callback) + return nil, nil +} + +// deferredLiteral runs at return, so its hazards report. +func deferredLiteral(ctx *task.OrchestrationContext) (any, error) { + defer func() { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + }() + return nil, nil +} + +// immediateGoLiteral is executed by the go statement itself. +func immediateGoLiteral(ctx *task.OrchestrationContext) (any, error) { + go func() { // want `raw go statement is not deterministic in an orchestrator` + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` + }() + return nil, nil +} + +// arbitraryFunctionArgument passes a literal to a helper that does not invoke +// it. The analyzer must not assume arbitrary function arguments execute. +func arbitraryFunctionArgument(ctx *task.OrchestrationContext) (any, error) { + acceptCallback(func() { + _ = time.Now() + }) + return nil, nil +} + +// acceptCallback stores its argument but never invokes it, so the literal +// passed above is not reachable from the orchestrator. +func acceptCallback(fn func()) { + _ = fn +} + +// singleAssignmentFuncVar exercises a package-level function variable held to +// a single literal and invoked through a local alias. The literal executes, +// so its hazards report. +var singleAssignmentFuncVar = func() { + _ = time.Now() // want `time\.Now is not deterministic in an orchestrator` +} + +func singleAssignmentInvoker(ctx *task.OrchestrationContext) (any, error) { + local := singleAssignmentFuncVar + local() + return nil, nil +} + +func register() { + registry := task.NewTaskRegistry() + _ = registry.AddOrchestrator(unusedNestedLiteral) + _ = registry.AddOrchestrator(invokedNestedLiteral) + _ = registry.AddOrchestrator(contextGoCallback) + _ = registry.AddOrchestrator(contextGoMethodExpression) + _ = registry.AddOrchestrator(contextGoNamedCallback) + _ = registry.AddOrchestrator(deferredLiteral) + _ = registry.AddOrchestrator(immediateGoLiteral) + _ = registry.AddOrchestrator(arbitraryFunctionArgument) + _ = registry.AddOrchestrator(singleAssignmentInvoker) +} diff --git a/samples/externalevents/externalevents.go b/samples/externalevents/externalevents.go index 07548c1c..7428befa 100644 --- a/samples/externalevents/externalevents.go +++ b/samples/externalevents/externalevents.go @@ -8,7 +8,9 @@ package main import ( "context" "fmt" + "io" "log" + "os" "time" "github.com/microsoft/durabletask-go/api" @@ -51,22 +53,20 @@ func run() error { return fmt.Errorf("failed to wait for orchestration to start: %w", err) } - // Prompt the user for their name and send that to the orchestrator - go func() { - fmt.Println("Enter your first name: ") - var nameInput string - if _, err := fmt.Scanln(&nameInput); err != nil { - log.Printf("Failed to read input: %v", err) - return - } - if err := app.Client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(nameInput)); err != nil { - log.Printf("Failed to raise event: %v", err) - } - }() + // Keep stdin asynchronous, but interrupt the completion wait on input or RPC failure. + waitCtx, cancelWait := context.WithCancelCause(ctx) + defer cancelWait(nil) + fmt.Println("Enter your first name: ") + go readAndRaiseEvent(waitCtx, cancelWait, os.Stdin, func(ctx context.Context, name string) error { + return app.Client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(name)) + }) // After the orchestration receives the event, it should complete on its own - metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(waitCtx, id) if err != nil { + if cause := context.Cause(waitCtx); cause != nil { + err = cause + } return fmt.Errorf("failed to wait for orchestration to complete: %w", err) } if metadata.FailureDetails != nil { @@ -77,6 +77,22 @@ func run() error { return nil } +func readAndRaiseEvent( + ctx context.Context, + cancel context.CancelCauseFunc, + input io.Reader, + raise func(context.Context, string) error, +) { + var name string + if _, err := fmt.Fscanln(input, &name); err != nil { + cancel(fmt.Errorf("failed to read input: %w", err)) + return + } + if err := raise(ctx, name); err != nil { + cancel(fmt.Errorf("failed to raise event: %w", err)) + } +} + // ExternalEventOrchestrator is an orchestrator function that blocks for 30 seconds or // until a "Name" event is sent to it. func ExternalEventOrchestrator(ctx *task.OrchestrationContext) (any, error) { diff --git a/samples/externalevents/externalevents_test.go b/samples/externalevents/externalevents_test.go new file mode 100644 index 00000000..a6bfe893 --- /dev/null +++ b/samples/externalevents/externalevents_test.go @@ -0,0 +1,60 @@ +package main + +import ( + "context" + "errors" + "io" + "strings" + "testing" + "testing/iotest" +) + +func TestReadAndRaiseEvent(t *testing.T) { + readError := errors.New("input unavailable") + raiseError := errors.New("event rejected") + for _, test := range []struct { + name string + input io.Reader + raiseErr error + wantErr error + wantStage string + wantRaise bool + }{ + {name: "EOF", input: strings.NewReader(""), wantErr: io.EOF, wantStage: "read input"}, + {name: "read error", input: iotest.ErrReader(readError), wantErr: readError, wantStage: "read input"}, + {name: "raise error", input: strings.NewReader("Taylor\n"), raiseErr: raiseError, wantErr: raiseError, wantStage: "raise event", wantRaise: true}, + {name: "success", input: strings.NewReader("Taylor\n"), wantRaise: true}, + } { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancelCause(t.Context()) + defer cancel(nil) + raised := false + readAndRaiseEvent(ctx, cancel, test.input, func(callCtx context.Context, name string) error { + raised = true + if callCtx != ctx || name != "Taylor" { + t.Fatalf("unexpected event context or name %q", name) + } + return test.raiseErr + }) + if raised != test.wantRaise { + t.Fatalf("event raised=%t, want %t", raised, test.wantRaise) + } + cause := context.Cause(ctx) + if !errors.Is(cause, test.wantErr) { + t.Fatalf("cancellation cause=%v, want %v", cause, test.wantErr) + } + if test.wantErr != nil { + select { + case <-ctx.Done(): + default: + t.Fatal("input failure did not interrupt the completion context") + } + if !strings.Contains(cause.Error(), test.wantStage) { + t.Fatalf("error %q does not describe %q", cause, test.wantStage) + } + } else if ctx.Err() != nil { + t.Fatalf("successful input canceled the completion wait: %v", ctx.Err()) + } + }) + } +} diff --git a/task/eventchannel.go b/task/eventchannel.go index 751a275c..726052e4 100644 --- a/task/eventchannel.go +++ b/task/eventchannel.go @@ -92,7 +92,7 @@ func (c *EventChannel[T]) TryReceiveErr() (T, bool, error) { } raw := []byte(buffered.event.GetEventRaised().GetInput().GetValue()) if err := unmarshalData(c.ctx.converter, raw, &value); err != nil { - return value, true, fmt.Errorf("failed to decode event %q as %s: %w", c.name, reflect.TypeOf(value), err) + return value, true, fmt.Errorf("failed to decode event %q as %s: %w", c.name, reflect.TypeFor[T](), err) } return value, true, nil } @@ -114,6 +114,10 @@ func OnEvent[T any](channel *EventChannel[T], handler func(T)) SelectCase { return &eventSelectCase[T]{channel: channel, handler: handler} } +func (c *eventSelectCase[T]) owner() *OrchestrationContext { + return c.channel.ctx +} + func (c *eventSelectCase[T]) ready() (bool, uint64) { event, ok := c.channel.peek() if !ok { diff --git a/task/select.go b/task/select.go index f01b67e7..06f9939f 100644 --- a/task/select.go +++ b/task/select.go @@ -4,6 +4,7 @@ import "fmt" // SelectCase is a durable selection case created by OnTask or OnEvent. type SelectCase interface { + owner() *OrchestrationContext ready() (bool, uint64) subscribe(*coroutine) unsubscribe(*coroutine) @@ -25,6 +26,10 @@ func OnTask(task Task, handler func(Task)) SelectCase { return &taskSelectCase{task: task, state: state, handler: handler} } +func (c *taskSelectCase) owner() *OrchestrationContext { + return c.state.orchestrationCtx.engineContext() +} + func (c *taskSelectCase) ready() (bool, uint64) { return c.state.isCompleted, c.state.completionID } @@ -83,6 +88,7 @@ func (ctx *OrchestrationContext) WhenAll(tasks ...Task) error { } // Select waits until one case is ready and invokes its handler. +// All cases must belong to the same orchestration, including its child scopes. func (ctx *OrchestrationContext) Select(cases ...SelectCase) { ctx.selectCase(cases) } @@ -91,7 +97,8 @@ func (ctx *OrchestrationContext) selectCase(cases []SelectCase) SelectCase { if len(cases) == 0 { panic("Select requires at least one case") } - scheduler := ctx.engineContext().scheduler + engine := ctx.engineContext() + scheduler := engine.scheduler if scheduler == nil { panic("Select called outside orchestrator execution") } @@ -101,6 +108,11 @@ func (ctx *OrchestrationContext) selectCase(cases []SelectCase) SelectCase { if current.scope.isCanceled() || ctx.scope.isCanceled() { panic(ErrTaskCanceled) } + for _, candidate := range cases { + if candidate.owner() != engine { + panic("Select case belongs to a different orchestration") + } + } var selected SelectCase var selectedOrder uint64 for _, candidate := range cases { diff --git a/task/select_test.go b/task/select_test.go index ca2cce53..aabc3257 100644 --- a/task/select_test.go +++ b/task/select_test.go @@ -1,8 +1,12 @@ package task import ( + "container/list" "context" + "encoding/json" "errors" + "fmt" + "strings" "testing" "github.com/microsoft/durabletask-go/api" @@ -11,6 +15,133 @@ import ( "google.golang.org/protobuf/types/known/wrapperspb" ) +func TestSelectRejectsForeignCases(t *testing.T) { + for _, test := range []struct { + name string + ready bool + event bool + whenAny bool + }{ + {name: "pending-task"}, + {name: "ready-task", ready: true}, + {name: "empty-event", event: true}, + {name: "buffered-event", event: true, ready: true}, + {name: "when-any-pending", whenAny: true}, + {name: "when-any-ready", whenAny: true, ready: true}, + } { + for _, localReady := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/local-ready=%t", test.name, localReady), func(t *testing.T) { + // Matching instance IDs must not make distinct execution engines interchangeable. + foreign := newTestOrchestrationContext(NewTaskRegistry(), "instance", nil, nil) + foreignTask := newTaskInScope(foreign, foreign.scope) + channel := NewEventChannel[int](foreign, "value") + if test.ready { + if test.event { + queue := list.New() + queue.PushBack(&bufferedEvent{ + event: helpers.NewEventRaisedEvent("value", wrapperspb.String("42")), + order: 0, + }) + foreign.bufferedExternalEvents["VALUE"] = queue + } else { + foreignTask.complete(nil) + } + } + var owner *OrchestrationContext + var local *completableTask + invoked := false + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("select-owner", func(ctx *OrchestrationContext) (any, error) { + owner = ctx + local = newTaskInScope(ctx, ctx.scope) + if localReady { + local.complete(nil) + } + var recovered any + func() { + defer func() { recovered = recover() }() + if test.whenAny { + ctx.WhenAny(local, foreignTask) + invoked = true + return + } + candidate := OnTask(foreignTask, func(Task) { invoked = true }) + if test.event { + candidate = OnEvent(channel, func(int) { invoked = true }) + } + ctx.Select(OnTask(local, func(Task) { invoked = true }), candidate) + }() + return recovered == "Select case belongs to a different orchestration" && !invoked, nil + }); err != nil { + t.Fatal(err) + } + response := executeOrchestrationTurn(t, registry, "instance", nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("select-owner", "instance", nil, nil, nil, nil), + }) + if got := completionResult(t, response); got != "true" { + t.Fatalf("foreign selection was not rejected: %s", got) + } + if len(foreignTask.waiters) != 0 || len(foreign.eventWaiters) != 0 || + len(local.waiters) != 0 || len(owner.scope.waiters) != 0 { + t.Fatal("invalid selection registered a waiter") + } + if test.event && test.ready { + if value, ok, err := channel.TryReceiveErr(); !ok || err != nil || value != 42 { + t.Fatalf("foreign event was not preserved: value=%d received=%t err=%v", value, ok, err) + } + } + }) + } + } +} + +func TestSelectAllowsSameEngineChildScopes(t *testing.T) { + registry := NewTaskRegistry() + if err := registry.AddOrchestratorN("select-scopes", func(ctx *OrchestrationContext) (any, error) { + child, _ := ctx.WithCancel() + rootTask := newTaskInScope(ctx, ctx.scope) + childTask := newTaskInScope(child, child.scope) + rootTask.complete(nil) + childTask.complete(nil) + selected := 0 + ctx.Select(OnTask(childTask, func(Task) { selected++ })) + child.Select(OnTask(rootTask, func(Task) { selected++ })) + child.Select(OnEvent(NewEventChannel[int](child, "value"), func(value int) { selected += value })) + return selected, nil + }); err != nil { + t.Fatal(err) + } + response := executeOrchestrationTurn(t, registry, "instance", nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("select-scopes", "instance", nil, nil, nil, nil), + helpers.NewEventRaisedEvent("value", wrapperspb.String("3")), + }) + if got := completionResult(t, response); got != "5" { + t.Fatalf("same-engine selection = %s, want 5", got) + } +} + +func TestEventChannelDecodeErrorNamesGenericType(t *testing.T) { + t.Run("interface", func(t *testing.T) { testEventDecodeType[any](t, "interface {}") }) + t.Run("pointer", func(t *testing.T) { testEventDecodeType[*int](t, "*int") }) + t.Run("slice", func(t *testing.T) { testEventDecodeType[[]int](t, "[]int") }) + t.Run("map", func(t *testing.T) { testEventDecodeType[map[string]int](t, "map[string]int") }) +} + +func testEventDecodeType[T any](t *testing.T, want string) { + t.Helper() + ctx := newTestOrchestrationContext(NewTaskRegistry(), "instance", nil, nil) + queue := list.New() + queue.PushBack(&bufferedEvent{event: helpers.NewEventRaisedEvent("value", wrapperspb.String("{invalid"))}) + ctx.bufferedExternalEvents["VALUE"] = queue + _, received, err := NewEventChannel[T](ctx, "value").TryReceiveErr() + var syntaxError *json.SyntaxError + if !received || err == nil || !strings.Contains(err.Error(), "as "+want+":") || !errors.As(err, &syntaxError) { + t.Fatalf("expected type %q and wrapped syntax error, received=%t err=%v", want, received, err) + } +} + func TestWhenAnyUsesHistoryOrder(t *testing.T) { registry := NewTaskRegistry() if err := registry.AddOrchestratorN("when-any", func(ctx *OrchestrationContext) (any, error) { From 5c5ea17df8b5426f204c825a721472b605464f52 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 17:24:07 -0700 Subject: [PATCH 04/26] remove unsupported DTS administrative APIs Remove rewind, skip-graceful termination, and SDK task-hub create/delete operations and their options, dead tests, and documentation claims. Preserve restart, purge, entity maintenance, connection errors, and the vendored protocol contract. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 7 +- README.md | 8 +- api/management.go | 19 --- client/client_grpc_management.go | 61 --------- client/client_grpc_management_test.go | 124 ++++-------------- client/errors.go | 4 +- client/grpc_errors.go | 14 +- client/grpc_errors_test.go | 5 +- durabletaskscheduler/README.md | 21 ++- internal/grpcerrors/reasons.go | 1 - .../durabletaskscheduler/dts_emulator_test.go | 83 ++---------- 11 files changed, 63 insertions(+), 284 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 49084a56..45c4e1f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, distinct activity and sub-orchestration action trace contexts, and legacy entity-operation trace forwarding. - Added the top-level `durabletaskscheduler` transport package, a dedicated resilient gRPC worker, DTS emulator tests, and an environment-driven sample. -- Added advanced management APIs for bounded instance queries/listing, restart, rewind, batch/filter purge polling, immediate termination, and task-hub lifecycle operations. -- Added orchestration tags to scheduling, metadata, queries, sub-orchestrations, continue-as-new, restart, and rewind. +- Added advanced management APIs for bounded instance queries/listing, restart, and batch/filter purge polling. +- Added orchestration tags to scheduling, metadata, queries, sub-orchestrations, continue-as-new, and restart. - Added explicit worker capability advertisement and orchestration/activity name/version filters with local fallback enforcement. - Added pluggable large-payload store/resolver support with size limits, SHA-256 integrity validation, memory/file implementations, and opt-in DTS capability advertisement. - Added API-owned structured failure details, typed task and entity operation errors, stable cross-language error types, bounded panic stacks, nested causes, and custom error-property enrichment. @@ -36,10 +36,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- Removed the unsupported SDK methods `RewindInstance`, `SkipGracefulOrchestrationTerminations`, `CreateTaskHub`, and `DeleteTaskHub`, along with `api.RewindOptions`, `api.WithRewindReason`, `api.CreateTaskHubOptions`, `api.WithRecreateTaskHub`, and the lifecycle-only `client.ErrTaskHubExists` sentinel. Provision and delete task hubs through the Azure control plane or Azure CLI. `RestartInstance`, normal termination, instance purge, entity maintenance, and passive decoding of rewind history records remain supported; `client.ErrTaskHubNotFound` still identifies a configured task hub that does not exist. - **Removed the sqlite and PostgreSQL storage backends** (`backend/sqlite` and `backend/postgres`) and the `modernc.org/sqlite` and `github.com/jackc/pgx/v5` dependencies. Durable Task Scheduler is now the only supported runtime, and this repository is the DTS Go SDK rather than an embeddable engine plus storage providers. Applications that hosted an embedded task hub must move to `durabletaskscheduler.NewClient` and `durabletaskscheduler.NewWorker`. - **Removed the standalone local gRPC server** (root `main.go` and its `Dockerfile`), along with the `--port`/`--host`/`--db` commands and the `.NET` client-SDK integration-test instructions that targeted it. - Removed the `samples/sequence` and `samples/heterogeneous` samples. The activity sequence is demonstrated by `samples/durabletaskscheduler`, and the heterogeneous sample only demonstrated routing between embedded task executor implementations, which has no Durable Task Scheduler equivalent. -- Removed the residual public `backend` package. Logging moved to `api`, executor and metric contracts moved to `task`, task-hub lifecycle errors moved to `client`, and worker-only entity conversion and abandon-delay contracts are now private to `client`. +- Removed the residual public `backend` package. Logging moved to `api`, executor and metric contracts moved to `task`, the missing-task-hub error moved to `client`, and worker-only entity conversion and abandon-delay contracts are now private to `client`. - Removed the unused `api.ErrNotStarted`, `api.ErrNoFailures`, and `api.ErrIgnoreInstance` sentinels and the test-only exported `task.NewOrchestrationContext` constructor. - Removed the embedded-backend integration harness under `./tests` and the SQLite-hosted generic gRPC suite under `./tests/grpc`. Their unique coverage was migrated to deterministic package tests (`client` task-hub management over a fake gRPC server, and `exporthistory.WithExportHistory` strict-version filter routing) and to the live `tests/durabletaskscheduler` suite. - Removed the `POSTGRES_ENABLED` environment variable and the PostgreSQL service from PR validation. diff --git a/README.md b/README.md index 49aa288f..516d7d10 100644 --- a/README.md +++ b/README.md @@ -329,11 +329,15 @@ This example does not show the client and worker setup. Read [Connection to DTS] The client also does these operations, if the connected service supplies them: - Query the instances with a limit. List the instance IDs. -- Restart and rewind an orchestration. +- Restart an orchestration. - Purge in a batch or with a filter. -- Terminate an orchestration immediately. +- Terminate an orchestration. - Read the tags and the worker capabilities. +Provision and delete task hubs through the Azure control plane or Azure CLI. +The SDK does not expose task-hub lifecycle, rewind, or skip-graceful-termination +operations. + To read a long history, use `StreamOrchestrationHistory`. This method reads the history one part at a time. If you buffer the history instead, the SDK applies a validated event cap. ## Durable entities diff --git a/api/management.go b/api/management.go index 79e4ad4a..b8065255 100644 --- a/api/management.go +++ b/api/management.go @@ -6,7 +6,6 @@ import ( "time" "github.com/microsoft/durabletask-go/internal/protos" - "google.golang.org/protobuf/types/known/wrapperspb" ) const ( @@ -64,15 +63,6 @@ func WithRestartNewInstanceID(restartWithNewInstanceID bool) RestartOptions { } } -type RewindOptions func(*protos.RewindInstanceRequest) error - -func WithRewindReason(reason string) RewindOptions { - return func(req *protos.RewindInstanceRequest) error { - req.Reason = wrapperspb.String(reason) - return nil - } -} - type PurgeInstanceFilter struct { CreatedTimeFrom time.Time CreatedTimeTo time.Time @@ -116,15 +106,6 @@ func (request PurgeInstancesRequest) Validate() error { return nil } -type CreateTaskHubOptions func(*protos.CreateTaskHubRequest) error - -func WithRecreateTaskHub(recreateIfExists bool) CreateTaskHubOptions { - return func(req *protos.CreateTaskHubRequest) error { - req.RecreateIfExists = recreateIfExists - return nil - } -} - func NormalizeInstanceQueryPageSize(pageSize int) (int, error) { switch { case pageSize < 0: diff --git a/client/client_grpc_management.go b/client/client_grpc_management.go index a566fc66..6e850001 100644 --- a/client/client_grpc_management.go +++ b/client/client_grpc_management.go @@ -209,19 +209,6 @@ func (c *TaskHubGrpcClient) RestartInstance(ctx context.Context, id api.Instance return api.InstanceID(resp.GetInstanceId()), nil } -func (c *TaskHubGrpcClient) RewindInstance(ctx context.Context, id api.InstanceID, opts ...api.RewindOptions) error { - req := &protos.RewindInstanceRequest{InstanceId: string(id)} - for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure rewind request: %w", api.WrapInvalidArgument(err)) - } - } - if _, err := c.client.RewindInstance(ctx, req); err != nil { - return clientRPCError(ctx, "failed to rewind orchestration instance", err) - } - return nil -} - func (c *TaskHubGrpcClient) PurgeInstances(ctx context.Context, request api.PurgeInstancesRequest) (*api.PurgeInstancesResult, error) { if err := request.Validate(); err != nil { return nil, err @@ -297,54 +284,6 @@ func (c *TaskHubGrpcClient) pollPurgeInstances(ctx context.Context, req *protos. } } -func (c *TaskHubGrpcClient) SkipGracefulOrchestrationTerminations(ctx context.Context, ids []api.InstanceID, reason string) ([]api.InstanceID, error) { - if len(ids) == 0 { - return nil, api.WrapInvalidArgument(errors.New("at least one instance ID is required")) - } - if len(ids) > api.MaxInstanceBatchSize { - return nil, api.WrapInvalidArgument(fmt.Errorf("instance batch cannot exceed %d IDs", api.MaxInstanceBatchSize)) - } - instanceIDs := make([]string, len(ids)) - for i, id := range ids { - if id == api.EmptyInstanceID { - return nil, api.WrapInvalidArgument(errors.New("instance ID cannot be empty")) - } - instanceIDs[i] = string(id) - } - resp, err := c.client.SkipGracefulOrchestrationTerminations(ctx, &protos.SkipGracefulOrchestrationTerminationsRequest{ - InstanceBatch: &protos.InstanceBatch{InstanceIds: instanceIDs}, - Reason: stringValue(reason), - }) - if err != nil { - return nil, clientRPCError(ctx, "failed to skip graceful orchestration terminations", err) - } - unterminated := make([]api.InstanceID, 0, len(resp.GetUnterminatedInstanceIds())) - for _, id := range resp.GetUnterminatedInstanceIds() { - unterminated = append(unterminated, api.InstanceID(id)) - } - return unterminated, nil -} - -func (c *TaskHubGrpcClient) CreateTaskHub(ctx context.Context, opts ...api.CreateTaskHubOptions) error { - req := &protos.CreateTaskHubRequest{} - for _, configure := range opts { - if err := configure(req); err != nil { - return fmt.Errorf("failed to configure task hub creation request: %w", api.WrapInvalidArgument(err)) - } - } - if _, err := c.client.CreateTaskHub(ctx, req); err != nil { - return clientRPCError(ctx, "failed to create task hub", err) - } - return nil -} - -func (c *TaskHubGrpcClient) DeleteTaskHub(ctx context.Context) error { - if _, err := c.client.DeleteTaskHub(ctx, &protos.DeleteTaskHubRequest{}); err != nil { - return clientRPCError(ctx, "failed to delete task hub", err) - } - return nil -} - func makePurgeFilterRequest(request api.PurgeInstancesRequest) (*protos.PurgeInstancesRequest, error) { filter := request.Filter wireFilter := &protos.PurgeInstanceFilter{ diff --git a/client/client_grpc_management_test.go b/client/client_grpc_management_test.go index e933ce3d..d7d65ed4 100644 --- a/client/client_grpc_management_test.go +++ b/client/client_grpc_management_test.go @@ -2,8 +2,6 @@ package client import ( "context" - "errors" - "sync" "testing" "time" @@ -21,35 +19,16 @@ import ( type managementServer struct { protos.UnimplementedTaskHubSidecarServiceServer - mu sync.Mutex - created bool - deleted bool + queryErr error } -func (s *managementServer) CreateTaskHub( - context.Context, - *protos.CreateTaskHubRequest, -) (*protos.CreateTaskHubResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.created = true - return &protos.CreateTaskHubResponse{}, nil -} - -func (s *managementServer) DeleteTaskHub( - context.Context, - *protos.DeleteTaskHubRequest, -) (*protos.DeleteTaskHubResponse, error) { - s.mu.Lock() - defer s.mu.Unlock() - s.deleted = true - return &protos.DeleteTaskHubResponse{}, nil -} - -func (*managementServer) QueryInstances( +func (s *managementServer) QueryInstances( _ context.Context, req *protos.QueryInstancesRequest, ) (*protos.QueryInstancesResponse, error) { + if s.queryErr != nil { + return nil, s.queryErr + } if req.GetQuery().GetMaxInstanceCount() <= 0 { return nil, status.Error(codes.InvalidArgument, "page size must be positive") } @@ -66,60 +45,6 @@ func (*managementServer) ListInstanceIds( return &protos.ListInstanceIdsResponse{}, nil } -func (s *managementServer) lifecycleCalls() (bool, bool) { - s.mu.Lock() - defer s.mu.Unlock() - return s.created, s.deleted -} - -// lifecycleErrorServer fails both lifecycle RPCs with the durable error reasons -// a task hub service attaches, so the client-side sentinel mapping is observed. -type lifecycleErrorServer struct { - protos.UnimplementedTaskHubSidecarServiceServer -} - -func (*lifecycleErrorServer) CreateTaskHub( - context.Context, - *protos.CreateTaskHubRequest, -) (*protos.CreateTaskHubResponse, error) { - return nil, grpcerrors.New( - codes.AlreadyExists, - ErrTaskHubExists.Error(), - grpcerrors.ReasonTaskHubExists, - ) -} - -func (*lifecycleErrorServer) DeleteTaskHub( - context.Context, - *protos.DeleteTaskHubRequest, -) (*protos.DeleteTaskHubResponse, error) { - return nil, grpcerrors.New( - codes.NotFound, - ErrTaskHubNotFound.Error(), - grpcerrors.ReasonTaskHubNotFound, - ) -} - -// bareLifecycleErrorServer omits the durable error reason detail so the -// status-code-only fallback in clientRPCError is exercised too. -type bareLifecycleErrorServer struct { - protos.UnimplementedTaskHubSidecarServiceServer -} - -func (*bareLifecycleErrorServer) CreateTaskHub( - context.Context, - *protos.CreateTaskHubRequest, -) (*protos.CreateTaskHubResponse, error) { - return nil, status.Error(codes.AlreadyExists, "task hub already exists") -} - -func (*bareLifecycleErrorServer) DeleteTaskHub( - context.Context, - *protos.DeleteTaskHubRequest, -) (*protos.DeleteTaskHubResponse, error) { - return nil, status.Error(codes.NotFound, "task hub not found") -} - func TestTaskHubGrpcManagementOverBufconn(t *testing.T) { server := &managementServer{} client := startQueryClient(t, server) @@ -127,38 +52,45 @@ func TestTaskHubGrpcManagementOverBufconn(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - require.NoError(t, client.CreateTaskHub(ctx)) query, err := client.QueryInstances(ctx, api.OrchestrationQuery{PageSize: 10}) require.NoError(t, err) require.Empty(t, query.Orchestrations) ids, err := client.ListInstanceIDs(ctx, api.InstanceIDQuery{PageSize: 10}) require.NoError(t, err) require.Empty(t, ids.InstanceIDs) - require.NoError(t, client.DeleteTaskHub(ctx)) - - created, deleted := server.lifecycleCalls() - require.True(t, created) - require.True(t, deleted) } -func TestTaskHubLifecycleErrorsRoundTripOverGRPC(t *testing.T) { +func TestTaskHubGrpcManagementErrorsRoundTrip(t *testing.T) { for _, test := range []struct { - name string - server protos.TaskHubSidecarServiceServer + name string + err error + want error }{ - {name: "with-durable-error-reason", server: &lifecycleErrorServer{}}, - {name: "status-code-only", server: &bareLifecycleErrorServer{}}, + { + name: "missing-task-hub", + err: grpcerrors.New(codes.NotFound, ErrTaskHubNotFound.Error(), grpcerrors.ReasonTaskHubNotFound), + want: ErrTaskHubNotFound, + }, + { + name: "missing-instance", + err: status.Error(codes.NotFound, "instance not found"), + want: api.ErrInstanceNotFound, + }, + { + name: "unsupported-feature", + err: status.Error(codes.Unimplemented, "query is not implemented"), + want: api.ErrFeatureNotSupported, + }, } { t.Run(test.name, func(t *testing.T) { - client := startQueryClient(t, test.server) + client := startQueryClient(t, &managementServer{queryErr: test.err}) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - err := client.CreateTaskHub(ctx) - require.True(t, errors.Is(err, ErrTaskHubExists), "CreateTaskHub() error = %v", err) - err = client.DeleteTaskHub(ctx) - require.True(t, errors.Is(err, ErrTaskHubNotFound), "DeleteTaskHub() error = %v", err) + _, err := client.QueryInstances(ctx, api.OrchestrationQuery{PageSize: 10}) + require.ErrorIs(t, err, test.want) + require.Equal(t, status.Code(test.err), status.Code(err)) }) } } diff --git a/client/errors.go b/client/errors.go index 04defa8f..3d8838f8 100644 --- a/client/errors.go +++ b/client/errors.go @@ -9,9 +9,7 @@ import ( ) var ( - // ErrTaskHubExists indicates that task-hub creation targeted an existing hub. - ErrTaskHubExists = errors.New("task hub already exists") - // ErrTaskHubNotFound indicates that a task-hub operation targeted a missing hub. + // ErrTaskHubNotFound indicates that the configured task hub does not exist. ErrTaskHubNotFound = errors.New("task hub not found") // ErrStreamedHistoryLimitExceeded indicates that a worker refused to retain // an orchestration history beyond its configured safety limit. diff --git a/client/grpc_errors.go b/client/grpc_errors.go index f457db96..add96690 100644 --- a/client/grpc_errors.go +++ b/client/grpc_errors.go @@ -56,19 +56,11 @@ func clientRPCError(ctx context.Context, operation string, err error) error { } else { switch code { case codes.AlreadyExists: - if operation == "failed to create task hub" { - category = ErrTaskHubExists - } else { - category = api.ErrDuplicateInstance - } + category = api.ErrDuplicateInstance case codes.InvalidArgument: category = api.ErrInvalidArgument case codes.NotFound: - if operation == "failed to delete task hub" { - category = ErrTaskHubNotFound - } else { - category = api.ErrInstanceNotFound - } + category = api.ErrInstanceNotFound case codes.FailedPrecondition: if strings.HasPrefix(grpcStatus.Message(), api.ErrNotCompleted.Error()) { category = api.ErrNotCompleted @@ -104,8 +96,6 @@ func clientErrorReasonCategory(reason string) error { return api.ErrInvalidState case grpcerrors.ReasonNotCompleted: return api.ErrNotCompleted - case grpcerrors.ReasonTaskHubExists: - return ErrTaskHubExists case grpcerrors.ReasonTaskHubNotFound: return ErrTaskHubNotFound default: diff --git a/client/grpc_errors_test.go b/client/grpc_errors_test.go index a456d351..48e26a2b 100644 --- a/client/grpc_errors_test.go +++ b/client/grpc_errors_test.go @@ -55,21 +55,18 @@ func TestClientRPCErrorPrefersCallerContext(t *testing.T) { } } -func TestClientRPCErrorPreservesLifecycleAndStateCategories(t *testing.T) { +func TestClientRPCErrorPreservesTaskHubAndStateCategories(t *testing.T) { tests := []struct { code codes.Code message string category error }{ - {codes.AlreadyExists, ErrTaskHubExists.Error(), ErrTaskHubExists}, {codes.NotFound, ErrTaskHubNotFound.Error(), ErrTaskHubNotFound}, {codes.FailedPrecondition, api.ErrNotCompleted.Error(), api.ErrNotCompleted}, } for _, test := range tests { var reason string switch { - case errors.Is(test.category, ErrTaskHubExists): - reason = grpcerrors.ReasonTaskHubExists case errors.Is(test.category, ErrTaskHubNotFound): reason = grpcerrors.ReasonTaskHubNotFound case errors.Is(test.category, api.ErrNotCompleted): diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 1d0b4702..9eb8cef4 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -129,12 +129,16 @@ configurations is replay-breaking for affected in-flight orchestrations. ### Advanced management `TaskHubGrpcClient` exposes bounded `QueryInstances` and `ListInstanceIDs` -operations with opaque continuation tokens, plus `RestartInstance`, -`RewindInstance`, batch/filter `PurgeInstances`, -`SkipGracefulOrchestrationTerminations`, and task-hub lifecycle RPCs. Queries +operations with opaque continuation tokens, plus `RestartInstance` and +batch/filter `PurgeInstances`. Queries can filter locally by exact tag key/value pairs when the current wire contract does not carry tag filters. +Provision and delete task hubs through the Azure control plane or Azure CLI, +not SDK RPCs. The SDK does not expose task-hub lifecycle, rewind, or +skip-graceful-termination operations. Use `TerminateOrchestration` for normal +orchestration termination. + `GetOrchestrationHistory` returns API-owned history records with validated event and approximate byte caps. `StreamOrchestrationHistory` invokes a callback in service order without retaining the history in memory. Both preserve execution identity, @@ -143,9 +147,7 @@ events. Serialized payloads remain raw until a `ReadInput`, `ReadResult`, or `ReadData` helper applies the configured data converter. The current DTS emulator supports query, restart, and batch purge, but has known -service limitations: `SkipGracefulOrchestrationTerminations` -is unimplemented, rewind can return success without transitioning the failed -instance, filtered purge can complete without deleting matches, and +service limitations: filtered purge can complete without deleting matches, and `ListInstanceIds` can omit matching IDs. The emulator integration tests record these limitations explicitly. @@ -251,8 +253,6 @@ before large-payload externalization and after hydration. Converter errors are returned; the SDK never retries a payload with JSON. Raw input/output APIs and serialized metadata fields bypass conversion. Converter identity is not stored by the protocol, so deployments must retain backward decoding compatibility. -The legacy skip-graceful termination `reason` remains a plain protocol string -for cross-version service compatibility. ### Large payloads @@ -305,11 +305,8 @@ cancels them only if the shutdown context expires. | Feature | Status | | --- | --- | | Schedule, bounded query/list, and wait for orchestrations | Supported | -| Tags on schedule, metadata, query, sub-orchestration, continue-as-new, restart, and rewind | Supported; distinct from immutable context fields | +| Tags on schedule, metadata, query, sub-orchestration, continue-as-new, and restart | Supported; distinct from immutable context fields | | Restart and batch/filter purge | Supported; see emulator limitations above | -| Rewind | Client and wire support are complete; current emulator does not transition instances | -| Skip-graceful termination | Client and wire support are complete; current emulator returns `Unimplemented` | -| Task-hub create/delete | Client and wire support are complete; remote-service behavior is provider-specific | | Raise events, suspend/resume, terminate, and single-instance purge | Supported | | Orchestration and activity execution | Supported | | Bounded orchestration/activity/entity concurrency | Supported | diff --git a/internal/grpcerrors/reasons.go b/internal/grpcerrors/reasons.go index 337722a4..4a6e6cba 100644 --- a/internal/grpcerrors/reasons.go +++ b/internal/grpcerrors/reasons.go @@ -15,7 +15,6 @@ const ( ReasonInvalidArgument = "INVALID_ARGUMENT" ReasonInvalidState = "INVALID_STATE" ReasonNotCompleted = "NOT_COMPLETED" - ReasonTaskHubExists = "TASK_HUB_EXISTS" ReasonTaskHubNotFound = "TASK_HUB_NOT_FOUND" ) diff --git a/tests/durabletaskscheduler/dts_emulator_test.go b/tests/durabletaskscheduler/dts_emulator_test.go index 9f61ffd6..eb5030f7 100644 --- a/tests/durabletaskscheduler/dts_emulator_test.go +++ b/tests/durabletaskscheduler/dts_emulator_test.go @@ -259,32 +259,13 @@ func TestDTSEmulatorAdvancedManagementOperations(t *testing.T) { } return "done", nil })) - var rewindAttempts atomic.Int32 - require.NoError(t, registry.AddActivityN("DTSAdvancedRewindActivity", func(task.ActivityContext) (any, error) { - if rewindAttempts.Add(1) == 1 { - return nil, errors.New("first attempt fails") - } - return "recovered", nil - })) - require.NoError(t, registry.AddOrchestratorN("DTSAdvancedRewind", func(ctx *task.OrchestrationContext) (any, error) { - var result string - if err := ctx.CallActivity("DTSAdvancedRewindActivity").Await(&result); err != nil { - return nil, err - } - return result, nil + require.NoError(t, registry.AddOrchestratorN("DTSAdvancedFail", func(*task.OrchestrationContext) (any, error) { + return nil, errors.New("expected failure") })) managementClient, _, _ := startEmulatorClientAndWorker(t, registry) ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - t.Run("create-task-hub", func(t *testing.T) { - err := managementClient.CreateTaskHub(ctx) - if errors.Is(err, api.ErrFeatureNotSupported) { - t.Log("DTS emulator limitation: CreateTaskHub is not implemented") - return - } - require.NoError(t, err) - }) prefix := "go-advanced-" + uuid.NewString() completedIDs := make([]api.InstanceID, 0, 3) for index := range 3 { @@ -394,63 +375,23 @@ func TestDTSEmulatorAdvancedManagementOperations(t *testing.T) { require.NoError(t, err) _, err = managementClient.WaitForOrchestrationStart(ctx, waitID) require.NoError(t, err) - t.Run("skip-graceful-termination", func(t *testing.T) { - unterminated, err := managementClient.SkipGracefulOrchestrationTerminations(ctx, []api.InstanceID{waitID}, "test") - if errors.Is(err, api.ErrFeatureNotSupported) { - t.Log("DTS emulator limitation: SkipGracefulOrchestrationTerminations is not implemented") - require.NoError(t, managementClient.TerminateOrchestration(ctx, waitID)) - _, waitErr := managementClient.WaitForOrchestrationCompletion(ctx, waitID) - require.NoError(t, waitErr) - return - } + t.Run("terminate", func(t *testing.T) { + require.NoError(t, managementClient.TerminateOrchestration(ctx, waitID)) + terminated, err := managementClient.WaitForOrchestrationCompletion(ctx, waitID) require.NoError(t, err) - require.Empty(t, unterminated) + require.Equal(t, api.RUNTIME_STATUS_TERMINATED, terminated.RuntimeStatus) }) - rewindID := api.InstanceID(prefix + "-rewind") + failedID := api.InstanceID(prefix + "-failed") _, err = managementClient.ScheduleNewOrchestration( ctx, - "DTSAdvancedRewind", - api.WithInstanceID(rewindID), + "DTSAdvancedFail", + api.WithInstanceID(failedID), ) require.NoError(t, err) - failed, err := managementClient.WaitForOrchestrationCompletion(ctx, rewindID) + failed, err := managementClient.WaitForOrchestrationCompletion(ctx, failedID) require.NoError(t, err) require.Equal(t, api.RUNTIME_STATUS_FAILED, failed.RuntimeStatus) - failedExecutionID := failed.ExecutionID - t.Run("rewind", func(t *testing.T) { - err := managementClient.RewindInstance(ctx, rewindID, api.WithRewindReason("retry")) - if errors.Is(err, api.ErrFeatureNotSupported) { - t.Log("DTS emulator limitation: RewindInstance is not implemented") - return - } - require.NoError(t, err) - transitioned := false - deadline := time.Now().Add(2 * time.Second) - for time.Now().Before(deadline) { - current, fetchErr := managementClient.FetchOrchestrationMetadata(ctx, rewindID) - if fetchErr == nil && - current.RuntimeStatus != api.RUNTIME_STATUS_FAILED && - current.ExecutionID != failedExecutionID { - transitioned = true - break - } - time.Sleep(50 * time.Millisecond) - } - if !transitioned { - t.Log("DTS emulator limitation: RewindInstance returns success without transitioning the failed instance") - return - } - rewound, err := managementClient.WaitForOrchestrationCompletion(ctx, rewindID, api.WithFetchPayloads(true)) - require.NoError(t, err) - require.Equal(t, api.RUNTIME_STATUS_COMPLETED, rewound.RuntimeStatus) - require.EqualValues(t, 2, rewindAttempts.Load()) - if rewound.SerializedOutput == "" { - t.Log("DTS emulator limitation: rewound completion output is not returned") - } else { - require.Equal(t, `"recovered"`, rewound.SerializedOutput) - } - }) t.Run("filter-purge", func(t *testing.T) { filterStart := time.Now().UTC() @@ -489,11 +430,11 @@ func TestDTSEmulatorAdvancedManagementOperations(t *testing.T) { t.Run("batch-purge", func(t *testing.T) { result, err := managementClient.PurgeInstances(ctx, api.PurgeInstancesRequest{ - InstanceIDs: append(completedIDs, waitID, rewindID), + InstanceIDs: append(completedIDs, waitID, failedID), }) if errors.Is(err, api.ErrFeatureNotSupported) { t.Log("DTS emulator limitation: batch PurgeInstances is not implemented") - for _, id := range append(completedIDs, waitID, rewindID) { + for _, id := range append(completedIDs, waitID, failedID) { purgeErr := managementClient.PurgeOrchestrationState(ctx, id) require.True(t, purgeErr == nil || errors.Is(purgeErr, api.ErrInstanceNotFound)) } From a1812015eb34cf265af8a06c981e9dd7e6fc2774 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:22:35 -0700 Subject: [PATCH 05/26] restore remote trace context for activity instrumentation Preserve DTS-provided W3C trace parents on ActivityContext without emitting duplicate durable spans. The real OTLP sample exposed this missing context boundary; add regression coverage for remote IDs, tracestate, values, and deadlines. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 1 + README.md | 5 ++++ internal/helpers/tracing.go | 18 ++++++++++---- task/activity_trace_test.go | 49 +++++++++++++++++++++++++++++++++++++ task/executor.go | 1 + 5 files changed, 69 insertions(+), 5 deletions(-) create mode 100644 task/activity_trace_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 45c4e1f5..203cf736 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -53,6 +53,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Restore DTS-provided activity trace parents on `ActivityContext.Context()` without emitting duplicate SDK durable spans, preserving trace continuity for application instrumentation. - Missing-instance orchestration waits now return `api.ErrInstanceNotFound` immediately instead of retrying `NotFound` until the caller deadline. - After a worker has started, it reconnects with bounded backoff after `Unauthenticated` or `PermissionDenied` stream and reconnect-handshake responses so refreshed credentials and propagated RBAC can recover without a process restart. The initial `Hello` remains fail-fast. - Oversized orchestration responses are checked after large-payload externalization against the smaller of the 3.9 MiB worker safety bound and the configured gRPC send limit. The Go worker intentionally does not use the deprecated response-chunking fields; a response that still exceeds the effective limit now fails once with non-retriable `api.ErrorTypeOrchestratorResponseTooLarge` guidance instead of repeatedly hitting `ResourceExhausted`. diff --git a/README.md b/README.md index 516d7d10..10cdc754 100644 --- a/README.md +++ b/README.md @@ -452,6 +452,11 @@ The [`payload`](./payload) package includes Azure Blob Storage support. It emits The SDK sends the W3C trace context of a sampled caller when it schedules an orchestration. DTS owns the spans for the orchestrations, activities, timers, and sub-orchestrations. Your application code can use standard [OpenTelemetry](https://opentelemetry.io/) instrumentation. Use it for caller spans, custom activity spans, and outbound dependencies. +When DTS supplies an activity trace parent, the SDK restores it on +`ActivityContext.Context()` without creating another durable-operation span. +Application instrumentation can therefore attach outbound requests to the +service-owned trace. + This example sends the traces of your process to an [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) over OTLP/HTTP. Configure the DTS telemetry separately for the service-owned spans. ```go diff --git a/internal/helpers/tracing.go b/internal/helpers/tracing.go index 41d137c3..7791e832 100644 --- a/internal/helpers/tracing.go +++ b/internal/helpers/tracing.go @@ -43,18 +43,26 @@ func traceStateValue(state trace.TraceState) *wrapperspb.StringValue { return nil } +// ContextWithTraceContext restores a wire parent without starting or recording a span. +func ContextWithTraceContext(ctx context.Context, value *protos.TraceContext) context.Context { + if value == nil { + return ctx + } + carrier := propagation.MapCarrier{"traceparent": value.GetTraceParent()} + if state := value.GetTraceState().GetValue(); state != "" { + carrier.Set("tracestate", state) + } + return propagation.TraceContext{}.Extract(ctx, carrier) +} + // OrchestratorActionTraceContext creates the trace context for a service-owned // activity or sub-orchestration scheduling span. func OrchestratorActionTraceContext(parent *protos.TraceContext) (*protos.TraceContext, error) { if parent == nil { return nil, nil } - carrier := propagation.MapCarrier{"traceparent": parent.GetTraceParent()} - if traceState := parent.GetTraceState().GetValue(); traceState != "" { - carrier.Set("tracestate", traceState) - } parentContext := trace.SpanContextFromContext( - propagation.TraceContext{}.Extract(context.Background(), carrier), + ContextWithTraceContext(context.Background(), parent), ) if !parentContext.IsValid() { return nil, fmt.Errorf("invalid parent trace context") diff --git a/task/activity_trace_test.go b/task/activity_trace_test.go new file mode 100644 index 00000000..3b28e9cc --- /dev/null +++ b/task/activity_trace_test.go @@ -0,0 +1,49 @@ +package task + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "go.opentelemetry.io/otel/trace" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestActivityRestoresRemoteTraceParentWithoutRecordingSpan(t *testing.T) { + type contextKey struct{} + parent, cancel := context.WithTimeout(context.WithValue(context.Background(), contextKey{}, "retained"), time.Minute) + defer cancel() + deadline, _ := parent.Deadline() + registry := NewTaskRegistry() + if err := registry.AddActivityN("trace-parent", func(ctx ActivityContext) (any, error) { + span := trace.SpanFromContext(ctx.Context()) + sc := span.SpanContext() + if !sc.IsRemote() || !sc.IsSampled() || span.IsRecording() || + sc.TraceID().String() != "0123456789abcdef0123456789abcdef" || + sc.SpanID().String() != "0123456789abcdef" || sc.TraceState().String() != "vendor=value" { + return nil, errors.New("activity did not receive the non-recording remote trace context") + } + actualDeadline, ok := ctx.Context().Deadline() + if !ok || actualDeadline != deadline || ctx.Context().Value(contextKey{}) != "retained" || + ctx.Context().Done() != parent.Done() { + return nil, errors.New("trace extraction changed the host context") + } + return "linked", nil + }); err != nil { + t.Fatal(err) + } + event := helpers.NewTaskScheduledEvent(0, "trace-parent", nil, nil, &protos.TraceContext{ + TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", + TraceState: wrapperspb.String("vendor=value"), + }) + result, err := NewTaskExecutor(registry).ExecuteActivity(parent, "instance", event) + if err != nil { + t.Fatal(err) + } + if result.GetTaskCompleted().GetResult().GetValue() != `"linked"` { + t.Fatalf("activity trace propagation failed: %v", result) + } +} diff --git a/task/executor.go b/task/executor.go index 1b644ce7..f610b402 100644 --- a/task/executor.go +++ b/task/executor.go @@ -187,6 +187,7 @@ func (te *taskExecutor) ExecuteActivity(ctx context.Context, id api.InstanceID, return helpers.NewTaskFailedEvent(e.EventId, versionFailureDetails(versionErr)), nil } ctx = api.ContextWithFields(ctx, te.contextFields) + ctx = helpers.ContextWithTraceContext(ctx, ts.GetParentTraceContext()) tagInfo, tagFields := contextprop.Decode(ts.GetTags()) ctx = api.ContextWithFields(ctx, tagFields) orchestrationInfo, _ := api.OrchestrationContextInfoFromContext(ctx) From 2a7e3a018f3f6799e23708ea1d20546a93d370c1 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:43:03 -0700 Subject: [PATCH 06/26] add validated core DTS workflow samples Demonstrate typed activity sequences, aggregation, durable coroutines, timers, and external events with owned-resource cleanup and observable outcomes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/coroutines/README.md | 46 +++ samples/coroutines/coroutines.go | 152 +++++++++ samples/durabletaskscheduler/README.md | 42 +++ .../durabletaskscheduler.go | 124 +++---- samples/externalevents/README.md | 47 +++ samples/externalevents/externalevents.go | 317 ++++++++++++++++-- samples/externalevents/externalevents_test.go | 10 + samples/internal/dtssample/validation.go | 60 ++++ samples/internal/dtssample/validation_test.go | 33 ++ samples/parallel/README.md | 45 +++ samples/parallel/parallel.go | 205 ++++++----- samples/timers/README.md | 46 +++ samples/timers/timers.go | 182 ++++++++++ 13 files changed, 1115 insertions(+), 194 deletions(-) create mode 100644 samples/coroutines/README.md create mode 100644 samples/coroutines/coroutines.go create mode 100644 samples/durabletaskscheduler/README.md create mode 100644 samples/externalevents/README.md create mode 100644 samples/internal/dtssample/validation.go create mode 100644 samples/internal/dtssample/validation_test.go create mode 100644 samples/parallel/README.md create mode 100644 samples/timers/README.md create mode 100644 samples/timers/timers.go diff --git a/samples/coroutines/README.md b/samples/coroutines/README.md new file mode 100644 index 00000000..846526d0 --- /dev/null +++ b/samples/coroutines/README.md @@ -0,0 +1,46 @@ +# Coroutines sample + +## Features + +- Starts bounded orchestration coroutines with `ctx.Go`. +- Coordinates coroutine completion with `ctx.NewWaitGroup`. +- Uses `Select` on a typed event channel and a durable timer from a child + cancellation scope. +- Cancels the losing timer when the approval event wins. +- Validates the returned doubled values and selected event outcome. + +## Prerequisites + +Start a Durable Task Scheduler endpoint or emulator and set: + +```sh +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```sh +go run ./samples/coroutines +``` + +## Expected result + +The host raises the `Approval` event after the orchestration starts. The event +wins over the durable timer, the timer scope is canceled, three activity-backed +coroutines return `4`, `8`, and `12`, and the sum is verified as `24`. On success +it prints: + +```text +SAMPLE_OK coroutines +``` + +## Cleanup + +The sample terminates if needed and recursively purges only its generated +`sample-coroutines-*` orchestration instance before shutting down. + +## Noninteractive command + +```sh +DTS_CONNECTION_STRING="$DTS_CONNECTION_STRING" go run ./samples/coroutines +``` diff --git a/samples/coroutines/coroutines.go b/samples/coroutines/coroutines.go new file mode 100644 index 00000000..af45a4e7 --- /dev/null +++ b/samples/coroutines/coroutines.go @@ -0,0 +1,152 @@ +// Command coroutines demonstrates orchestration coroutines, durable wait +// groups, Select, and child cancellation scopes. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/coroutines +package main + +import ( + "context" + "errors" + "fmt" + "log" + "reflect" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +type coroutineInput struct { + Values []int `json:"values"` + ApprovalEvent string `json:"approvalEvent"` + ApprovalWindow time.Duration `json:"approvalWindow"` +} + +type coroutineOutput struct { + Selected string `json:"selected"` + Doubled []int `json:"doubled"` + Sum int `json:"sum"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK coroutines") +} + +func run() (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("CoroutineFanInOrchestrator", CoroutineFanInOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) + } + if err := registry.AddActivityN("DoubleValue", DoubleValue); err != nil { + return fmt.Errorf("failed to register activity: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, app.Shutdown()) }() + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...)) }() + + id := dtssample.NewInstanceID("coroutines") + ownedIDs = append(ownedIDs, id) + input := coroutineInput{ + Values: []int{2, 4, 6}, + ApprovalEvent: "approved", + ApprovalWindow: 30 * time.Second, + } + if _, err := app.Client.ScheduleNewOrchestration( + ctx, + "CoroutineFanInOrchestrator", + api.WithInstanceID(id), + api.WithInput(input), + ); err != nil { + return fmt.Errorf("failed to schedule coroutine orchestration: %w", err) + } + if _, err := app.Client.WaitForOrchestrationStart(ctx, id); err != nil { + return fmt.Errorf("failed to wait for coroutine orchestration start: %w", err) + } + if err := app.Client.RaiseEvent(ctx, id, "Approval", api.WithEventPayload(input.ApprovalEvent)); err != nil { + return fmt.Errorf("failed to raise approval event: %w", err) + } + + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for coroutine orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output coroutineOutput + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode coroutine output: %w", err) + } + expected := coroutineOutput{ + Selected: "event:approved", + Doubled: []int{4, 8, 12}, + Sum: 24, + } + if !reflect.DeepEqual(output, expected) { + return fmt.Errorf("coroutine output = %#v, want %#v", output, expected) + } + return nil +} + +func CoroutineFanInOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input coroutineInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + + doubled := make([]int, len(input.Values)) + waitGroup := ctx.NewWaitGroup() + waitGroup.Add(len(input.Values)) + for i, value := range input.Values { + i, value := i, value + ctx.Go(func(ctx *task.OrchestrationContext) { + defer waitGroup.Done() + var result int + if err := ctx.CallActivity("DoubleValue", task.WithActivityInput(value)).Await(&result); err != nil { + panic(err) + } + doubled[i] = result + }) + } + + timerCtx, cancelTimer := ctx.WithCancel() + timer := timerCtx.CreateTimer(input.ApprovalWindow) + approvals := task.NewEventChannel[string](ctx, "Approval") + selected := "timeout" + ctx.Select( + task.OnEvent(approvals, func(value string) { + selected = "event:" + value + cancelTimer() + }), + task.OnTask(timer, func(task.Task) { + selected = "timeout" + }), + ) + + waitGroup.Wait(ctx) + sum := 0 + for _, value := range doubled { + sum += value + } + return coroutineOutput{Selected: selected, Doubled: doubled, Sum: sum}, nil +} + +func DoubleValue(ctx task.ActivityContext) (any, error) { + var value int + if err := ctx.GetInput(&value); err != nil { + return nil, err + } + return value * 2, nil +} diff --git a/samples/durabletaskscheduler/README.md b/samples/durabletaskscheduler/README.md new file mode 100644 index 00000000..b98244cf --- /dev/null +++ b/samples/durabletaskscheduler/README.md @@ -0,0 +1,42 @@ +# Durable Task Scheduler sample + +## Features + +- Connects a Durable Task Scheduler client and worker using `DTS_CONNECTION_STRING`. +- Schedules an orchestration with an explicit sample-owned instance ID. +- Passes typed orchestration input and validates typed activity output. +- Cleans up only the orchestration instance created by this run. + +## Prerequisites + +Start a Durable Task Scheduler endpoint or emulator and set: + +```sh +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```sh +go run ./samples/durabletaskscheduler +``` + +## Expected result + +The orchestration calls the `SayHello` activity for Tokyo, London, and Seattle, +decodes the typed output, and verifies the exact greetings. On success it prints: + +```text +SAMPLE_OK durabletaskscheduler +``` + +## Cleanup + +The sample terminates if needed and recursively purges only its generated +`sample-durabletaskscheduler-*` orchestration instance before shutting down. + +## Noninteractive command + +```sh +DTS_CONNECTION_STRING="$DTS_CONNECTION_STRING" go run ./samples/durabletaskscheduler +``` diff --git a/samples/durabletaskscheduler/durabletaskscheduler.go b/samples/durabletaskscheduler/durabletaskscheduler.go index 32d137b2..f1966535 100644 --- a/samples/durabletaskscheduler/durabletaskscheduler.go +++ b/samples/durabletaskscheduler/durabletaskscheduler.go @@ -1,6 +1,5 @@ -// Command durabletaskscheduler demonstrates the Durable Task Scheduler surface: -// versioned registrations, tagged scheduling, orchestration history, and -// recurring scheduled tasks. +// Command durabletaskscheduler demonstrates the basic Durable Task Scheduler +// client and worker flow with typed orchestration input and output. // // export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" // go run ./samples/durabletaskscheduler @@ -8,117 +7,94 @@ package main import ( "context" - "encoding/json" + "errors" "fmt" "log" + "reflect" "time" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/durabletaskscheduler" "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +type itineraryInput struct { + Cities []string `json:"cities"` +} + +type greeting struct { + City string `json:"city"` + Message string `json:"message"` +} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK durabletaskscheduler") } -func run() error { - options, err := dtssample.Options() - if err != nil { - return err - } - options.Versioning = &task.VersioningOptions{ - DefaultVersion: "1.0", - MatchStrategy: task.VersionMatchNone, - } - +func run() (err error) { registry := task.NewTaskRegistry() - if err := registry.AddOrchestratorNVersion("ActivitySequence", "1.0", activitySequence); err != nil { - return err - } - if err := registry.AddActivityNVersion("SayHello", "1.0", sayHello); err != nil { + if err := registry.AddOrchestratorN("ActivitySequence", activitySequence); err != nil { return err } - if err := durabletaskscheduler.RegisterScheduledTasksWithDefaultVersion(registry, options.Versioning.DefaultVersion); err != nil { + if err := registry.AddActivityN("SayHello", sayHello); err != nil { return err } - // Connect a client and worker to the Durable Task Scheduler task hub ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - app, err := dtssample.StartWithOptions(ctx, options, registry, durabletaskscheduler.WithScheduledTasks()) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } - defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) - } - }() - schedulerClient := app.Client + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, app.Shutdown()) }() + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...)) }() - instanceID, err := schedulerClient.ScheduleNewOrchestration( + id := dtssample.NewInstanceID("durabletaskscheduler") + ownedIDs = append(ownedIDs, id) + input := itineraryInput{Cities: []string{"Tokyo", "London", "Seattle"}} + if _, err := app.Client.ScheduleNewOrchestration( ctx, "ActivitySequence", - api.WithTags(map[string]string{"sample": "durable-task-scheduler"}), - ) - if err != nil { - return err - } - metadata, err := schedulerClient.WaitForOrchestrationCompletion(ctx, instanceID) - if err != nil { - return err - } - output, err := json.MarshalIndent(metadata, "", " ") - if err != nil { - return err + api.WithInstanceID(id), + api.WithInput(input), + ); err != nil { + return fmt.Errorf("failed to schedule activity sequence: %w", err) } - fmt.Println(string(output)) - query, err := schedulerClient.QueryInstances(ctx, api.OrchestrationQuery{ - Tags: map[string]string{"sample": "durable-task-scheduler"}, - }) - if err != nil { - return err - } - fmt.Printf("matched %d tagged orchestration(s)\n", len(query.Orchestrations)) - history, err := schedulerClient.GetOrchestrationHistory(ctx, instanceID, api.HistoryQuery{ - ExecutionID: metadata.ExecutionID, - }) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return err + return fmt.Errorf("failed to wait for activity sequence: %w", err) } - fmt.Printf("history contains %d event(s)\n", len(history.Events)) - - scheduleID := "sample-hourly" - schedule, err := schedulerClient.ScheduledTasks().Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ - ScheduleID: scheduleID, - OrchestrationName: "ActivitySequence", - Interval: time.Hour, - StartAt: time.Now().UTC().Add(time.Hour), - Tags: map[string]string{"sample": "scheduled-task"}, - }) - if err != nil { + if err := dtssample.RequireCompleted(metadata); err != nil { return err } - description, err := schedule.Describe(ctx) - if err != nil { - return err + var output []greeting + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode activity sequence output: %w", err) } - fmt.Printf("schedule %s is %s; next run: %s\n", description.ScheduleID, description.Status, description.NextRunAt) - if err := schedule.Delete(ctx); err != nil { - return err + expected := []greeting{ + {City: "Tokyo", Message: "Hello, Tokyo!"}, + {City: "London", Message: "Hello, London!"}, + {City: "Seattle", Message: "Hello, Seattle!"}, + } + if !reflect.DeepEqual(output, expected) { + return fmt.Errorf("activity sequence output = %#v, want %#v", output, expected) } return nil } func activitySequence(ctx *task.OrchestrationContext) (any, error) { - var results []string - for _, city := range []string{"Tokyo", "London", "Seattle"} { - var result string + var input itineraryInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + results := make([]greeting, 0, len(input.Cities)) + for _, city := range input.Cities { + var result greeting if err := ctx.CallActivity("SayHello", task.WithActivityInput(city)).Await(&result); err != nil { return nil, err } @@ -132,5 +108,5 @@ func sayHello(ctx task.ActivityContext) (any, error) { if err := ctx.GetInput(&city); err != nil { return nil, err } - return "Hello, " + city + "!", nil + return greeting{City: city, Message: "Hello, " + city + "!"}, nil } diff --git a/samples/externalevents/README.md b/samples/externalevents/README.md new file mode 100644 index 00000000..fc4127dd --- /dev/null +++ b/samples/externalevents/README.md @@ -0,0 +1,47 @@ +# External events sample + +## Features + +- Reads the `Name` event payload from standard input asynchronously. +- Preserves the five-minute host timeout and the 30-second durable event timeout + on the single-event orchestration. +- Demonstrates repeated typed `EventChannel` receives and verifies event order. +- Demonstrates cross-instance durable `SendEvent`. +- Runs a controlled expected timeout path that completes successfully only when + the durable event wait times out. +- Cleans up only orchestration instances created by this run. + +## Prerequisites + +Start a Durable Task Scheduler endpoint or emulator and set: + +```sh +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```sh +printf 'Taylor\n' | go run ./samples/externalevents +``` + +## Expected result + +The sample verifies `Hello, Taylor!`, receives three typed checkpoint events in +order, sends a typed signal from one orchestration to another, and confirms the +expected timeout scenario. On success it prints: + +```text +SAMPLE_OK externalevents +``` + +## Cleanup + +The sample terminates if needed and recursively purges only generated +`sample-externalevents-*` orchestration instances before shutting down. + +## Noninteractive command + +```sh +printf 'Taylor\n' | DTS_CONNECTION_STRING="$DTS_CONNECTION_STRING" go run ./samples/externalevents +``` diff --git a/samples/externalevents/externalevents.go b/samples/externalevents/externalevents.go index 7428befa..68778747 100644 --- a/samples/externalevents/externalevents.go +++ b/samples/externalevents/externalevents.go @@ -1,16 +1,18 @@ -// Command externalevents demonstrates raising an external event into a running -// orchestration that is blocked on WaitForSingleEvent. +// Command externalevents demonstrates single external events, repeated typed +// event channels, cross-instance durable SendEvent, and an expected timeout. // // export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" -// go run ./samples/externalevents +// printf 'Taylor\n' | go run ./samples/externalevents package main import ( "context" + "errors" "fmt" "io" "log" "os" + "reflect" "time" "github.com/microsoft/durabletask-go/api" @@ -18,61 +20,260 @@ import ( "github.com/microsoft/durabletask-go/task" ) +const ( + sampleHostTimeout = 5 * time.Minute + nameEventTimeout = 30 * time.Second +) + +type checkpointEvent struct { + Index int `json:"index"` + Label string `json:"label"` +} + +type signalEvent struct { + From string `json:"from"` + Message string `json:"message"` +} + +type forwardInput struct { + Target api.InstanceID `json:"target"` + Signal signalEvent `json:"signal"` +} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK externalevents") } -func run() error { - // Create a new task registry and add the orchestrator and activities - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN("ExternalEventOrchestrator", ExternalEventOrchestrator); err != nil { - return fmt.Errorf("failed to register orchestrator: %w", err) +func run() (err error) { + registry := task.NewTaskRegistry() + for name, orchestrator := range map[string]func(*task.OrchestrationContext) (any, error){ + "ExternalEventOrchestrator": ExternalEventOrchestrator, + "RepeatedEventChannelOrch": RepeatedEventChannelOrchestrator, + "CrossInstanceReceiverOrch": CrossInstanceReceiverOrchestrator, + "CrossInstanceSenderOrch": CrossInstanceSenderOrchestrator, + "ExternalEventTimeoutOrchestrator": ExternalEventTimeoutOrchestrator, + } { + if err := registry.AddOrchestratorN(name, orchestrator); err != nil { + return fmt.Errorf("failed to register %s: %w", name, err) + } } - // Connect a client and worker to the Durable Task Scheduler task hub - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), sampleHostTimeout) defer cancel() - app, err := dtssample.Start(ctx, r) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } - defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) - } - }() + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, app.Shutdown()) }() + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...)) }() - // Start a new orchestration - id, err := app.Client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrator") - if err != nil { - return fmt.Errorf("failed to schedule new orchestration: %w", err) + if err := runSingleNameEventScenario(ctx, app.Client, &ownedIDs); err != nil { + return err } - if _, err := app.Client.WaitForOrchestrationStart(ctx, id); err != nil { - return fmt.Errorf("failed to wait for orchestration to start: %w", err) + if err := runRepeatedEventChannelScenario(ctx, app.Client, &ownedIDs); err != nil { + return err + } + if err := runCrossInstanceSendEventScenario(ctx, app.Client, &ownedIDs); err != nil { + return err + } + if err := runExpectedTimeoutScenario(ctx, app.Client, &ownedIDs); err != nil { + return err + } + return nil +} + +func runSingleNameEventScenario( + ctx context.Context, + client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationStart(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + RaiseEvent(context.Context, api.InstanceID, string, ...api.RaiseEventOptions) error + }, + ownedIDs *[]api.InstanceID, +) error { + id := dtssample.NewInstanceID("externalevents-name") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration(ctx, "ExternalEventOrchestrator", api.WithInstanceID(id)); err != nil { + return fmt.Errorf("failed to schedule name event orchestration: %w", err) + } + if _, err := client.WaitForOrchestrationStart(ctx, id); err != nil { + return fmt.Errorf("failed to wait for name event orchestration start: %w", err) } - // Keep stdin asynchronous, but interrupt the completion wait on input or RPC failure. waitCtx, cancelWait := context.WithCancelCause(ctx) defer cancelWait(nil) - fmt.Println("Enter your first name: ") go readAndRaiseEvent(waitCtx, cancelWait, os.Stdin, func(ctx context.Context, name string) error { - return app.Client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(name)) + return client.RaiseEvent(ctx, id, "Name", api.WithEventPayload(name)) }) - // After the orchestration receives the event, it should complete on its own - metadata, err := app.Client.WaitForOrchestrationCompletion(waitCtx, id) + metadata, err := client.WaitForOrchestrationCompletion(waitCtx, id, api.WithFetchPayloads(true)) if err != nil { if cause := context.Cause(waitCtx); cause != nil { err = cause } - return fmt.Errorf("failed to wait for orchestration to complete: %w", err) + return fmt.Errorf("failed to wait for name event orchestration completion: %w", err) } - if metadata.FailureDetails != nil { - log.Println("orchestration failed:", metadata.FailureDetails.ErrorMessage) - } else { - log.Println("orchestration completed:", metadata.SerializedOutput) + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode name event output: %w", err) + } + if output != "Hello, Taylor!" { + return fmt.Errorf("name event output = %q, want %q", output, "Hello, Taylor!") + } + return nil +} + +func runRepeatedEventChannelScenario( + ctx context.Context, + client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationStart(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + RaiseEvent(context.Context, api.InstanceID, string, ...api.RaiseEventOptions) error + }, + ownedIDs *[]api.InstanceID, +) error { + id := dtssample.NewInstanceID("externalevents-repeated") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + "RepeatedEventChannelOrch", + api.WithInstanceID(id), + api.WithInput(3), + ); err != nil { + return fmt.Errorf("failed to schedule repeated event orchestration: %w", err) + } + if _, err := client.WaitForOrchestrationStart(ctx, id); err != nil { + return fmt.Errorf("failed to wait for repeated event orchestration start: %w", err) + } + events := []checkpointEvent{ + {Index: 1, Label: "received"}, + {Index: 2, Label: "validated"}, + {Index: 3, Label: "approved"}, + } + for _, event := range events { + if err := client.RaiseEvent(ctx, id, "Checkpoint", api.WithEventPayload(event)); err != nil { + return fmt.Errorf("failed to raise checkpoint %d: %w", event.Index, err) + } + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for repeated event orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output []string + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode repeated event output: %w", err) + } + expected := []string{"01:received", "02:validated", "03:approved"} + if !reflect.DeepEqual(output, expected) { + return fmt.Errorf("repeated event output = %#v, want %#v", output, expected) + } + return nil +} + +func runCrossInstanceSendEventScenario( + ctx context.Context, + client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationStart(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + }, + ownedIDs *[]api.InstanceID, +) error { + receiverID := dtssample.NewInstanceID("externalevents-receiver") + senderID := dtssample.NewInstanceID("externalevents-sender") + *ownedIDs = append(*ownedIDs, receiverID, senderID) + if _, err := client.ScheduleNewOrchestration( + ctx, + "CrossInstanceReceiverOrch", + api.WithInstanceID(receiverID), + ); err != nil { + return fmt.Errorf("failed to schedule receiver orchestration: %w", err) + } + if _, err := client.WaitForOrchestrationStart(ctx, receiverID); err != nil { + return fmt.Errorf("failed to wait for receiver orchestration start: %w", err) + } + signal := signalEvent{From: "sender", Message: "approved"} + if _, err := client.ScheduleNewOrchestration( + ctx, + "CrossInstanceSenderOrch", + api.WithInstanceID(senderID), + api.WithInput(forwardInput{Target: receiverID, Signal: signal}), + ); err != nil { + return fmt.Errorf("failed to schedule sender orchestration: %w", err) + } + sender, err := client.WaitForOrchestrationCompletion(ctx, senderID, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for sender orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(sender); err != nil { + return err + } + var senderOutput string + if err := sender.ReadOutput(&senderOutput); err != nil { + return fmt.Errorf("failed to decode sender output: %w", err) + } + if senderOutput != "sent" { + return fmt.Errorf("sender output = %q, want sent", senderOutput) + } + receiver, err := client.WaitForOrchestrationCompletion(ctx, receiverID, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for receiver orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(receiver); err != nil { + return err + } + var receiverOutput signalEvent + if err := receiver.ReadOutput(&receiverOutput); err != nil { + return fmt.Errorf("failed to decode receiver output: %w", err) + } + if receiverOutput != signal { + return fmt.Errorf("receiver output = %#v, want %#v", receiverOutput, signal) + } + return nil +} + +func runExpectedTimeoutScenario( + ctx context.Context, + client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) + }, + ownedIDs *[]api.InstanceID, +) error { + id := dtssample.NewInstanceID("externalevents-timeout") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + "ExternalEventTimeoutOrchestrator", + api.WithInstanceID(id), + ); err != nil { + return fmt.Errorf("failed to schedule timeout orchestration: %w", err) + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for timeout orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode timeout output: %w", err) + } + if output != "timeout-observed" { + return fmt.Errorf("timeout output = %q, want timeout-observed", output) } return nil } @@ -93,14 +294,56 @@ func readAndRaiseEvent( } } -// ExternalEventOrchestrator is an orchestrator function that blocks for 30 seconds or -// until a "Name" event is sent to it. +// ExternalEventOrchestrator blocks for 30 seconds or until a Name event arrives. func ExternalEventOrchestrator(ctx *task.OrchestrationContext) (any, error) { var nameInput string - if err := ctx.WaitForSingleEvent("Name", 30*time.Second).Await(&nameInput); err != nil { - // Timeout expired + if err := ctx.WaitForSingleEvent("Name", nameEventTimeout).Await(&nameInput); err != nil { return nil, err } - return fmt.Sprintf("Hello, %s!", nameInput), nil } + +func RepeatedEventChannelOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var count int + if err := ctx.GetInput(&count); err != nil { + return nil, err + } + channel := task.NewEventChannel[checkpointEvent](ctx, "Checkpoint") + results := make([]string, 0, count) + for range count { + event, err := channel.ReceiveErr(ctx) + if err != nil { + return nil, err + } + results = append(results, fmt.Sprintf("%02d:%s", event.Index, event.Label)) + } + return results, nil +} + +func CrossInstanceReceiverOrchestrator(ctx *task.OrchestrationContext) (any, error) { + return task.NewEventChannel[signalEvent](ctx, "Forwarded").Receive(ctx), nil +} + +func CrossInstanceSenderOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input forwardInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.SendEvent(input.Target, "Forwarded", input.Signal); err != nil { + return nil, err + } + if err := ctx.CreateTimer(100 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + return "sent", nil +} + +func ExternalEventTimeoutOrchestrator(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.WaitForSingleEvent("Never", time.Second).Await(nil); !errors.Is(err, task.ErrTaskCanceled) { + if err == nil { + return nil, errors.New("timeout event completed without timing out") + } + return nil, err + } + return "timeout-observed", nil +} diff --git a/samples/externalevents/externalevents_test.go b/samples/externalevents/externalevents_test.go index a6bfe893..aca120a0 100644 --- a/samples/externalevents/externalevents_test.go +++ b/samples/externalevents/externalevents_test.go @@ -7,8 +7,18 @@ import ( "strings" "testing" "testing/iotest" + "time" ) +func TestTimeoutSemantics(t *testing.T) { + if sampleHostTimeout != 5*time.Minute { + t.Fatalf("sampleHostTimeout = %s, want 5m", sampleHostTimeout) + } + if nameEventTimeout != 30*time.Second { + t.Fatalf("nameEventTimeout = %s, want 30s", nameEventTimeout) + } +} + func TestReadAndRaiseEvent(t *testing.T) { readError := errors.New("input unavailable") raiseError := errors.New("event rejected") diff --git a/samples/internal/dtssample/validation.go b/samples/internal/dtssample/validation.go new file mode 100644 index 00000000..c06fc5f2 --- /dev/null +++ b/samples/internal/dtssample/validation.go @@ -0,0 +1,60 @@ +package dtssample + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" +) + +// NewInstanceID gives each sample run its own orchestration namespace. +func NewInstanceID(sample string) api.InstanceID { + return api.InstanceID("sample-" + sample + "-" + uuid.NewString()) +} + +// RequireCompleted rejects unsuccessful terminal states as well as missing results. +func RequireCompleted(metadata *api.OrchestrationMetadata) error { + if metadata == nil { + return errors.New("sample received no orchestration metadata") + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_COMPLETED { + return fmt.Errorf("orchestration %s ended in %s: %v", + metadata.InstanceID, metadata.RuntimeStatus, metadata.FailureDetails) + } + return nil +} + +// Cleanup terminates and purges only the supplied, sample-owned instance IDs and +// their children. Call it before shutting down the worker. Entity state, schedules, +// export jobs, and storage objects need their own sample-specific cleanup. +func Cleanup(client *durabletaskscheduler.Client, instanceIDs ...api.InstanceID) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + var failures []error + for _, id := range instanceIDs { + metadata, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(false)) + if errors.Is(err, api.ErrInstanceNotFound) { + continue + } + if err == nil && metadata == nil { + err = errors.New("missing orchestration metadata") + } + if err == nil && !metadata.IsComplete() { + err = client.TerminateOrchestration(ctx, id) + if err == nil { + _, err = client.WaitForOrchestrationCompletion(ctx, id) + } + } + if err == nil { + err = client.PurgeOrchestrationState(ctx, id, api.WithRecursivePurge(true)) + } + if err != nil && !errors.Is(err, api.ErrInstanceNotFound) { + failures = append(failures, fmt.Errorf("clean up %s: %w", id, err)) + } + } + return errors.Join(failures...) +} diff --git a/samples/internal/dtssample/validation_test.go b/samples/internal/dtssample/validation_test.go new file mode 100644 index 00000000..bb7a2f2f --- /dev/null +++ b/samples/internal/dtssample/validation_test.go @@ -0,0 +1,33 @@ +package dtssample + +import ( + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" +) + +func TestRequireCompleted(t *testing.T) { + if RequireCompleted(nil) == nil { + t.Fatal("missing metadata must fail") + } + for _, status := range []api.OrchestrationStatus{ + api.RUNTIME_STATUS_PENDING, api.RUNTIME_STATUS_RUNNING, + api.RUNTIME_STATUS_SUSPENDED, api.RUNTIME_STATUS_FAILED, + api.RUNTIME_STATUS_CANCELED, api.RUNTIME_STATUS_TERMINATED, + api.RUNTIME_STATUS_CONTINUED_AS_NEW, api.RUNTIME_STATUS_COMPLETED, + } { + err := RequireCompleted(&api.OrchestrationMetadata{InstanceID: "sample", RuntimeStatus: status}) + if (err == nil) != (status == api.RUNTIME_STATUS_COMPLETED) { + t.Errorf("status %s: unexpected error %v", status, err) + } + } +} + +func TestNewInstanceID(t *testing.T) { + first := NewInstanceID("example") + second := NewInstanceID("example") + if !strings.HasPrefix(string(first), "sample-example-") || first == second { + t.Fatalf("invalid sample IDs: %q, %q", first, second) + } +} diff --git a/samples/parallel/README.md b/samples/parallel/README.md new file mode 100644 index 00000000..e577a0e6 --- /dev/null +++ b/samples/parallel/README.md @@ -0,0 +1,45 @@ +# Parallel sample + +## Features + +- Uses durable timers with `WhenAny` so the winning branch is deterministic. +- Fans out a controlled set of activity calls. +- Uses `WhenAll` to wait for all activity results. +- Aggregates successful updates and one expected business failure without + random outcomes or host I/O in the orchestrator. +- Cleans up only the orchestration instance created by this run. + +## Prerequisites + +Start a Durable Task Scheduler endpoint or emulator and set: + +```sh +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```sh +go run ./samples/parallel +``` + +## Expected result + +The `fast-timer` branch wins the durable race. Four device update activities run +in parallel; three are updated and `door-lock-02` returns the expected +`blocked-by-policy` business result. On success it prints: + +```text +SAMPLE_OK parallel +``` + +## Cleanup + +The sample terminates if needed and recursively purges only its generated +`sample-parallel-*` orchestration instance before shutting down. + +## Noninteractive command + +```sh +DTS_CONNECTION_STRING="$DTS_CONNECTION_STRING" go run ./samples/parallel +``` diff --git a/samples/parallel/parallel.go b/samples/parallel/parallel.go index 86837ad7..b5a90004 100644 --- a/samples/parallel/parallel.go +++ b/samples/parallel/parallel.go @@ -1,5 +1,5 @@ -// Command parallel demonstrates fan-out/fan-in: an orchestration starts a -// dynamic number of activities at once and then awaits all of them. +// Command parallel demonstrates deterministic fan-out/fan-in with WhenAll and +// a timer-controlled race with WhenAny. // // export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" // go run ./samples/parallel @@ -7,133 +7,172 @@ package main import ( "context" - "encoding/json" + "errors" "fmt" "log" - "math/rand" + "reflect" "time" - "github.com/google/uuid" - + "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +type parallelInput struct { + Devices []deviceUpdate `json:"devices"` +} + +type deviceUpdate struct { + DeviceID string `json:"deviceId"` + TargetVersion int `json:"targetVersion"` + ExpectedToFail bool `json:"expectedToFail,omitempty"` +} + +type deviceResult struct { + DeviceID string `json:"deviceId"` + AppliedVersion int `json:"appliedVersion"` + Status string `json:"status"` +} + +type parallelSummary struct { + RaceWinner string `json:"raceWinner"` + Total int `json:"total"` + Updated []string `json:"updated"` + Failed []string `json:"failed"` + Results []deviceResult `json:"results"` +} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK parallel") } -func run() error { - // Create a new task registry and add the orchestrator and activities - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN("UpdateDevicesOrchestrator", UpdateDevicesOrchestrator); err != nil { +func run() (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("UpdateDevicesOrchestrator", UpdateDevicesOrchestrator); err != nil { return fmt.Errorf("failed to register orchestrator: %w", err) } - if err := r.AddActivityN("GetDevicesToUpdate", GetDevicesToUpdate); err != nil { - return fmt.Errorf("failed to register activity: %w", err) - } - if err := r.AddActivityN("UpdateDevice", UpdateDevice); err != nil { + if err := registry.AddActivityN("UpdateDevice", UpdateDevice); err != nil { return fmt.Errorf("failed to register activity: %w", err) } - // Connect a client and worker to the Durable Task Scheduler task hub ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() - app, err := dtssample.Start(ctx, r) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } - defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) - } - }() - - // Start a new orchestration - id, err := app.Client.ScheduleNewOrchestration(ctx, "UpdateDevicesOrchestrator") - if err != nil { - return fmt.Errorf("failed to schedule new orchestration: %w", err) + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, app.Shutdown()) }() + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...)) }() + + id := dtssample.NewInstanceID("parallel") + ownedIDs = append(ownedIDs, id) + input := parallelInput{Devices: []deviceUpdate{ + {DeviceID: "thermostat-01", TargetVersion: 7}, + {DeviceID: "door-lock-02", TargetVersion: 7, ExpectedToFail: true}, + {DeviceID: "light-03", TargetVersion: 7}, + {DeviceID: "sensor-04", TargetVersion: 7}, + }} + if _, err := app.Client.ScheduleNewOrchestration( + ctx, + "UpdateDevicesOrchestrator", + api.WithInstanceID(id), + api.WithInput(input), + ); err != nil { + return fmt.Errorf("failed to schedule parallel orchestration: %w", err) } - // Wait for the orchestration to complete - metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("failed to wait for orchestration to complete: %w", err) + return fmt.Errorf("failed to wait for parallel orchestration: %w", err) } - - // Print the results - metadataEnc, err := json.MarshalIndent(metadata, "", " ") - if err != nil { - return fmt.Errorf("failed to encode result to JSON: %w", err) + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output parallelSummary + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode parallel output: %w", err) + } + expected := parallelSummary{ + RaceWinner: "fast-timer", + Total: 4, + Updated: []string{"thermostat-01", "light-03", "sensor-04"}, + Failed: []string{"door-lock-02"}, + Results: []deviceResult{ + {DeviceID: "thermostat-01", AppliedVersion: 7, Status: "updated"}, + {DeviceID: "door-lock-02", AppliedVersion: 0, Status: "blocked-by-policy"}, + {DeviceID: "light-03", AppliedVersion: 7, Status: "updated"}, + {DeviceID: "sensor-04", AppliedVersion: 7, Status: "updated"}, + }, + } + if !reflect.DeepEqual(output, expected) { + return fmt.Errorf("parallel output = %#v, want %#v", output, expected) } - log.Printf("Orchestration completed: %v", string(metadataEnc)) return nil } -// UpdateDevicesOrchestrator is an orchestrator that runs activities in parallel +// UpdateDevicesOrchestrator runs all device updates together, waits for every +// activity with WhenAll, and uses durable timers to make the WhenAny winner +// independent from worker wall-clock scheduling. func UpdateDevicesOrchestrator(ctx *task.OrchestrationContext) (any, error) { - // Get a dynamic list of devices to perform updates on - var devices []string - if err := ctx.CallActivity("GetDevicesToUpdate").Await(&devices); err != nil { + var input parallelInput + if err := ctx.GetInput(&input); err != nil { return nil, err } - // Start a dynamic number of tasks in parallel, not waiting for any to complete (yet) - tasks := make([]task.Task, len(devices)) - for i, id := range devices { - tasks[i] = ctx.CallActivity("UpdateDevice", task.WithActivityInput(id)) + fastTimer := ctx.CreateTimer(time.Second) + slowTimerCtx, cancelSlowTimer := ctx.WithCancel() + slowTimer := slowTimerCtx.CreateTimer(3 * time.Second) + raceWinner := "slow-timer" + if ctx.WhenAny(fastTimer, slowTimer) == fastTimer { + raceWinner = "fast-timer" + cancelSlowTimer() } - // Now that all are started, wait for them to complete and then return the success rate - successCount := 0 - for _, task := range tasks { - var succeeded bool - if err := task.Await(&succeeded); err == nil && succeeded { - successCount++ - } + tasks := make([]task.Task, len(input.Devices)) + for i, device := range input.Devices { + tasks[i] = ctx.CallActivity("UpdateDevice", task.WithActivityInput(device)) + } + if err := ctx.WhenAll(tasks...); err != nil { + return nil, err } - return float32(successCount) / float32(len(devices)), nil -} - -// GetDevicesToUpdate is an activity that returns a list of random device IDs to an orchestration. -func GetDevicesToUpdate(task.ActivityContext) (any, error) { - // Return a fake list of device IDs - const deviceCount = 10 - deviceIDs := make([]string, deviceCount) - for i := 0; i < deviceCount; i++ { - u, err := uuid.NewV7() - if err != nil { - deviceIDs[i] = uuid.NewString() - continue + summary := parallelSummary{ + RaceWinner: raceWinner, + Total: len(input.Devices), + Results: make([]deviceResult, len(tasks)), + } + for i, task := range tasks { + var result deviceResult + if err := task.Await(&result); err != nil { + return nil, err + } + summary.Results[i] = result + if result.Status == "updated" { + summary.Updated = append(summary.Updated, result.DeviceID) + } else { + summary.Failed = append(summary.Failed, result.DeviceID) } - deviceIDs[i] = u.String() } - return deviceIDs, nil + return summary, nil } -// UpdateDevice is an activity that takes a device ID (string) and pretends to perform an update -// on the corresponding device, with a random 67% success rate. +// UpdateDevice is deterministic: the expected failure is part of the activity +// input rather than a random or host-clock result. func UpdateDevice(ctx task.ActivityContext) (any, error) { - var deviceID string - if err := ctx.GetInput(&deviceID); err != nil { + var input deviceUpdate + if err := ctx.GetInput(&input); err != nil { return nil, err } - log.Printf("updating device: %s", deviceID) - - // Delay and success results are randomly generated - delay := time.Duration(rand.Int31n(500)) * time.Millisecond - select { - case <-ctx.Context().Done(): - return nil, ctx.Context().Err() - case <-time.After(delay): - // All good, continue + if input.ExpectedToFail { + return deviceResult{DeviceID: input.DeviceID, Status: "blocked-by-policy"}, nil } - - // Simulate random failures - success := rand.Intn(3) > 0 - - return success, nil + return deviceResult{ + DeviceID: input.DeviceID, + AppliedVersion: input.TargetVersion, + Status: "updated", + }, nil } diff --git a/samples/timers/README.md b/samples/timers/README.md new file mode 100644 index 00000000..0c0bd397 --- /dev/null +++ b/samples/timers/README.md @@ -0,0 +1,46 @@ +# Timers sample + +## Features + +- Schedules an orchestration to start in the future. +- Uses one logical durable timer with a five-second deadline. +- Configures a two-second maximum physical timer interval so the long timer is + split into multiple history records. +- Demonstrates replay-stable `CurrentTimeUtc` and `NewGuid` values by recording + them through an activity and validating the final payload. +- Reads orchestration history to verify the physical timer split. + +## Prerequisites + +Start a Durable Task Scheduler endpoint or emulator and set: + +```sh +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```sh +go run ./samples/timers +``` + +## Expected result + +The orchestration starts at the scheduled time, waits for one five-second +logical timer, and history contains three created/fired physical timer chunks +ending at the single logical deadline. On success it prints: + +```text +SAMPLE_OK timers +``` + +## Cleanup + +The sample terminates if needed and recursively purges only its generated +`sample-timers-*` orchestration instance before shutting down. + +## Noninteractive command + +```sh +DTS_CONNECTION_STRING="$DTS_CONNECTION_STRING" go run ./samples/timers +``` diff --git a/samples/timers/timers.go b/samples/timers/timers.go new file mode 100644 index 00000000..b1a4f728 --- /dev/null +++ b/samples/timers/timers.go @@ -0,0 +1,182 @@ +// Command timers demonstrates durable timers, scheduled start, long-timer +// splitting, replay-stable orchestration time, and deterministic GUIDs. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/timers +package main + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + maximumPhysicalTimerInterval = 2 * time.Second + logicalTimerDelay = 5 * time.Second + expectedPhysicalTimers = 3 +) + +type timerInput struct { + Delay time.Duration `json:"delay"` +} + +type stableObservation struct { + StartedAt time.Time `json:"startedAt"` + Deadline time.Time `json:"deadline"` + GUID string `json:"guid"` +} + +type timerOutput struct { + StartedAt time.Time `json:"startedAt"` + Deadline time.Time `json:"deadline"` + FiredAt time.Time `json:"firedAt"` + FirstGUID string `json:"firstGuid"` + SecondGUID string `json:"secondGuid"` + Recorded stableObservation `json:"recorded"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK timers") +} + +func run() (err error) { + options, err := dtssample.Options() + if err != nil { + return err + } + options.MaximumTimerInterval = maximumPhysicalTimerInterval + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("TimerOrchestrator", TimerOrchestrator); err != nil { + return fmt.Errorf("failed to register orchestrator: %w", err) + } + if err := registry.AddActivityN("RecordStableObservation", RecordStableObservation); err != nil { + return fmt.Errorf("failed to register activity: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + app, err := dtssample.StartWithOptions(ctx, options, registry) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, app.Shutdown()) }() + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...)) }() + + scheduledStart := time.Now().UTC().Add(2 * time.Second) + id := dtssample.NewInstanceID("timers") + ownedIDs = append(ownedIDs, id) + if _, err := app.Client.ScheduleNewOrchestration( + ctx, + "TimerOrchestrator", + api.WithInstanceID(id), + api.WithInput(timerInput{Delay: logicalTimerDelay}), + api.WithStartTime(scheduledStart), + ); err != nil { + return fmt.Errorf("failed to schedule timer orchestration: %w", err) + } + + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return fmt.Errorf("failed to wait for timer orchestration completion: %w", err) + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output timerOutput + if err := metadata.ReadOutput(&output); err != nil { + return fmt.Errorf("failed to decode timer output: %w", err) + } + if metadata.ScheduledStartAt.IsZero() { + return errors.New("metadata did not include the scheduled start time") + } + if scheduledStart.Sub(metadata.ScheduledStartAt).Abs() > time.Second { + return fmt.Errorf("scheduled start = %s, want near %s", metadata.ScheduledStartAt, scheduledStart) + } + if output.StartedAt.Before(metadata.ScheduledStartAt.Add(-time.Second)) { + return fmt.Errorf("orchestration started at %s before scheduled start %s", output.StartedAt, metadata.ScheduledStartAt) + } + if !output.Deadline.Equal(output.StartedAt.Add(logicalTimerDelay)) { + return fmt.Errorf("deadline = %s, want %s", output.Deadline, output.StartedAt.Add(logicalTimerDelay)) + } + if output.FiredAt.Before(output.Deadline) { + return fmt.Errorf("timer fired at %s before logical deadline %s", output.FiredAt, output.Deadline) + } + if output.FirstGUID == "" || output.SecondGUID == "" || output.FirstGUID == output.SecondGUID { + return fmt.Errorf("unexpected deterministic GUIDs: first=%q second=%q", output.FirstGUID, output.SecondGUID) + } + if output.Recorded != (stableObservation{StartedAt: output.StartedAt, Deadline: output.Deadline, GUID: output.FirstGUID}) { + return fmt.Errorf("recorded stable values = %#v, want orchestration values", output.Recorded) + } + + history, err := app.Client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + if err != nil { + return fmt.Errorf("failed to read timer orchestration history: %w", err) + } + timerCreated, timerFired := 0, 0 + var lastTimerFireAt time.Time + for _, event := range history.Events { + switch event.Type { + case api.HistoryEventTimerCreated: + timerCreated++ + case api.HistoryEventTimerFired: + timerFired++ + lastTimerFireAt = event.TimerFired.FireAt + } + } + if timerCreated != expectedPhysicalTimers || timerFired != expectedPhysicalTimers { + return fmt.Errorf("physical timers created/fired = %d/%d, want %d/%d", + timerCreated, timerFired, expectedPhysicalTimers, expectedPhysicalTimers) + } + if !lastTimerFireAt.Equal(output.Deadline) { + return fmt.Errorf("last physical timer fire = %s, want logical deadline %s", lastTimerFireAt, output.Deadline) + } + return nil +} + +func TimerOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input timerInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + startedAt := ctx.CurrentTimeUtc + firstGUID := ctx.NewGuid() + deadline := startedAt.Add(input.Delay) + if err := ctx.CreateTimer(input.Delay).Await(nil); err != nil { + return nil, err + } + firedAt := ctx.CurrentTimeUtc + secondGUID := ctx.NewGuid() + observation := stableObservation{StartedAt: startedAt, Deadline: deadline, GUID: firstGUID} + var recorded stableObservation + if err := ctx.CallActivity("RecordStableObservation", task.WithActivityInput(observation)).Await(&recorded); err != nil { + return nil, err + } + return timerOutput{ + StartedAt: startedAt, + Deadline: deadline, + FiredAt: firedAt, + FirstGUID: firstGUID, + SecondGUID: secondGUID, + Recorded: recorded, + }, nil +} + +func RecordStableObservation(ctx task.ActivityContext) (any, error) { + var observation stableObservation + if err := ctx.GetInput(&observation); err != nil { + return nil, err + } + return observation, nil +} From f2eafbe13f6472a8361d185fd00e9706ada1c1d8 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:43:03 -0700 Subject: [PATCH 07/26] cover stateful workflow and worker feature families Add executable scenarios for child workflows, continuation and turn budgets, lifecycle management, schedules, versions, entities, conversion, history, observability, and worker drain/restart. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/continueasnew/README.md | 48 ++ samples/continueasnew/continueasnew.go | 421 +++++++++++++ samples/dataconverter/README.md | 26 + samples/dataconverter/dataconverter.go | 296 ++++++++++ samples/entity/README.md | 32 + samples/entity/entity.go | 551 ++++++++++++++---- samples/history/README.md | 27 + samples/history/history.go | 267 +++++++++ samples/management/README.md | 45 ++ samples/management/management.go | 525 +++++++++++++++++ samples/observability/README.md | 28 + samples/observability/observability.go | 359 ++++++++++++ samples/retries/README.md | 26 + samples/retries/retries.go | 320 ++++++++-- samples/scheduledtasks/README.md | 44 ++ samples/scheduledtasks/scheduledtasks.go | 345 +++++++++++ samples/suborchestrations/README.md | 42 ++ .../suborchestrations/suborchestrations.go | 293 ++++++++++ samples/versioning/README.md | 42 ++ samples/versioning/versioning.go | 532 +++++++++++++++++ samples/worker/README.md | 33 ++ samples/worker/worker.go | 493 ++++++++++++++++ 22 files changed, 4627 insertions(+), 168 deletions(-) create mode 100644 samples/continueasnew/README.md create mode 100644 samples/continueasnew/continueasnew.go create mode 100644 samples/dataconverter/README.md create mode 100644 samples/dataconverter/dataconverter.go create mode 100644 samples/entity/README.md create mode 100644 samples/history/README.md create mode 100644 samples/history/history.go create mode 100644 samples/management/README.md create mode 100644 samples/management/management.go create mode 100644 samples/observability/README.md create mode 100644 samples/observability/observability.go create mode 100644 samples/retries/README.md create mode 100644 samples/scheduledtasks/README.md create mode 100644 samples/scheduledtasks/scheduledtasks.go create mode 100644 samples/suborchestrations/README.md create mode 100644 samples/suborchestrations/suborchestrations.go create mode 100644 samples/versioning/README.md create mode 100644 samples/versioning/versioning.go create mode 100644 samples/worker/README.md create mode 100644 samples/worker/worker.go diff --git a/samples/continueasnew/README.md b/samples/continueasnew/README.md new file mode 100644 index 00000000..0aa1e9be --- /dev/null +++ b/samples/continueasnew/README.md @@ -0,0 +1,48 @@ +# Continue-as-new sample + +This sample demonstrates three bounded ContinueAsNew scenarios: + +- `checkpoint`: explicit checkpointed generations with external-event carryover +- `history-limit`: `MaxHistoryEvents` and `OnHistoryLimitExceeded` compacting a running orchestration without dropping carried business events +- `event-budget`: `MaxEventsPerTurn` consuming a queued batch across bounded service turns without losing or duplicating work + +## Prerequisites + +Set `DTS_CONNECTION_STRING` to a task hub you can use for samples: + +```bash +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +Run every scenario: + +```bash +go run ./samples/continueasnew +``` + +Run one or more named scenarios: + +```bash +go run ./samples/continueasnew checkpoint +go run ./samples/continueasnew history-limit event-budget +``` + +## Expected result + +Each scenario starts real orchestrations, raises real external events, validates fetched outputs/history, and cleans up its own instance IDs. The program prints: + +```text +SAMPLE_OK continueasnew +``` + +only after selected scenarios validate and cleanup succeeds. + +The event-budget scenario queues its input before starting the worker and checks +the public history metrics: every turn processes at most one work-item event, +and all queued business items must still be returned. + +## Cleanup + +Every scenario creates a unique owned instance ID with `dtssample.NewInstanceID`, records it before scheduling, calls `dtssample.Cleanup` before worker shutdown, and does not purge by broad filters. diff --git a/samples/continueasnew/continueasnew.go b/samples/continueasnew/continueasnew.go new file mode 100644 index 00000000..c5024a1f --- /dev/null +++ b/samples/continueasnew/continueasnew.go @@ -0,0 +1,421 @@ +// Command continueasnew demonstrates bounded ContinueAsNew patterns with +// explicit checkpoints, MaxHistoryEvents compaction, and MaxEventsPerTurn. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "slices" + "strings" + "sync" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + carryoverName = "SampleContinueCarryover" + compactionName = "SampleContinueHistoryCompaction" + eventBudgetName = "SampleContinueEventBudget" +) + +type continueCarryState struct { + Want int `json:"want"` + Items []string `json:"items"` + Generation int `json:"generation"` +} + +type continueCompactionState struct { + WantItems int `json:"wantItems"` + Items []string `json:"items"` + Compactions int `json:"compactions"` + LastHistoryLength int `json:"lastHistoryLength"` + LastUnprocessedEventCnt int `json:"lastUnprocessedEventCount"` +} + +type continueBudgetInput struct { + Count int `json:"count"` +} + +type continueBudgetOutput struct { + Items []int `json:"items"` +} + +func main() { + if err := run(os.Args[1:]); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK continueasnew") +} + +func run(args []string) error { + scenarios := []struct { + name string + run func(context.Context) error + }{ + {"checkpoint", runCheckpointScenario}, + {"history-limit", runHistoryLimitScenario}, + {"event-budget", runEventBudgetScenario}, + } + selected := map[string]bool{} + if len(args) == 0 { + selected["all"] = true + } else { + for _, arg := range args { + name := strings.ToLower(strings.TrimSpace(arg)) + switch name { + case "all", "checkpoint", "history-limit", "event-budget": + selected[name] = true + default: + return fmt.Errorf("unknown scenario %q; use all, checkpoint, history-limit, or event-budget", arg) + } + } + } + for _, scenario := range scenarios { + if !selected["all"] && !selected[scenario.name] { + continue + } + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + err := scenario.run(ctx) + cancel() + if err != nil { + return fmt.Errorf("%s scenario: %w", scenario.name, err) + } + fmt.Printf("verified %s\n", scenario.name) + } + return nil +} + +func runCheckpointScenario(ctx context.Context) (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(carryoverName, carryoverOrchestrator); err != nil { + return err + } + app, err := startContinueApp(ctx, registry) + if err != nil { + return err + } + instanceID := dtssample.NewInstanceID("continueasnew-checkpoint") + ownedIDs := []api.InstanceID{instanceID} + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) }() + + if _, err := app.Client.ScheduleNewOrchestration( + ctx, + carryoverName, + api.WithInstanceID(instanceID), + api.WithInput(continueCarryState{Want: 3}), + api.WithTags(map[string]string{"sample": "continueasnew", "scenario": "checkpoint"}), + ); err != nil { + return err + } + if _, err := app.Client.WaitForOrchestrationStart(ctx, instanceID); err != nil { + return err + } + if err := app.Client.SuspendOrchestration(ctx, instanceID, "queue events before checkpoint"); err != nil { + return err + } + for _, item := range []string{"one", "two", "three"} { + if err := app.Client.RaiseEvent(ctx, instanceID, "item", api.WithEventPayload(item)); err != nil { + return err + } + } + if err := app.Client.ResumeOrchestration(ctx, instanceID, "drain carried events"); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output continueCarryState + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.Generation != 3 || !sameStringSet(output.Items, []string{"one", "two", "three"}) { + return fmt.Errorf("checkpoint output = %#v", output) + } + if metadata.ExecutionID == "" { + return errors.New("checkpoint completion did not include the latest execution ID") + } + return nil +} + +func runHistoryLimitScenario(ctx context.Context) (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(compactionName, compactionOrchestrator); err != nil { + return err + } + app, err := startContinueApp(ctx, registry, task.WithOrchestrationOptions(task.OrchestrationOptions{ + MaxHistoryEvents: 8, + OnHistoryLimitExceeded: func(info task.HistoryLimitInfo) (any, error) { + var state continueCompactionState + if err := info.GetInput(&state); err != nil { + return nil, err + } + state.Compactions++ + state.LastHistoryLength = info.HistoryLength + state.LastUnprocessedEventCnt = info.UnprocessedEventCount + return state, nil + }, + })) + if err != nil { + return err + } + instanceID := dtssample.NewInstanceID("continueasnew-history") + ownedIDs := []api.InstanceID{instanceID} + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) }() + + if _, err := app.Client.ScheduleNewOrchestration( + ctx, + compactionName, + api.WithInstanceID(instanceID), + api.WithInput(continueCompactionState{WantItems: 3}), + api.WithTags(map[string]string{"sample": "continueasnew", "scenario": "history-limit"}), + ); err != nil { + return err + } + started, err := app.Client.WaitForOrchestrationStart(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := app.Client.SuspendOrchestration(ctx, instanceID, "build bounded carryover set"); err != nil { + return err + } + for _, item := range []string{"red", "green", "blue"} { + if err := app.Client.RaiseEvent(ctx, instanceID, "work", api.WithEventPayload(item)); err != nil { + return err + } + } + if err := app.Client.ResumeOrchestration(ctx, instanceID, "allow history-limit handler"); err != nil { + return err + } + if err := waitForExecutionChange(ctx, app.Client, instanceID, started.ExecutionID); err != nil { + return err + } + if err := app.Client.RaiseEvent(ctx, instanceID, "release", api.WithEventPayload(3)); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output continueCompactionState + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.Compactions == 0 || output.LastHistoryLength <= output.LastUnprocessedEventCnt { + return fmt.Errorf("history-limit handler did not report a real compaction: %#v", output) + } + if !sameStringSet(output.Items, []string{"red", "green", "blue"}) { + return fmt.Errorf("compacted business items were lost or duplicated: %#v", output.Items) + } + history, err := app.Client.GetOrchestrationHistory(ctx, instanceID, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + if err != nil { + return err + } + if len(history.Events) > 16 { + return fmt.Errorf("latest execution history has %d events after compaction; expected a bounded current generation", len(history.Events)) + } + return nil +} + +func runEventBudgetScenario(ctx context.Context) (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(eventBudgetName, eventBudgetOrchestrator); err != nil { + return err + } + options, err := dtssample.Options() + if err != nil { + return err + } + client, err := durabletaskscheduler.NewClient(ctx, options, api.DefaultLogger()) + if err != nil { + return err + } + defer func() { err = errors.Join(err, client.Close()) }() + instanceID := dtssample.NewInstanceID("continueasnew-event-budget") + var mu sync.Mutex + var processed []int + worker, err := durabletaskscheduler.NewWorker(options, registry, api.DefaultLogger(), + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithTaskExecutorOptions( + task.WithOrchestrationOptions(task.OrchestrationOptions{MaxEventsPerTurn: 1}), + task.WithMetricsHooks(task.MetricsHooks{History: func(metric task.HistoryMetric) { + if metric.InstanceID == instanceID { + mu.Lock() + processed = append(processed, metric.ProcessedEvents) + mu.Unlock() + } + }}), + ), + ) + if err != nil { + return err + } + defer func() { + stopCtx, stop := context.WithTimeout(context.Background(), 30*time.Second) + defer stop() + err = errors.Join(err, worker.Shutdown(stopCtx)) + }() + defer func() { err = errors.Join(err, dtssample.Cleanup(client, instanceID)) }() + + if _, err := client.ScheduleNewOrchestration( + ctx, + eventBudgetName, + api.WithInstanceID(instanceID), + api.WithInput(continueBudgetInput{Count: 5}), + api.WithTags(map[string]string{"sample": "continueasnew", "scenario": "event-budget"}), + ); err != nil { + return err + } + // Queue the input before starting the worker, rather than relying on live arrival timing. + for index := range 5 { + if err := client.RaiseEvent(ctx, instanceID, "item", api.WithEventPayload(index)); err != nil { + return err + } + } + if err := worker.Start(ctx); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output continueBudgetOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + slices.Sort(output.Items) + if !slices.Equal(output.Items, []int{0, 1, 2, 3, 4}) { + return fmt.Errorf("MaxEventsPerTurn output = %#v", output.Items) + } + mu.Lock() + counts := slices.Clone(processed) + mu.Unlock() + total := 0 + for _, count := range counts { + if count > 1 { + return fmt.Errorf("turn exceeded MaxEventsPerTurn=1: %v", counts) + } + total += count + } + if total < 5 { + return fmt.Errorf("missing turn-level evidence for the five inputs: %v", counts) + } + fmt.Printf("verified per-turn event counts %v\n", counts) + return nil +} + +func startContinueApp( + ctx context.Context, + registry *task.TaskRegistry, + executorOptions ...task.TaskExecutorOption, +) (*dtssample.App, error) { + var workerOptions []durabletaskclient.TaskHubGrpcWorkerOption + if len(executorOptions) > 0 { + workerOptions = append(workerOptions, durabletaskclient.WithTaskExecutorOptions(executorOptions...)) + } + return dtssample.Start(ctx, registry, workerOptions...) +} + +func carryoverOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var state continueCarryState + if err := ctx.GetInput(&state); err != nil { + return nil, err + } + for len(state.Items) < state.Want { + var item string + if err := ctx.WaitForSingleEvent("item", time.Minute).Await(&item); err != nil { + return nil, err + } + state.Items = append(state.Items, item) + state.Generation++ + if len(state.Items) < state.Want { + ctx.ContinueAsNew(state, task.WithKeepUnprocessedEvents()) + return nil, nil + } + } + return state, nil +} + +func compactionOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var state continueCompactionState + if err := ctx.GetInput(&state); err != nil { + return nil, err + } + var count int + if err := ctx.WaitForSingleEvent("release", time.Minute).Await(&count); err != nil { + return nil, err + } + for len(state.Items) < count { + var item string + if err := ctx.WaitForSingleEvent("work", time.Minute).Await(&item); err != nil { + return nil, err + } + state.Items = append(state.Items, item) + } + return state, nil +} + +func eventBudgetOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input continueBudgetInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + output := continueBudgetOutput{Items: make([]int, 0, input.Count)} + for len(output.Items) < input.Count { + var item int + if err := ctx.WaitForSingleEvent("item", time.Minute).Await(&item); err != nil { + return nil, err + } + output.Items = append(output.Items, item) + } + return output, nil +} + +func waitForExecutionChange( + ctx context.Context, + client *durabletaskscheduler.Client, + instanceID api.InstanceID, + originalExecutionID string, +) error { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + current, err := client.FetchOrchestrationMetadata(ctx, instanceID, api.WithFetchPayloads(true)) + if err == nil && current.ExecutionID != "" && current.ExecutionID != originalExecutionID { + return nil + } + if err != nil && !errors.Is(err, api.ErrInstanceNotFound) { + return err + } + select { + case <-ctx.Done(): + return fmt.Errorf("MaxHistoryEvents did not compact to a new execution before release was sent: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func sameStringSet(left, right []string) bool { + leftCopy := append([]string(nil), left...) + rightCopy := append([]string(nil), right...) + slices.Sort(leftCopy) + slices.Sort(rightCopy) + return slices.Equal(leftCopy, rightCopy) +} diff --git a/samples/dataconverter/README.md b/samples/dataconverter/README.md new file mode 100644 index 00000000..23b851fd --- /dev/null +++ b/samples/dataconverter/README.md @@ -0,0 +1,26 @@ +# Custom data converter + +This sample configures a small `api.DataConverter` on both the DTS management +client and worker. The converter writes `sample-dc:` plus base64-encoded JSON so +the custom encoding is visible in metadata and history without adding another +serialization dependency. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/dataconverter +``` + +## What it proves + +- Typed orchestration input/output, activity input/output, external event data, + custom status, and entity state all round-trip through the configured + converter. +- A raw-input workflow intentionally bypasses converter serialization and + verifies that the raw metadata is unchanged. +- The converter can still read legacy/plain JSON payloads for compatibility. + +`SAMPLE_OK dataconverter` is printed only after decoded values, raw encodings, +and entity cleanup are verified. The sample deletes only its exact entity key +and purges only its declared orchestration IDs. diff --git a/samples/dataconverter/dataconverter.go b/samples/dataconverter/dataconverter.go new file mode 100644 index 00000000..5f82a1eb --- /dev/null +++ b/samples/dataconverter/dataconverter.go @@ -0,0 +1,296 @@ +// Command dataconverter demonstrates a visible custom api.DataConverter on DTS. +package main + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "log" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + converterPrefix = "sample-dc:" + converterEntity = "sampledataconverterentity" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK dataconverter") +} + +func run() (err error) { + converter := prefixedJSONConverter{} + options, err := dtssample.Options() + if err != nil { + return err + } + options.DataConverter = converter + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleDataConverterTyped", typedConverterWorkflow); err != nil { + return err + } + if err := registry.AddOrchestratorN("SampleDataConverterRawBypass", rawBypassWorkflow); err != nil { + return err + } + if err := registry.AddActivityN("SampleDataConverterEcho", converterEchoActivity); err != nil { + return err + } + if err := registry.AddEntityN(converterEntity, task.NewEntityFor[converterCounter]()); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + app, err := dtssample.StartWithOptions(ctx, options, registry) + if err != nil { + return err + } + typedID := dtssample.NewInstanceID("dataconverter-typed") + rawID := dtssample.NewInstanceID("dataconverter-raw") + entityID := api.NewEntityID(converterEntity, string(dtssample.NewInstanceID("dataconverter-entity"))) + defer func() { + err = errors.Join(err, deleteConverterEntity(app.Client, entityID), dtssample.Cleanup(app.Client, typedID, rawID), app.Shutdown()) + }() + + if err := verifyTypedRoundTrip(ctx, app.Client, typedID, entityID); err != nil { + return err + } + if err := verifyRawBypass(ctx, app.Client, rawID); err != nil { + return err + } + return nil +} + +func verifyTypedRoundTrip(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, entityID api.EntityID) error { + input := samplePayload{Text: "typed-input", Count: 1} + if _, err := client.ScheduleNewOrchestration(ctx, "SampleDataConverterTyped", + api.WithInstanceID(id), api.WithInput(typedWorkflowInput{Payload: input, EntityID: entityID})); err != nil { + return err + } + if _, err := client.WaitForOrchestrationStart(ctx, id, api.WithFetchPayloads(true)); err != nil { + return err + } + if err := client.RaiseEvent(ctx, id, "payload", api.WithEventPayload(samplePayload{Text: "event", Count: 4})); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + for name, payload := range map[string]string{ + "typed input metadata": metadata.SerializedInput, + "typed output metadata": metadata.SerializedOutput, + "typed custom status": metadata.SerializedCustomStatus, + } { + if err := assertEncoded(name, payload); err != nil { + return err + } + } + + var status samplePayload + if err := metadata.ReadCustomStatus(&status); err != nil { + return err + } + if status != (samplePayload{Text: "custom-status", Count: 2}) { + return fmt.Errorf("custom status = %+v", status) + } + var output typedWorkflowOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.Input != input || + output.Activity != (samplePayload{Text: "activity:typed-input", Count: 2}) || + output.Event != (samplePayload{Text: "event", Count: 4}) || + output.EntityTotal != 4 { + return fmt.Errorf("unexpected typed converter output: %+v", output) + } + entity, err := client.GetEntity(ctx, entityID) + if err != nil { + return err + } + if entity == nil || !entity.HasState { + return errors.New("converter entity state was not persisted") + } + if err := assertEncoded("entity state", entity.SerializedState); err != nil { + return err + } + var state converterCounter + if err := entity.ReadState(&state); err != nil { + return err + } + if state.Total != 4 { + return fmt.Errorf("converter entity total = %d, want 4", state.Total) + } + fmt.Println("verified custom converter on typed input, result, event, status, and entity state") + return nil +} + +func verifyRawBypass(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID) error { + rawInput := `{"text":"legacy-json","count":5}` + if _, err := client.ScheduleNewOrchestration(ctx, "SampleDataConverterRawBypass", + api.WithInstanceID(id), api.WithRawInput(rawInput)); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + if metadata.SerializedInput != rawInput { + return fmt.Errorf("raw input was changed: %q", metadata.SerializedInput) + } + if metadata.SerializedCustomStatus != "raw-status:not-converted" { + return fmt.Errorf("raw custom status = %q", metadata.SerializedCustomStatus) + } + if err := assertEncoded("raw bypass output", metadata.SerializedOutput); err != nil { + return err + } + var output samplePayload + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output != (samplePayload{Text: "raw:legacy-json", Count: 6}) { + return fmt.Errorf("raw bypass output = %+v", output) + } + fmt.Println("verified intentional raw input/status bypass and JSON compatibility fallback") + return nil +} + +func typedConverterWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input typedWorkflowInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.SetCustomStatusValue(samplePayload{Text: "custom-status", Count: 2}); err != nil { + return nil, err + } + var activity samplePayload + if err := ctx.CallActivity("SampleDataConverterEcho", task.WithActivityInput(input.Payload)).Await(&activity); err != nil { + return nil, err + } + var event samplePayload + if err := ctx.WaitForSingleEvent("payload", 20*time.Second).Await(&event); err != nil { + return nil, err + } + var entityTotal int + if err := ctx.CallEntity(input.EntityID, "Add", task.WithEntityInput(event.Count)).Await(&entityTotal); err != nil { + return nil, err + } + return typedWorkflowOutput{Input: input.Payload, Activity: activity, Event: event, EntityTotal: entityTotal}, nil +} + +func rawBypassWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input samplePayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + ctx.SetRawCustomStatus("raw-status:not-converted") + return samplePayload{Text: "raw:" + input.Text, Count: input.Count + 1}, nil +} + +func converterEchoActivity(ctx task.ActivityContext) (any, error) { + var input samplePayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return samplePayload{Text: "activity:" + input.Text, Count: input.Count + 1}, nil +} + +type typedWorkflowInput struct { + Payload samplePayload `json:"payload"` + EntityID api.EntityID `json:"entityId"` +} + +type typedWorkflowOutput struct { + Input samplePayload `json:"input"` + Activity samplePayload `json:"activity"` + Event samplePayload `json:"event"` + EntityTotal int `json:"entityTotal"` +} + +type samplePayload struct { + Text string `json:"text"` + Count int `json:"count"` +} + +type converterCounter struct { + Total int `json:"total"` +} + +func (c *converterCounter) Add(amount int) int { + c.Total += amount + return c.Total +} + +func deleteConverterEntity(client *durabletaskscheduler.Client, id api.EntityID) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if err := client.SignalEntity(ctx, id, "delete"); err != nil { + return err + } + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + metadata, err := client.GetEntity(ctx, id) + if err != nil { + return err + } + if metadata == nil || !metadata.HasState { + return nil + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + +type prefixedJSONConverter struct{} + +func (prefixedJSONConverter) Serialize(value any) (string, error) { + data, err := json.Marshal(value) + if err != nil { + return "", err + } + return converterPrefix + base64.StdEncoding.EncodeToString(data), nil +} + +func (prefixedJSONConverter) Deserialize(payload string, target any) error { + if target == nil { + return nil + } + if strings.HasPrefix(payload, converterPrefix) { + encoded := strings.TrimPrefix(payload, converterPrefix) + data, err := base64.StdEncoding.DecodeString(encoded) + if err != nil { + return fmt.Errorf("failed to decode sample converter payload: %w", err) + } + return json.Unmarshal(data, target) + } + return json.Unmarshal([]byte(payload), target) +} + +func assertEncoded(name, payload string) error { + if !strings.HasPrefix(payload, converterPrefix) { + return fmt.Errorf("%s was not encoded with %q: %q", name, converterPrefix, payload) + } + return nil +} diff --git a/samples/entity/README.md b/samples/entity/README.md new file mode 100644 index 00000000..5907b44b --- /dev/null +++ b/samples/entity/README.md @@ -0,0 +1,32 @@ +# Durable entities + +This sample exercises entity state and routing against a real Durable Task +Scheduler task hub. It reuses one instructional flow rather than generating +separate toy programs for every API option. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/entity +``` + +## What it proves + +- Raw counter entity state, signals, scheduled signals, and entity queries. +- Entity-to-entity signals and orchestrator-to-entity calls/signals. +- Reflected entity registration with optional inputs. +- Factory registration with captured dependencies, initialized state, and + thread-safe batch cleanup. +- Persistent bank-account objects with a locked transfer and balance invariant. +- Entity-started orchestration with an explicit, sample-owned instance ID. + +Every scenario asserts the resulting state or orchestration output. The sample +prints `SAMPLE_OK entity` only after those assertions and cleanup succeed. + +## Cleanup + +All orchestration IDs are declared before scheduling and then terminated/purged +by exact ID. Entity state is deleted by signaling `delete` to the sample-owned +entity keys and waiting until their state is gone. The sample never calls +hub-wide entity maintenance or deletes a task hub. diff --git a/samples/entity/entity.go b/samples/entity/entity.go index fb75f5ee..ed7124a8 100644 --- a/samples/entity/entity.go +++ b/samples/entity/entity.go @@ -1,142 +1,295 @@ -// This sample demonstrates how to use durable entities with the Durable Task Go SDK. -// It shows two patterns: -// -// 1. A raw entity function (Counter) with manual operation dispatch -// -// 2. An auto-dispatch entity (BankAccount) where operations map to methods on a struct -// -// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" -// go run ./samples/entity +// Command entity demonstrates durable entity state, calls, signals, factories, and locks. package main import ( "context" + "errors" "fmt" "log" + "sync" "time" - "github.com/google/uuid" - "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/durabletaskscheduler" "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +const ( + counterEntityName = "samplecounter" + reflectedCounterEntityName = "samplereflectedcounter" + factoryCounterEntityName = "samplefactorycounter" + bankAccountEntityName = "samplebankaccount" +) + +var factoryStats = &factoryRecorder{} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK entity") } -func run() error { - r := task.NewTaskRegistry() - - // Pattern 1: Register a raw entity function with manual dispatch - if err := r.AddEntityN("counter", CounterEntity); err != nil { - return fmt.Errorf("failed to register counter entity: %w", err) +func run() (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddEntityN(counterEntityName, CounterEntity); err != nil { + return err } - - // Pattern 2: Register a persistent entity object with separate durable state. - bankAccountFactory := task.NewEntityObjectFactory[BankAccountState, *BankAccount]( - func(task.EntityFactoryContext) (*BankAccount, error) { - return new(BankAccount), nil + if err := registry.AddEntityN(reflectedCounterEntityName, task.NewEntityFor[ReflectedCounter]()); err != nil { + return err + } + if err := registry.AddEntityFactoryN(factoryCounterEntityName, task.NewEntityObjectFactory[FactoryCounterState, *FactoryCounter]( + func(ctx task.EntityFactoryContext) (*FactoryCounter, error) { + factoryStats.Created(ctx.ID) + return &FactoryCounter{recorder: factoryStats, id: ctx.ID}, nil }, - ) - if err := r.AddEntityFactoryN("bankaccount", bankAccountFactory); err != nil { - return fmt.Errorf("failed to register bank account entity: %w", err) + task.WithEntityStateInitializer(func(*task.EntityContext) (FactoryCounterState, error) { + return FactoryCounterState{Value: 100}, nil + }), + )); err != nil { + return err + } + if err := registry.AddEntityFactoryN(bankAccountEntityName, task.NewEntityObjectFactory[BankAccountState, *BankAccount]( + func(task.EntityFactoryContext) (*BankAccount, error) { return new(BankAccount), nil }, + )); err != nil { + return err + } + if err := registry.AddOrchestratorN("SampleEntityCounterCalls", counterCallsOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN("SampleEntityTransfer", transferOrchestrator); err != nil { + return err } - if err := r.AddOrchestratorN("transfer", TransferOrchestrator); err != nil { - return fmt.Errorf("failed to register transfer orchestrator: %w", err) + if err := registry.AddOrchestratorN("SampleEntityStartedWorkflow", entityStartedWorkflow); err != nil { + return err } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - app, err := dtssample.Start(ctx, r) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } - client := app.Client + + runID := string(dtssample.NewInstanceID("entity")) + counterID := api.NewEntityID(counterEntityName, runID+"-counter") + forwardedCounterID := api.NewEntityID(counterEntityName, runID+"-forwarded") + reflectedID := api.NewEntityID(reflectedCounterEntityName, runID+"-reflected") + factoryID := api.NewEntityID(factoryCounterEntityName, runID+"-factory") + checkingID := api.NewEntityID(bankAccountEntityName, runID+"-checking") + savingsID := api.NewEntityID(bankAccountEntityName, runID+"-savings") + entities := []api.EntityID{counterID, forwardedCounterID, reflectedID, factoryID, checkingID, savingsID} + + counterWorkflowID := dtssample.NewInstanceID("entity-counter-calls") + transferID := dtssample.NewInstanceID("entity-transfer") + startedID := dtssample.NewInstanceID("entity-started") + ids := []api.InstanceID{counterWorkflowID, transferID, startedID} defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) - } + err = errors.Join(err, deleteEntities(app.Client, entities...), dtssample.Cleanup(app.Client, ids...), app.Shutdown()) }() - // Entity state is durable, so each run uses fresh keys rather than - // accumulating on top of a previous run's balances. - run := uuid.NewString() - - // --- Demo 1: Counter entity (raw function) --- - fmt.Println("=== Counter Entity Demo ===") - counterID := api.NewEntityID("counter", "myCounter-"+run) + if err := verifyRawCounterSignalsAndQueries(ctx, app.Client, counterID); err != nil { + return err + } + if err := verifyEntityToEntitySignal(ctx, app.Client, counterID, forwardedCounterID); err != nil { + return err + } + if err := verifyCounterCallWorkflow(ctx, app.Client, counterWorkflowID, counterID); err != nil { + return err + } + if err := verifyReflectedCounter(ctx, app.Client, reflectedID); err != nil { + return err + } + if err := verifyFactoryCounter(ctx, app.Client, factoryID); err != nil { + return err + } + if err := verifyLockedTransfer(ctx, app.Client, transferID, checkingID, savingsID); err != nil { + return err + } + if err := verifyEntityStartedWorkflow(ctx, app.Client, counterID, startedID); err != nil { + return err + } + return nil +} - // Signal the entity to perform operations +func verifyRawCounterSignalsAndQueries(ctx context.Context, client *durabletaskscheduler.Client, counterID api.EntityID) error { if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(10)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) + return err } if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(5)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) + return err } - if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(-3)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) + if _, err := waitForEntityState[int](ctx, client, counterID, 15); err != nil { + return err } - - // Query the entity state - meta, err := waitForEntityState(ctx, client, counterID, "12") + if err := client.SignalEntity(ctx, counterID, "add", api.WithSignalInput(4), + api.WithSignalScheduledTime(time.Now().UTC().Add(300*time.Millisecond))); err != nil { + return err + } + if _, err := waitForEntityState[int](ctx, client, counterID, 19); err != nil { + return err + } + query, err := client.QueryEntities(ctx, api.EntityQuery{InstanceIDStartsWith: counterID.String(), PageSize: 10}) if err != nil { - return fmt.Errorf("failed to fetch entity: %w", err) + return err + } + if len(query.Entities) != 1 || query.Entities[0].InstanceID.String() != counterID.String() { + return fmt.Errorf("counter query returned %d entities, want exactly %s", len(query.Entities), counterID) + } + fmt.Println("verified raw counter state, signals, scheduled signals, and query") + return nil +} + +func verifyEntityToEntitySignal(ctx context.Context, client *durabletaskscheduler.Client, source, target api.EntityID) error { + if err := client.SignalEntity(ctx, source, "forwardAdd", api.WithSignalInput(forwardAdd{Target: target, Amount: 7})); err != nil { + return err } - fmt.Printf("Counter state: %s\n", meta.SerializedState) // Expected: 12 + if _, err := waitForEntityState[int](ctx, client, target, 7); err != nil { + return err + } + fmt.Println("verified entity-to-entity signal") + return nil +} - // --- Demo 2: BankAccount entity (auto-dispatch) --- - fmt.Println("\n=== Bank Account Entity Demo ===") - accountID := api.NewEntityID("bankaccount", "checking-"+run) +func verifyCounterCallWorkflow(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, counterID api.EntityID) error { + if _, err := client.ScheduleNewOrchestration(ctx, "SampleEntityCounterCalls", + api.WithInstanceID(id), api.WithInput(counterWorkflowInput{Counter: counterID})); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output counterWorkflowOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.AfterCall != 22 || output.AfterSignal != 28 { + return fmt.Errorf("counter workflow output = %+v, want call 22 and signal 28", output) + } + fmt.Println("verified orchestrator entity calls and signals") + return nil +} - if err := client.SignalEntity(ctx, accountID, "Deposit", api.WithSignalInput(1000)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) +func verifyReflectedCounter(ctx context.Context, client *durabletaskscheduler.Client, id api.EntityID) error { + if err := client.SignalEntity(ctx, id, "Add"); err != nil { + return err } - if err := client.SignalEntity(ctx, accountID, "Deposit", api.WithSignalInput(500)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) + if _, err := waitForEntityState[ReflectedCounter](ctx, client, id, ReflectedCounter{Value: 1}); err != nil { + return err } - if err := client.SignalEntity(ctx, accountID, "Withdraw", api.WithSignalInput(200)); err != nil { - return fmt.Errorf("failed to signal entity: %w", err) + if err := client.SignalEntity(ctx, id, "Add", api.WithSignalInput(9)); err != nil { + return err } + if _, err := waitForEntityState[ReflectedCounter](ctx, client, id, ReflectedCounter{Value: 10}); err != nil { + return err + } + fmt.Println("verified reflected entity model and optional input") + return nil +} - meta, err = waitForEntityState(ctx, client, accountID, `{"balance":1300}`) - if err != nil { - return fmt.Errorf("failed to fetch entity: %w", err) +func verifyFactoryCounter(ctx context.Context, client *durabletaskscheduler.Client, id api.EntityID) error { + beforeCreated, beforeClosed := factoryStats.Counts() + if err := client.SignalEntity(ctx, id, "Add"); err != nil { + return err + } + if _, err := waitForEntityState[FactoryCounterState](ctx, client, id, FactoryCounterState{Value: 101}); err != nil { + return err + } + created, closed := factoryStats.Counts() + if created <= beforeCreated || closed <= beforeClosed { + return fmt.Errorf("factory cleanup not observed; before=(%d,%d) after=(%d,%d)", beforeCreated, beforeClosed, created, closed) } - fmt.Printf("Bank account state: %s\n", meta.SerializedState) // Expected: {"balance":1300} + fmt.Println("verified factory capture, initialized state, optional input, and batch cleanup") + return nil +} - savingsID := api.NewEntityID("bankaccount", "savings-"+run) - if err := client.SignalEntity(ctx, savingsID, "Deposit", api.WithSignalInput(100)); err != nil { - return fmt.Errorf("failed to initialize savings account: %w", err) +func verifyLockedTransfer(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, checking, savings api.EntityID) error { + if err := client.SignalEntity(ctx, checking, "Deposit", api.WithSignalInput(1000)); err != nil { + return err } - if _, err := waitForEntityState(ctx, client, savingsID, `{"balance":100}`); err != nil { - return fmt.Errorf("failed to initialize savings account: %w", err) + if err := client.SignalEntity(ctx, savings, "Deposit", api.WithSignalInput(100)); err != nil { + return err } - transferID, err := client.ScheduleNewOrchestration( - ctx, - "transfer", - api.WithInput(TransferInput{From: accountID, To: savingsID, Amount: 300}), - ) - if err != nil { - return fmt.Errorf("failed to schedule transfer: %w", err) + if _, err := waitForEntityState[BankAccountState](ctx, client, checking, BankAccountState{Balance: 1000}); err != nil { + return err + } + if _, err := waitForEntityState[BankAccountState](ctx, client, savings, BankAccountState{Balance: 100}); err != nil { + return err } - transfer, err := client.WaitForOrchestrationCompletion(ctx, transferID) + input := transferInput{From: checking, To: savings, Amount: 300} + if _, err := client.ScheduleNewOrchestration(ctx, "SampleEntityTransfer", + api.WithInstanceID(id), api.WithInput(input)); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("transfer failed: %w", err) + return err } - fmt.Printf("Transfer result: %s\n", transfer.SerializedOutput) + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output transferOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.From != 700 || output.To != 400 || output.Total != 1100 { + return fmt.Errorf("transfer output = %+v, want balances 700/400 and invariant 1100", output) + } + fmt.Println("verified locked transfer balance invariant") + return nil +} - fmt.Println("\nDone!") +func verifyEntityStartedWorkflow(ctx context.Context, client *durabletaskscheduler.Client, entityID api.EntityID, startedID api.InstanceID) error { + if err := client.SignalEntity(ctx, entityID, "startWorkflow", + api.WithSignalInput(startWorkflowInput{InstanceID: string(startedID), Message: "from-entity"})); err != nil { + return err + } + metadata, err := waitForOrchestrationCreatedAndCompleted(ctx, client, startedID) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output != "started:from-entity" { + return fmt.Errorf("entity-started workflow output = %q", output) + } + fmt.Println("verified entity-started orchestration") return nil } -// --- Pattern 1: Raw entity function --- +func waitForOrchestrationCreatedAndCompleted( + ctx context.Context, + client *durabletaskscheduler.Client, + id api.InstanceID, +) (*api.OrchestrationMetadata, error) { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + _, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(false)) + if err == nil { + return client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + } + if !errors.Is(err, api.ErrInstanceNotFound) { + return nil, err + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-ticker.C: + } + } +} -// CounterEntity is a simple counter entity that supports "add", "get", and "reset" operations. func CounterEntity(ctx *task.EntityContext) (any, error) { var count int if ctx.HasState() { @@ -144,7 +297,6 @@ func CounterEntity(ctx *task.EntityContext) (any, error) { return nil, err } } - switch ctx.Operation { case "add": var amount int @@ -153,23 +305,113 @@ func CounterEntity(ctx *task.EntityContext) (any, error) { } count += amount case "get": - // just return current value case "reset": count = 0 + case "delete": + ctx.DeleteState() + return nil, nil + case "forwardAdd": + var input forwardAdd + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.SignalEntity(input.Target, "add", input.Amount); err != nil { + return nil, err + } + case "startWorkflow": + var input startWorkflowInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.StartNewOrchestration("SampleEntityStartedWorkflow", + task.WithEntityStartOrchestrationInstanceID(input.InstanceID), + task.WithEntityStartOrchestrationInput(input.Message)); err != nil { + return nil, err + } default: - return nil, fmt.Errorf("unknown operation: %s", ctx.Operation) + return nil, fmt.Errorf("unknown counter operation %q", ctx.Operation) } - if err := ctx.SetState(count); err != nil { return nil, err } return count, nil } -// --- Pattern 2: Auto-dispatch entity --- +type forwardAdd struct { + Target api.EntityID `json:"target"` + Amount int `json:"amount"` +} + +type startWorkflowInput struct { + InstanceID string `json:"instanceId"` + Message string `json:"message"` +} + +type ReflectedCounter struct { + Value int `json:"value"` +} + +func (c *ReflectedCounter) Add(input task.OptionalEntityInput[int]) (int, error) { + c.Value += input.Or(1) + return c.Value, nil +} + +type FactoryCounterState struct { + Value int `json:"value"` +} + +type FactoryCounter struct { + task.EntityObjectBase[FactoryCounterState] + recorder *factoryRecorder + id api.EntityID +} + +func (c *FactoryCounter) Add(input task.OptionalEntityInput[int]) (int, error) { + c.State().Value += input.Or(1) + return c.State().Value, nil +} + +func (c *FactoryCounter) CloseEntityBatch(context.Context) error { + c.recorder.Closed(c.id) + return nil +} + +type factoryRecorder struct { + mu sync.Mutex + created map[string]int + closed map[string]int +} + +func (r *factoryRecorder) Created(id api.EntityID) { + r.mu.Lock() + defer r.mu.Unlock() + if r.created == nil { + r.created = make(map[string]int) + } + r.created[id.String()]++ +} + +func (r *factoryRecorder) Closed(id api.EntityID) { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed == nil { + r.closed = make(map[string]int) + } + r.closed[id.String()]++ +} + +func (r *factoryRecorder) Counts() (created, closed int) { + r.mu.Lock() + defer r.mu.Unlock() + for _, count := range r.created { + created += count + } + for _, count := range r.closed { + closed += count + } + return created, closed +} -// BankAccount is a struct-based entity. Public methods are automatically -// dispatched by operation name (case-insensitive). type BankAccountState struct { Balance int `json:"balance"` } @@ -178,35 +420,52 @@ type BankAccount struct { task.EntityObjectBase[BankAccountState] } -func (a *BankAccount) Deposit(amount int) (any, error) { +func (a *BankAccount) Deposit(amount int) (int, error) { a.State().Balance += amount return a.State().Balance, nil } -func (a *BankAccount) Withdraw(amount int) (any, error) { +func (a *BankAccount) Withdraw(amount int) (int, error) { if amount > a.State().Balance { - return nil, fmt.Errorf( - "insufficient funds: balance=%d, withdrawal=%d", - a.State().Balance, - amount, - ) + return 0, fmt.Errorf("insufficient funds: balance=%d withdrawal=%d", a.State().Balance, amount) } a.State().Balance -= amount return a.State().Balance, nil } -func (a *BankAccount) Get() (any, error) { - return a.State().Balance, nil +func counterCallsOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input counterWorkflowInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var afterCall int + if err := ctx.CallEntity(input.Counter, "add", task.WithEntityInput(3)).Await(&afterCall); err != nil { + return nil, err + } + if err := ctx.SignalEntity(input.Counter, "add", task.WithSignalEntityInput(6)); err != nil { + return nil, err + } + if err := ctx.CreateTimer(250 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var afterSignal int + if err := ctx.CallEntity(input.Counter, "get").Await(&afterSignal); err != nil { + return nil, err + } + return counterWorkflowOutput{AfterCall: afterCall, AfterSignal: afterSignal}, nil } -type TransferInput struct { - From api.EntityID `json:"from"` - To api.EntityID `json:"to"` - Amount int `json:"amount"` +type counterWorkflowInput struct { + Counter api.EntityID `json:"counter"` +} + +type counterWorkflowOutput struct { + AfterCall int `json:"afterCall"` + AfterSignal int `json:"afterSignal"` } -func TransferOrchestrator(ctx *task.OrchestrationContext) (any, error) { - var input TransferInput +func transferOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input transferInput if err := ctx.GetInput(&input); err != nil { return nil, err } @@ -217,49 +476,95 @@ func TransferOrchestrator(ctx *task.OrchestrationContext) (any, error) { defer unlock() var fromBalance int - if err := ctx.CallEntity( - input.From, - "Withdraw", - task.WithEntityInput(input.Amount), - ).Await(&fromBalance); err != nil { + if err := ctx.CallEntity(input.From, "Withdraw", task.WithEntityInput(input.Amount)).Await(&fromBalance); err != nil { return nil, err } var toBalance int - if err := ctx.CallEntity( - input.To, - "Deposit", - task.WithEntityInput(input.Amount), - ).Await(&toBalance); err != nil { + if err := ctx.CallEntity(input.To, "Deposit", task.WithEntityInput(input.Amount)).Await(&toBalance); err != nil { return nil, err } - return map[string]int{"from": fromBalance, "to": toBalance}, nil + return transferOutput{From: fromBalance, To: toBalance, Total: fromBalance + toBalance}, nil } -func waitForEntityState( +type transferInput struct { + From api.EntityID `json:"from"` + To api.EntityID `json:"to"` + Amount int `json:"amount"` +} + +type transferOutput struct { + From int `json:"from"` + To int `json:"to"` + Total int `json:"total"` +} + +func entityStartedWorkflow(ctx *task.OrchestrationContext) (any, error) { + var message string + if err := ctx.GetInput(&message); err != nil { + return nil, err + } + return "started:" + message, nil +} + +func waitForEntityState[T comparable]( ctx context.Context, client *durabletaskscheduler.Client, entityID api.EntityID, - expected string, + expected T, +) (*api.EntityMetadata, error) { + return waitForEntity(ctx, client, entityID, func(metadata *api.EntityMetadata) (bool, error) { + if metadata == nil || !metadata.HasState { + return false, nil + } + var actual T + if err := metadata.ReadState(&actual); err != nil { + return false, err + } + return actual == expected, nil + }) +} + +func deleteEntities(client *durabletaskscheduler.Client, entityIDs ...api.EntityID) error { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + var failures []error + for _, id := range entityIDs { + if err := client.SignalEntity(ctx, id, "delete"); err != nil { + failures = append(failures, fmt.Errorf("delete %s: %w", id, err)) + continue + } + if _, err := waitForEntity(ctx, client, id, func(metadata *api.EntityMetadata) (bool, error) { + return metadata == nil || !metadata.HasState, nil + }); err != nil { + failures = append(failures, fmt.Errorf("wait for %s delete: %w", id, err)) + } + } + return errors.Join(failures...) +} + +func waitForEntity( + ctx context.Context, + client *durabletaskscheduler.Client, + entityID api.EntityID, + ready func(*api.EntityMetadata) (bool, error), ) (*api.EntityMetadata, error) { - timeout := time.NewTimer(30 * time.Second) - defer timeout.Stop() ticker := time.NewTicker(100 * time.Millisecond) defer ticker.Stop() for { metadata, err := client.GetEntity(ctx, entityID) - // The entity does not exist until the service has processed its first - // signal, so a not-found result means "not ready yet", not a failure. if err != nil { return nil, err } - if metadata != nil && metadata.SerializedState == expected { + ok, err := ready(metadata) + if err != nil { + return nil, err + } + if ok { return metadata, nil } select { case <-ctx.Done(): return nil, ctx.Err() - case <-timeout.C: - return nil, fmt.Errorf("timed out waiting for %s state %s", entityID.String(), expected) case <-ticker.C: } } diff --git a/samples/history/README.md b/samples/history/README.md new file mode 100644 index 00000000..1c614a57 --- /dev/null +++ b/samples/history/README.md @@ -0,0 +1,27 @@ +# Orchestration history + +This sample runs a bounded workflow and reads the service's actual orchestration +history through the public buffered and streaming APIs. It does not construct +internal protocol events or use hand-built history as a substitute. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/history +``` + +## What it proves + +- The completed execution ID is captured from metadata and used to pin history + reads to one execution. +- Buffered history preserves meaningful event order and typed payload readers + decode started input, activity input/result, and completed output. +- Streaming history returns the same event identity and order as the buffered + snapshot. +- Event and byte caps fail with `api.ErrHistoryLimitExceeded`. + +`SAMPLE_OK history` is printed only after the real history checks and exact-ID +cleanup succeed. Some emulator builds may not support every history surface; an +unsupported or inconsistent response is reported as a failed run, not fake +coverage. diff --git a/samples/history/history.go b/samples/history/history.go new file mode 100644 index 00000000..b7b28319 --- /dev/null +++ b/samples/history/history.go @@ -0,0 +1,267 @@ +// Command history reads real DTS orchestration history through buffered and streaming APIs. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "slices" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK history") +} + +func run() (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleHistoryWorkflow", historyWorkflow); err != nil { + return err + } + if err := registry.AddActivityN("SampleHistoryEcho", historyEchoActivity); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + id := dtssample.NewInstanceID("history") + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, id), app.Shutdown()) }() + + input := historyPayload{Label: "history", Count: 2} + if _, err := app.Client.ScheduleNewOrchestration(ctx, "SampleHistoryWorkflow", + api.WithInstanceID(id), api.WithInput(input)); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + if metadata.ExecutionID == "" { + return errors.New("completed metadata did not include an execution ID") + } + var output historyOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.Activity != "history:2" || output.FinalCount != 3 { + return fmt.Errorf("history workflow output = %+v", output) + } + + query := api.HistoryQuery{ExecutionID: metadata.ExecutionID, MaxEvents: 100, MaxBytes: 1 << 20} + history, err := app.Client.GetOrchestrationHistory(ctx, id, query) + if err != nil { + return err + } + if err := verifyBufferedHistory(history, input, output, metadata.ExecutionID); err != nil { + return err + } + if err := verifyStreamedHistory(ctx, app.Client, id, query, history); err != nil { + return err + } + if err := verifyHistoryCaps(ctx, app.Client, id, metadata.ExecutionID); err != nil { + return err + } + return nil +} + +func historyWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input historyPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + activityInput := historyPayload{Label: input.Label, Count: input.Count} + var activityResult string + if err := ctx.CallActivity("SampleHistoryEcho", task.WithActivityInput(activityInput)).Await(&activityResult); err != nil { + return nil, err + } + if err := ctx.CreateTimer(100 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + return historyOutput{Activity: activityResult, FinalCount: input.Count + 1}, nil +} + +func historyEchoActivity(ctx task.ActivityContext) (any, error) { + var input historyPayload + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return fmt.Sprintf("%s:%d", input.Label, input.Count), nil +} + +type historyPayload struct { + Label string `json:"label"` + Count int `json:"count"` +} + +type historyOutput struct { + Activity string `json:"activity"` + FinalCount int `json:"finalCount"` +} + +func verifyBufferedHistory(history *api.OrchestrationHistory, input historyPayload, output historyOutput, executionID string) error { + if history == nil { + return errors.New("buffered history was nil") + } + if history.ExecutionID != executionID { + return fmt.Errorf("history execution ID = %q, want %q", history.ExecutionID, executionID) + } + if len(history.Events) < 6 { + return fmt.Errorf("history contained %d events, want a real completed workflow history", len(history.Events)) + } + if err := verifyEventOrder(history.Events); err != nil { + return err + } + started := firstEvent(history.Events, api.HistoryEventExecutionStarted) + if started == nil || started.ExecutionStarted == nil { + return errors.New("history missing ExecutionStarted") + } + var startedInput historyPayload + if err := started.ReadInput(&startedInput); err != nil { + return err + } + if startedInput != input { + return fmt.Errorf("ExecutionStarted input = %+v", startedInput) + } + + scheduled := firstTaskScheduled(history.Events, "SampleHistoryEcho") + if scheduled == nil { + return errors.New("history missing SampleHistoryEcho schedule") + } + var scheduledInput historyPayload + if err := scheduled.ReadInput(&scheduledInput); err != nil { + return err + } + if scheduledInput != input { + return fmt.Errorf("TaskScheduled input = %+v", scheduledInput) + } + + completedTask := firstEvent(history.Events, api.HistoryEventTaskCompleted) + if completedTask == nil { + return errors.New("history missing TaskCompleted") + } + var activityResult string + if err := completedTask.ReadResult(&activityResult); err != nil { + return err + } + if activityResult != output.Activity { + return fmt.Errorf("TaskCompleted result = %q", activityResult) + } + completed := firstEvent(history.Events, api.HistoryEventExecutionCompleted) + if completed == nil || completed.ExecutionCompleted == nil { + return errors.New("history missing ExecutionCompleted") + } + var completedOutput historyOutput + if err := completed.ReadResult(&completedOutput); err != nil { + return err + } + if completedOutput != output { + return fmt.Errorf("ExecutionCompleted output = %+v", completedOutput) + } + fmt.Printf("verified buffered history with %d events pinned to execution %s\n", len(history.Events), executionID) + return nil +} + +func verifyStreamedHistory( + ctx context.Context, + client *durabletaskscheduler.Client, + id api.InstanceID, + query api.HistoryQuery, + buffered *api.OrchestrationHistory, +) error { + var streamed []*api.HistoryEvent + if err := client.StreamOrchestrationHistory(ctx, id, query, func(event *api.HistoryEvent) error { + streamed = append(streamed, event) + return nil + }); err != nil { + return err + } + if len(streamed) != len(buffered.Events) { + return fmt.Errorf("streamed %d events, buffered %d", len(streamed), len(buffered.Events)) + } + for i := range streamed { + if streamed[i].Type != buffered.Events[i].Type || streamed[i].EventID != buffered.Events[i].EventID { + return fmt.Errorf("streamed event %d = (%s,%d), buffered (%s,%d)", + i, streamed[i].Type, streamed[i].EventID, buffered.Events[i].Type, buffered.Events[i].EventID) + } + } + fmt.Println("verified streamed history identity and order match buffered history") + return nil +} + +func verifyHistoryCaps(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, executionID string) error { + _, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID, MaxEvents: 1}) + if !errors.Is(err, api.ErrHistoryLimitExceeded) { + return fmt.Errorf("MaxEvents cap error = %v, want %v", err, api.ErrHistoryLimitExceeded) + } + _, err = client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID, MaxBytes: 64}) + if !errors.Is(err, api.ErrHistoryLimitExceeded) { + return fmt.Errorf("MaxBytes cap error = %v, want %v", err, api.ErrHistoryLimitExceeded) + } + fmt.Println("verified buffered history event and byte caps") + return nil +} + +func verifyEventOrder(events []*api.HistoryEvent) error { + positions := map[api.HistoryEventType]int{} + for i, event := range events { + if _, ok := positions[event.Type]; !ok { + positions[event.Type] = i + } + } + required := []api.HistoryEventType{ + api.HistoryEventExecutionStarted, + api.HistoryEventTaskScheduled, + api.HistoryEventTaskCompleted, + api.HistoryEventTimerCreated, + api.HistoryEventTimerFired, + api.HistoryEventExecutionCompleted, + } + for _, eventType := range required { + if _, ok := positions[eventType]; !ok { + return fmt.Errorf("history missing %s", eventType) + } + } + for i := 1; i < len(required); i++ { + if positions[required[i-1]] > positions[required[i]] { + return fmt.Errorf("%s appeared after %s", required[i-1], required[i]) + } + } + return nil +} + +func firstEvent(events []*api.HistoryEvent, eventType api.HistoryEventType) *api.HistoryEvent { + index := slices.IndexFunc(events, func(event *api.HistoryEvent) bool { + return event != nil && event.Type == eventType + }) + if index < 0 { + return nil + } + return events[index] +} + +func firstTaskScheduled(events []*api.HistoryEvent, name string) *api.HistoryEvent { + index := slices.IndexFunc(events, func(event *api.HistoryEvent) bool { + return event != nil && event.Type == api.HistoryEventTaskScheduled && + event.TaskScheduled != nil && event.TaskScheduled.Name == name + }) + if index < 0 { + return nil + } + return events[index] +} diff --git a/samples/management/README.md b/samples/management/README.md new file mode 100644 index 00000000..ee19b960 --- /dev/null +++ b/samples/management/README.md @@ -0,0 +1,45 @@ +# Management sample + +This sample demonstrates safe, ID-scoped Durable Task Scheduler management operations: + +- start and wait for completion +- typed custom status progress +- `QueryInstances` with tags, ID prefix, payload fetching, and continuation pages +- `ListInstanceIDs` for completed orchestration IDs +- suspend and resume +- terminate with a supplied output +- restart and validate the restarted payload +- status-based instance-ID reuse +- single and batch ID-scoped purge + +The sample intentionally does **not** use filtered purge or clean-hub operations. + +## Prerequisites + +Set `DTS_CONNECTION_STRING` to a task hub you can use for samples: + +```bash +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +`ListInstanceIDs` and `RestartInstance` must be implemented by the target service. If a target reports success but omits owned IDs or drops restart payloads, the sample exits with a precise error instead of treating the limitation as a pass. + +## Run + +```bash +go run ./samples/management +``` + +## Expected result + +The program validates each management scenario and prints: + +```text +SAMPLE_OK management +``` + +only after every scenario and cleanup succeeds. + +## Cleanup + +Every orchestration uses a `sample-management-*` unique owned ID that is recorded before scheduling. Cleanup uses `dtssample.Cleanup` before worker shutdown, and the purge scenario uses only explicit single/batch ID-scoped purge requests. diff --git a/samples/management/management.go b/samples/management/management.go new file mode 100644 index 00000000..fc0779eb --- /dev/null +++ b/samples/management/management.go @@ -0,0 +1,525 @@ +// Command management demonstrates instance-management operations scoped to +// sample-owned orchestration IDs. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "slices" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + managementCompleteName = "SampleManagementComplete" + managementProgressName = "SampleManagementProgress" + managementWaitName = "SampleManagementWait" + managementTimerName = "SampleManagementTimer" +) + +type progressStatus struct { + Step int `json:"step"` + State string `json:"state"` +} + +type waitInput struct { + Count int `json:"count"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK management") +} + +func run() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(managementCompleteName, managementCompleteOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN(managementProgressName, managementProgressOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN(managementWaitName, managementWaitOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN(managementTimerName, managementTimerOrchestrator); err != nil { + return err + } + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + defer func() { + err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) + }() + + checks := []struct { + name string + run func(context.Context, *durabletaskscheduler.Client, *[]api.InstanceID) error + }{ + {"start-wait-custom-status", verifyStartWaitAndCustomStatus}, + {"query-pages-and-id-list", verifyQueryPagesAndIDList}, + {"suspend-resume", verifySuspendResume}, + {"terminate", verifyTerminate}, + {"restart", verifyRestart}, + {"status-based-id-reuse", verifyStatusBasedIDReuse}, + {"single-and-batch-purge", verifySingleAndBatchPurge}, + } + for _, check := range checks { + if err := check.run(ctx, app.Client, &ownedIDs); err != nil { + return fmt.Errorf("%s: %w", check.name, err) + } + fmt.Printf("verified %s\n", check.name) + } + return nil +} + +func verifyStartWaitAndCustomStatus(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("management-progress") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementProgressName, + api.WithInstanceID(id), + api.WithTags(map[string]string{"sample": "management", "scenario": "progress"}), + ); err != nil { + return err + } + if err := waitForCustomStatus(ctx, client, id, progressStatus{Step: 1, State: "waiting"}); err != nil { + return err + } + if err := client.RaiseEvent(ctx, id, "finish", api.WithEventPayload("done")); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output != "done" { + return fmt.Errorf("progress output=%q, want done", output) + } + return nil +} + +func verifyQueryPagesAndIDList(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + prefix := string(dtssample.NewInstanceID("management-query")) + ids := make([]api.InstanceID, 0, 3) + for index := range 3 { + id := api.InstanceID(fmt.Sprintf("%s-%d", prefix, index)) + ids = append(ids, id) + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementCompleteName, + api.WithInstanceID(id), + api.WithInput(fmt.Sprintf("query-%d", index)), + api.WithTags(map[string]string{"sample": "management", "query-page": fmt.Sprintf("%d", index%2)}), + ); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + } + + first, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + InstanceIDPrefix: prefix, + Tags: map[string]string{"sample": "management", "query-page": "0"}, + PageSize: 1, + FetchInputsAndOutputs: true, + }) + if err != nil { + return err + } + if len(first.Orchestrations) != 1 || first.ContinuationToken == "" { + return fmt.Errorf("first query page = %+v; expected one result and a continuation token", first) + } + second, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + InstanceIDPrefix: prefix, + Tags: map[string]string{"sample": "management", "query-page": "0"}, + PageSize: 1, + ContinuationToken: first.ContinuationToken, + FetchInputsAndOutputs: true, + }) + if err != nil { + return err + } + if len(second.Orchestrations) != 1 || second.Orchestrations[0].InstanceID == first.Orchestrations[0].InstanceID { + return fmt.Errorf("second query page = %+v; expected a different matching instance", second) + } + + listed, err := collectCompletedInstanceIDs(ctx, client, 20) + if err != nil { + return err + } + for _, id := range ids { + if !listed[id] { + return fmt.Errorf("ListInstanceIDs did not return owned completed instance %s; this target needs service fallback instead of treating omission as pass", id) + } + } + return nil +} + +func verifySuspendResume(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("management-suspend") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementWaitName, + api.WithInstanceID(id), + api.WithInput(waitInput{Count: 2}), + api.WithTags(map[string]string{"sample": "management", "scenario": "suspend-resume"}), + ); err != nil { + return err + } + if _, err := client.WaitForOrchestrationStart(ctx, id); err != nil { + return err + } + if err := client.SuspendOrchestration(ctx, id, "management sample"); err != nil { + return err + } + for _, value := range []string{"a", "b"} { + if err := client.RaiseEvent(ctx, id, "value", api.WithEventPayload(value)); err != nil { + return err + } + } + waitCtx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + _, waitErr := client.WaitForOrchestrationCompletion(waitCtx, id) + cancel() + if !errors.Is(waitErr, context.DeadlineExceeded) { + return fmt.Errorf("suspended instance completed unexpectedly: %v", waitErr) + } + suspended, err := client.FetchOrchestrationMetadata(ctx, id) + if err != nil { + return err + } + if suspended.RuntimeStatus != api.RUNTIME_STATUS_SUSPENDED { + return fmt.Errorf("suspended status=%s, want SUSPENDED", suspended.RuntimeStatus) + } + if err := client.ResumeOrchestration(ctx, id, "management sample"); err != nil { + return err + } + completed, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(completed); err != nil { + return err + } + var values []string + if err := completed.ReadOutput(&values); err != nil { + return err + } + if !slices.Equal(values, []string{"a", "b"}) { + return fmt.Errorf("suspend/resume output=%#v", values) + } + return nil +} + +func verifyTerminate(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("management-terminate") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementTimerName, + api.WithInstanceID(id), + api.WithTags(map[string]string{"sample": "management", "scenario": "terminate"}), + ); err != nil { + return err + } + if _, err := client.WaitForOrchestrationStart(ctx, id); err != nil { + return err + } + if err := client.TerminateOrchestration(ctx, id, api.WithOutput("terminated by management sample")); err != nil { + return err + } + terminated, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if terminated.RuntimeStatus != api.RUNTIME_STATUS_TERMINATED { + return fmt.Errorf("terminate status=%s, want TERMINATED", terminated.RuntimeStatus) + } + var output string + if err := terminated.ReadOutput(&output); err != nil { + return err + } + if output != "terminated by management sample" { + return fmt.Errorf("terminate output=%q", output) + } + return nil +} + +func verifyRestart(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("management-restart") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementCompleteName, + api.WithInstanceID(id), + api.WithInput("restart-payload"), + api.WithTags(map[string]string{"sample": "management", "scenario": "restart"}), + ); err != nil { + return err + } + original, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(original); err != nil { + return err + } + restartedID, err := client.RestartInstance(ctx, id, api.WithRestartNewInstanceID(true)) + if errors.Is(err, api.ErrFeatureNotSupported) { + return fmt.Errorf("RestartInstance is not supported by this target; run the management sample against a scheduler that implements restart") + } + if err != nil { + return err + } + *ownedIDs = append(*ownedIDs, restartedID) + restarted, err := client.WaitForOrchestrationCompletion(ctx, restartedID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(restarted); err != nil { + return err + } + var output string + if err := restarted.ReadOutput(&output); err != nil { + return err + } + if output != "restart-payload" { + return fmt.Errorf("RestartInstance returned output %q, want restart-payload; missing restart payloads are a service blocker, not a sample pass", output) + } + return nil +} + +func verifyStatusBasedIDReuse(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("management-id-reuse") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + managementCompleteName, + api.WithInstanceID(id), + api.WithInput("first"), + api.WithTags(map[string]string{"sample": "management", "scenario": "id-reuse"}), + ); err != nil { + return err + } + first, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(first); err != nil { + return err + } + if _, err := client.ScheduleNewOrchestration( + ctx, + managementCompleteName, + api.WithInstanceID(id), + api.WithInput("duplicate"), + api.WithOrchestrationIDReusePolicy(&api.OrchestrationIDReusePolicy{ + DedupeStatuses: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + }), + ); !errors.Is(err, api.ErrDuplicateInstance) { + return fmt.Errorf("completed-status duplicate scheduling error=%v, want ErrDuplicateInstance", err) + } + if _, err := client.ScheduleNewOrchestration( + ctx, + managementCompleteName, + api.WithInstanceID(id), + api.WithInput("second"), + api.WithOrchestrationIDReusePolicy(&api.OrchestrationIDReusePolicy{ + DedupeStatuses: []api.OrchestrationStatus{api.RUNTIME_STATUS_RUNNING, api.RUNTIME_STATUS_PENDING}, + }), + api.WithTags(map[string]string{"sample": "management", "scenario": "id-reuse"}), + ); err != nil { + return err + } + second, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(second); err != nil { + return err + } + var output string + if err := second.ReadOutput(&output); err != nil { + return err + } + if output != "second" || second.ExecutionID == first.ExecutionID { + return fmt.Errorf("ID reuse output=%q execution=%q first=%q", output, second.ExecutionID, first.ExecutionID) + } + return nil +} + +func verifySingleAndBatchPurge(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + singleID := dtssample.NewInstanceID("management-single-purge") + batchA := dtssample.NewInstanceID("management-batch-purge") + batchB := dtssample.NewInstanceID("management-batch-purge") + ids := []api.InstanceID{singleID, batchA, batchB} + *ownedIDs = append(*ownedIDs, ids...) + for _, id := range ids { + if _, err := client.ScheduleNewOrchestration(ctx, managementCompleteName, api.WithInstanceID(id), api.WithInput(string(id))); err != nil { + return err + } + if metadata, err := client.WaitForOrchestrationCompletion(ctx, id); err != nil { + return err + } else if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + } + single, err := client.PurgeInstances(ctx, api.PurgeInstancesRequest{ + InstanceIDs: []api.InstanceID{singleID}, + Recursive: true, + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + return fmt.Errorf("ID-scoped PurgeInstances is not supported by this target") + } + if err != nil { + return err + } + if !single.IsComplete { + return fmt.Errorf("single purge did not complete: %+v", single) + } + batch, err := client.PurgeInstances(ctx, api.PurgeInstancesRequest{ + InstanceIDs: []api.InstanceID{batchA, batchB}, + Recursive: true, + }) + if err != nil { + return err + } + if !batch.IsComplete { + return fmt.Errorf("batch purge did not complete: %+v", batch) + } + for _, id := range ids { + if _, err := client.FetchOrchestrationMetadata(ctx, id); !errors.Is(err, api.ErrInstanceNotFound) { + return fmt.Errorf("purged instance %s remains readable or lookup failed: %v", id, err) + } + } + return nil +} + +func managementCompleteOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil +} + +func managementProgressOrchestrator(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.SetCustomStatusValue(progressStatus{Step: 1, State: "waiting"}); err != nil { + return nil, err + } + var value string + if err := ctx.WaitForSingleEvent("finish", time.Minute).Await(&value); err != nil { + return nil, err + } + if err := ctx.SetCustomStatusValue(progressStatus{Step: 2, State: "complete"}); err != nil { + return nil, err + } + return value, nil +} + +func managementWaitOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input waitInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + values := make([]string, 0, input.Count) + for len(values) < input.Count { + var value string + if err := ctx.WaitForSingleEvent("value", time.Minute).Await(&value); err != nil { + return nil, err + } + values = append(values, value) + } + return values, nil +} + +func managementTimerOrchestrator(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CreateTimer(time.Hour).Await(nil); err != nil { + return nil, err + } + return "timer fired", nil +} + +func waitForCustomStatus( + ctx context.Context, + client *durabletaskscheduler.Client, + id api.InstanceID, + want progressStatus, +) error { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + metadata, err := client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + if err == nil && metadata != nil { + var status progressStatus + if readErr := metadata.ReadCustomStatus(&status); readErr == nil && status == want { + return nil + } + } else if err != nil && !errors.Is(err, api.ErrInstanceNotFound) { + return err + } + select { + case <-ctx.Done(): + return fmt.Errorf("custom status %v was not observed for %s: %w", want, id, ctx.Err()) + case <-ticker.C: + } + } +} + +func collectCompletedInstanceIDs( + ctx context.Context, + client *durabletaskscheduler.Client, + maxPages int, +) (map[api.InstanceID]bool, error) { + listed := map[api.InstanceID]bool{} + token := "" + for range maxPages { + result, err := client.ListInstanceIDs(ctx, api.InstanceIDQuery{ + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + PageSize: 100, + ContinuationToken: token, + }) + if errors.Is(err, api.ErrFeatureNotSupported) { + return nil, fmt.Errorf("ListInstanceIDs is not supported by this target; run against a scheduler that implements ID listing") + } + if err != nil { + return nil, err + } + for _, id := range result.InstanceIDs { + listed[id] = true + } + token = result.ContinuationToken + if token == "" { + break + } + } + return listed, nil +} diff --git a/samples/observability/README.md b/samples/observability/README.md new file mode 100644 index 00000000..a80f5e9e --- /dev/null +++ b/samples/observability/README.md @@ -0,0 +1,28 @@ +# Observability hooks + +This sample uses the SDK's standard-library observability surfaces: tags, +immutable context fields, `slog` logging, and retry/history metric hooks. It +does not add a telemetry framework or an OpenTelemetry collector; DTS-owned +durable spans are covered elsewhere. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/observability +``` + +## What it proves + +- Root tags are visible in metadata and history. +- Immutable root context fields are persisted and visible in orchestration + context/history. Activity contexts receive fields carried by the activity work + item plus worker-local fields; current DTS emulator activity work items expose + the worker-local field used here. +- `ctx.Logger()` suppresses replayed orchestrator output while activity logging + keeps task identity fields. +- Retry and history metric hooks receive bounded, thread-safe signals without + depending on exact batch-turn counts. + +`SAMPLE_OK observability` is printed only after field values, log records, +metrics, orchestration output, and exact-ID cleanup are verified. diff --git a/samples/observability/observability.go b/samples/observability/observability.go new file mode 100644 index 00000000..3248663c --- /dev/null +++ b/samples/observability/observability.go @@ -0,0 +1,359 @@ +// Command observability demonstrates tags, immutable context fields, logs, and metric hooks. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "log/slog" + "slices" + "sync" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +var observabilityAttempts = &attemptCounter{counts: make(map[string]int)} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK observability") +} + +func run() (err error) { + logs := newLogCollector() + metrics := newMetricCollector() + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleObservability", observabilityWorkflow); err != nil { + return err + } + if err := registry.AddActivityN("SampleObservabilityActivity", observabilityActivity); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + app, err := dtssample.Start(ctx, registry, + durabletaskclient.WithTaskExecutorOptions( + task.WithLogger(slog.New(logs)), + task.WithContextFields(api.ContextFields{"worker": "sample-worker"}), + task.WithMetricsHooks(task.MetricsHooks{ + Retry: metrics.AddRetry, + History: metrics.AddHistory, + }), + ), + ) + if err != nil { + return err + } + id := dtssample.NewInstanceID("observability") + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, id), app.Shutdown()) }() + + input := observabilityInput{Key: string(id)} + tags := map[string]string{"sample": "observability", "scenario": "propagation"} + fields := api.ContextFields{"tenant": "alpha", "immutable": "root"} + if _, err := app.Client.ScheduleNewOrchestration(ctx, "SampleObservability", + api.WithInstanceID(id), api.WithInput(input), api.WithTags(tags), api.WithContextFields(fields)); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output observabilityOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.InstanceID != string(id) || output.OrchestrationName != "SampleObservability" || + output.OrchestrationFields["tenant"] != "alpha" || output.OrchestrationFields["immutable"] != "root" || + output.ActivityName != "SampleObservabilityActivity" || + output.ActivityFields["worker"] != "sample-worker" || output.Attempts != 2 { + return fmt.Errorf("unexpected observability output: %+v", output) + } + if _, leaked := output.OrchestrationFields["worker"]; leaked { + return errors.New("worker-local fields leaked into durable orchestration context") + } + + refetched, err := app.Client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if refetched.Tags["sample"] != "observability" || refetched.Tags["scenario"] != "propagation" { + return fmt.Errorf("metadata tags = %+v", refetched.Tags) + } + if refetched.ExecutionID == "" { + return errors.New("completed metadata did not include an execution ID") + } + if err := verifyHistoryFields(ctx, app.Client, id, refetched.ExecutionID, tags, fields); err != nil { + return err + } + if err := metrics.Verify(id); err != nil { + return err + } + if err := logs.Verify(id); err != nil { + return err + } + fmt.Println("verified tags, context fields, replay-safe logs, retry metrics, and history metrics") + return nil +} + +func observabilityWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input observabilityInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + ctx.Logger().Info("sample observability orchestrator log", "checkpoint", "before-activity") + info, _ := api.OrchestrationContextInfoFromContext(ctx.Context()) + var result observabilityOutput + if err := ctx.CallActivity("SampleObservabilityActivity", + task.WithActivityInput(input), + task.WithActivityTags(map[string]string{"activity-tag": "echo"}), + task.WithActivityRetryPolicy(&task.RetryPolicy{ + MaxAttempts: 3, + InitialRetryInterval: 100 * time.Millisecond, + BackoffCoefficient: 1, + MaxRetryInterval: 100 * time.Millisecond, + Handle: func(ctx task.RetryContext) bool { + return ctx.LastFailure.IsCausedBy("SampleObservabilityTransient") + }, + }), + ).Await(&result); err != nil { + return nil, err + } + result.InstanceID = string(info.InstanceID) + result.OrchestrationName = info.Name + result.OrchestrationFields = api.ContextFieldsFromContext(ctx.Context()) + return result, nil +} + +func observabilityActivity(ctx task.ActivityContext) (any, error) { + var input observabilityInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + attempt := observabilityAttempts.Increment(input.Key) + if attempt == 1 { + return nil, observabilityTransient{key: input.Key} + } + task.LoggerFromContext(ctx.Context()).Info("sample observability activity log", "attempt", attempt) + activity, _ := api.ActivityContextInfoFromContext(ctx.Context()) + return observabilityOutput{ + ActivityName: activity.Name, + ActivityFields: api.ContextFieldsFromContext(ctx.Context()), + Attempts: attempt, + }, nil +} + +type observabilityInput struct { + Key string `json:"key"` +} + +type observabilityOutput struct { + InstanceID string `json:"instanceId"` + OrchestrationName string `json:"orchestrationName"` + OrchestrationFields api.ContextFields `json:"orchestrationFields"` + ActivityName string `json:"activityName"` + ActivityFields api.ContextFields `json:"activityFields"` + Attempts int `json:"attempts"` +} + +type observabilityTransient struct { + key string +} + +func (e observabilityTransient) Error() string { + return "observability transient failure for " + e.key +} + +func (e observabilityTransient) DurableTaskErrorType() api.ErrorType { + return "SampleObservabilityTransient" +} + +type attemptCounter struct { + mu sync.Mutex + counts map[string]int +} + +func (c *attemptCounter) Increment(key string) int { + c.mu.Lock() + defer c.mu.Unlock() + c.counts[key]++ + return c.counts[key] +} + +type metricCollector struct { + mu sync.Mutex + retries []task.RetryMetric + history []task.HistoryMetric +} + +func newMetricCollector() *metricCollector { + return &metricCollector{} +} + +func (c *metricCollector) AddRetry(metric task.RetryMetric) { + c.mu.Lock() + defer c.mu.Unlock() + c.retries = append(c.retries, metric) +} + +func (c *metricCollector) AddHistory(metric task.HistoryMetric) { + c.mu.Lock() + defer c.mu.Unlock() + c.history = append(c.history, metric) +} + +func (c *metricCollector) Verify(id api.InstanceID) error { + c.mu.Lock() + defer c.mu.Unlock() + if !slices.ContainsFunc(c.retries, func(metric task.RetryMetric) bool { + return metric.InstanceID == id && + metric.TaskKind == task.WorkItemKindActivity && + metric.TaskName == "SampleObservabilityActivity" && + metric.FailedAttempt == 1 && + metric.NextAttempt == 2 && + metric.MaxAttempts == 3 + }) { + return fmt.Errorf("retry metrics missing expected retry: %+v", c.retries) + } + if !slices.ContainsFunc(c.history, func(metric task.HistoryMetric) bool { + return metric.InstanceID == id && + metric.OrchestrationName == "SampleObservability" && + metric.HistoryLength > 0 && + !metric.HistoryLimitExceeded + }) { + return fmt.Errorf("history metrics missing expected turn: %+v", c.history) + } + return nil +} + +type logCollector struct { + mu sync.Mutex + records []logRecord +} + +type logRecord struct { + message string + attrs map[string]any +} + +func newLogCollector() *logCollector { + return &logCollector{} +} + +func (h *logCollector) Enabled(context.Context, slog.Level) bool { + return true +} + +func (h *logCollector) Handle(_ context.Context, record slog.Record) error { + attrs := make(map[string]any) + record.Attrs(func(attr slog.Attr) bool { + attrs[attr.Key] = attr.Value.Any() + return true + }) + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, logRecord{message: record.Message, attrs: attrs}) + return nil +} + +func (h *logCollector) WithAttrs(attrs []slog.Attr) slog.Handler { + return &logCollectorWith{parent: h, attrs: attrs} +} + +func (h *logCollector) WithGroup(string) slog.Handler { + return h +} + +func (h *logCollector) Verify(id api.InstanceID) error { + h.mu.Lock() + defer h.mu.Unlock() + orchestratorLogs := 0 + activityLogs := 0 + for _, record := range h.records { + if record.message == "sample observability orchestrator log" && record.attrs["durabletask.instance_id"] == string(id) { + orchestratorLogs++ + } + if record.message == "sample observability activity log" && record.attrs["durabletask.activity.name"] == "SampleObservabilityActivity" { + activityLogs++ + } + } + if orchestratorLogs != 1 { + return fmt.Errorf("orchestrator replay-safe log count = %d, want 1; records=%+v", orchestratorLogs, h.records) + } + if activityLogs != 1 { + return fmt.Errorf("activity log count = %d, want 1; records=%+v", activityLogs, h.records) + } + return nil +} + +type logCollectorWith struct { + parent *logCollector + attrs []slog.Attr +} + +func (h *logCollectorWith) Enabled(ctx context.Context, level slog.Level) bool { + return h.parent.Enabled(ctx, level) +} + +func (h *logCollectorWith) Handle(ctx context.Context, record slog.Record) error { + for _, attr := range h.attrs { + record.AddAttrs(attr) + } + return h.parent.Handle(ctx, record) +} + +func (h *logCollectorWith) WithAttrs(attrs []slog.Attr) slog.Handler { + merged := make([]slog.Attr, 0, len(h.attrs)+len(attrs)) + merged = append(merged, h.attrs...) + merged = append(merged, attrs...) + return &logCollectorWith{parent: h.parent, attrs: merged} +} + +func (h *logCollectorWith) WithGroup(string) slog.Handler { + return h +} + +func verifyHistoryFields( + ctx context.Context, + client interface { + GetOrchestrationHistory(context.Context, api.InstanceID, api.HistoryQuery) (*api.OrchestrationHistory, error) + }, + id api.InstanceID, + executionID string, + tags map[string]string, + fields api.ContextFields, +) error { + history, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID, MaxEvents: 100, MaxBytes: 1 << 20}) + if err != nil { + return err + } + startIndex := slices.IndexFunc(history.Events, func(event *api.HistoryEvent) bool { + return event.Type == api.HistoryEventExecutionStarted + }) + if startIndex < 0 || history.Events[startIndex].ExecutionStarted == nil { + return errors.New("observability history missing ExecutionStarted") + } + started := history.Events[startIndex].ExecutionStarted + for key, value := range tags { + if started.Tags[key] != value { + return fmt.Errorf("ExecutionStarted tag %s = %q, want %q", key, started.Tags[key], value) + } + } + for key, value := range fields { + if started.ContextFields[key] != value { + return fmt.Errorf("ExecutionStarted context field %s = %q, want %q", key, started.ContextFields[key], value) + } + } + return nil +} diff --git a/samples/retries/README.md b/samples/retries/README.md new file mode 100644 index 00000000..7fdd9e6b --- /dev/null +++ b/samples/retries/README.md @@ -0,0 +1,26 @@ +# Retries and durable failures + +This sample demonstrates retry behavior with deterministic, activity-owned +failure injection. It does not use randomness, wall-clock time, or atomics inside +orchestrator or retry-handler code. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/retries +``` + +## What it proves + +- A transient activity failure is retried and recovers after the known first two + failures. +- The retry handler uses `task.RetryContext` and durable failure details to stop + retrying a typed permanent failure. +- A custom non-retriable error bypasses retry and preserves custom properties in + the failure chain. + +Successful paths use `api.WithFetchPayloads(true)` and `dtssample.RequireCompleted`. +Intentional failure paths assert `FAILED` status, concrete durable error types, +properties, and attempt counts. `SAMPLE_OK retries` appears only after all +assertions and exact-ID cleanup succeed. diff --git a/samples/retries/retries.go b/samples/retries/retries.go index 9c324005..79daa185 100644 --- a/samples/retries/retries.go +++ b/samples/retries/retries.go @@ -1,90 +1,320 @@ -// Command retries demonstrates activity retry policies: an activity fails -// randomly and the orchestration retries it with exponential backoff. -// -// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" -// go run ./samples/retries +// Command retries demonstrates deterministic activity retry policies against DTS. package main import ( "context" - "encoding/json" "errors" "fmt" "log" - "math/rand" + "sync" "time" + "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +const ( + transientFailureType api.ErrorType = "SampleTransientFailure" + handlerStopFailureType api.ErrorType = "SampleHandlerStoppedFailure" + nonRetriableType api.ErrorType = "SampleNonRetriableFailure" +) + +var attempts = &attemptStore{counts: make(map[string]int)} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK retries") } -func run() error { - // Create a new task registry and add the orchestrator and activities - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN("RetryActivityOrchestrator", RetryActivityOrchestrator); err != nil { - return fmt.Errorf("failed to register orchestrator: %w", err) +func run() (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleRetryRecovery", retryRecoveryOrchestrator); err != nil { + return err } - if err := r.AddActivityN("RandomFailActivity", RandomFailActivity); err != nil { - return fmt.Errorf("failed to register activity: %w", err) + if err := registry.AddOrchestratorN("SampleRetryHandlerStops", retryHandlerStopsOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN("SampleRetryNonRetriable", retryNonRetriableOrchestrator); err != nil { + return err + } + if err := registry.AddActivityN("SampleControlledFailure", controlledFailureActivity); err != nil { + return err } - // Connect a client and worker to the Durable Task Scheduler task hub - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() - app, err := dtssample.Start(ctx, r) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } - defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) - } - }() + recoveryID := dtssample.NewInstanceID("retries-recovery") + handlerID := dtssample.NewInstanceID("retries-handler") + nonRetriableID := dtssample.NewInstanceID("retries-non-retriable") + ids := []api.InstanceID{recoveryID, handlerID, nonRetriableID} + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ids...), app.Shutdown()) }() + + if err := verifyRecovery(ctx, app.Client, recoveryID); err != nil { + return err + } + if err := verifyHandlerStop(ctx, app.Client, handlerID); err != nil { + return err + } + if err := verifyNonRetriable(ctx, app.Client, nonRetriableID); err != nil { + return err + } + return nil +} - // Start a new orchestration - id, err := app.Client.ScheduleNewOrchestration(ctx, "RetryActivityOrchestrator") +func verifyRecovery(ctx context.Context, client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) +}, id api.InstanceID) error { + input := failurePlan{Key: string(id), FailuresBeforeSuccess: 2} + if _, err := client.ScheduleNewOrchestration(ctx, "SampleRetryRecovery", + api.WithInstanceID(id), api.WithInput(input)); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("failed to schedule new orchestration: %w", err) + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output retryResult + if err := metadata.ReadOutput(&output); err != nil { + return err } + if output.Key != input.Key || output.Attempts != 3 || output.Value != "recovered" { + return fmt.Errorf("retry recovery result = %+v, want key %q attempts 3 recovered", output, input.Key) + } + fmt.Printf("verified deterministic recovery after %d attempts\n", output.Attempts) + return nil +} - // Wait for the orchestration to complete - metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id) +func verifyHandlerStop(ctx context.Context, client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) +}, id api.InstanceID) error { + input := failurePlan{Key: string(id), Mode: "handler-stop"} + if _, err := client.ScheduleNewOrchestration(ctx, "SampleRetryHandlerStops", + api.WithInstanceID(id), api.WithInput(input)); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("failed to wait for orchestration to complete: %w", err) + return err + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_FAILED { + return fmt.Errorf("handler-stop status = %s, want FAILED", metadata.RuntimeStatus) } + if !failureChainContains(metadata.FailureDetails, handlerStopFailureType) { + return fmt.Errorf("handler-stop failure chain missing %s: %v", handlerStopFailureType, metadata.FailureDetails) + } + if got := attempts.Count(input.Key); got != 1 { + return fmt.Errorf("handler-stop attempts = %d, want 1", got) + } + fmt.Println("verified retry handler stops on a typed durable failure") + return nil +} - // Print the results - metadataEnc, err := json.MarshalIndent(metadata, "", " ") +func verifyNonRetriable(ctx context.Context, client interface { + ScheduleNewOrchestration(context.Context, string, ...api.NewOrchestrationOptions) (api.InstanceID, error) + WaitForOrchestrationCompletion(context.Context, api.InstanceID, ...api.FetchOrchestrationMetadataOptions) (*api.OrchestrationMetadata, error) +}, id api.InstanceID) error { + input := failurePlan{Key: string(id), Mode: "non-retriable"} + if _, err := client.ScheduleNewOrchestration(ctx, "SampleRetryNonRetriable", + api.WithInstanceID(id), api.WithInput(input)); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("failed to encode result to JSON: %w", err) + return err + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_FAILED { + return fmt.Errorf("non-retriable status = %s, want FAILED", metadata.RuntimeStatus) + } + details := findFailure(metadata.FailureDetails, nonRetriableType) + if details == nil { + return fmt.Errorf("non-retriable failure chain missing %s: %v", nonRetriableType, metadata.FailureDetails) + } + if !details.IsNonRetriable { + return fmt.Errorf("%s was not marked non-retriable", nonRetriableType) } - log.Printf("Orchestration completed: %v", string(metadataEnc)) + if details.Properties["category"] != "validation" || details.Properties["sample"] != "retries" { + return fmt.Errorf("unexpected custom failure properties: %+v", details.Properties) + } + if got := attempts.Count(input.Key); got != 1 { + return fmt.Errorf("non-retriable attempts = %d, want 1", got) + } + fmt.Println("verified non-retriable typed failure and custom properties") return nil } -func RetryActivityOrchestrator(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity("RandomFailActivity", task.WithActivityRetryPolicy(&task.RetryPolicy{ - MaxAttempts: 10, +func retryRecoveryOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input failurePlan + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output retryResult + if err := ctx.CallActivity("SampleControlledFailure", + task.WithActivityInput(input), + task.WithActivityRetryPolicy(sampleRetryPolicy()), + ).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func retryHandlerStopsOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input failurePlan + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return nil, ctx.CallActivity("SampleControlledFailure", + task.WithActivityInput(input), + task.WithActivityRetryPolicy(sampleRetryPolicy()), + ).Await(nil) +} + +func retryNonRetriableOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input failurePlan + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return nil, ctx.CallActivity("SampleControlledFailure", + task.WithActivityInput(input), + task.WithActivityRetryPolicy(sampleRetryPolicy()), + ).Await(nil) +} + +func sampleRetryPolicy() *task.RetryPolicy { + return &task.RetryPolicy{ + MaxAttempts: 5, InitialRetryInterval: 100 * time.Millisecond, - BackoffCoefficient: 2, - MaxRetryInterval: 3 * time.Second, - })).Await(nil); err != nil { + BackoffCoefficient: 1, + MaxRetryInterval: 100 * time.Millisecond, + RetryTimeout: 10 * time.Second, + Handle: func(ctx task.RetryContext) bool { + return ctx.LastFailure.IsCausedBy(transientFailureType) + }, + } +} + +type failurePlan struct { + Key string `json:"key"` + Mode string `json:"mode,omitempty"` + FailuresBeforeSuccess int `json:"failuresBeforeSuccess,omitempty"` +} + +type retryResult struct { + Key string `json:"key"` + Attempts int `json:"attempts"` + Value string `json:"value"` +} + +func controlledFailureActivity(ctx task.ActivityContext) (any, error) { + var input failurePlan + if err := ctx.GetInput(&input); err != nil { return nil, err } - return nil, nil + attempt := attempts.Increment(input.Key) + switch input.Mode { + case "handler-stop": + return nil, handlerStoppedError{key: input.Key} + case "non-retriable": + return nil, nonRetriableSampleError{key: input.Key} + default: + if attempt <= input.FailuresBeforeSuccess { + return nil, fmt.Errorf("controlled attempt %d: %w", attempt, transientSampleError{key: input.Key, attempt: attempt}) + } + return retryResult{Key: input.Key, Attempts: attempt, Value: "recovered"}, nil + } +} + +type attemptStore struct { + mu sync.Mutex + counts map[string]int +} + +func (s *attemptStore) Increment(key string) int { + s.mu.Lock() + defer s.mu.Unlock() + s.counts[key]++ + return s.counts[key] +} + +func (s *attemptStore) Count(key string) int { + s.mu.Lock() + defer s.mu.Unlock() + return s.counts[key] +} + +type transientSampleError struct { + key string + attempt int +} + +func (e transientSampleError) Error() string { + return fmt.Sprintf("transient failure for %s on attempt %d", e.key, e.attempt) } -func RandomFailActivity(ctx task.ActivityContext) (any, error) { - // 70% possibility for activity failure - if rand.Intn(100) <= 70 { - log.Println("random activity failure") - return "", errors.New("random activity failure") +func (e transientSampleError) DurableTaskErrorType() api.ErrorType { + return transientFailureType +} + +func (e transientSampleError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"key": e.key, "attempt": e.attempt} +} + +type handlerStoppedError struct { + key string +} + +func (e handlerStoppedError) Error() string { + return "handler-selected permanent failure for " + e.key +} + +func (e handlerStoppedError) DurableTaskErrorType() api.ErrorType { + return handlerStopFailureType +} + +func (e handlerStoppedError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"category": "handler-stop", "key": e.key} +} + +type nonRetriableSampleError struct { + key string +} + +func (e nonRetriableSampleError) Error() string { + return "non-retriable validation failure for " + e.key +} + +func (e nonRetriableSampleError) DurableTaskErrorType() api.ErrorType { + return nonRetriableType +} + +func (e nonRetriableSampleError) DurableTaskErrorProperties() map[string]any { + return map[string]any{"category": "validation", "sample": "retries", "key": e.key} +} + +func (e nonRetriableSampleError) NonRetriable() bool { + return true +} + +func failureChainContains(details *api.FailureDetails, errorType api.ErrorType) bool { + return findFailure(details, errorType) != nil +} + +func findFailure(details *api.FailureDetails, errorType api.ErrorType) *api.FailureDetails { + for current := details; current != nil; current = current.InnerFailure { + if current.ErrorType == errorType { + return current + } } - return "ok", nil + return nil } diff --git a/samples/scheduledtasks/README.md b/samples/scheduledtasks/README.md new file mode 100644 index 00000000..9820654f --- /dev/null +++ b/samples/scheduledtasks/README.md @@ -0,0 +1,44 @@ +# Scheduled tasks sample + +This sample demonstrates the Go scheduled-task APIs for Durable Task Scheduler: + +- registering system scheduled-task orchestrators/entities with `RegisterScheduledTasks` +- starting a worker with `WithScheduledTasks` +- creating a unique schedule ID +- observing real target orchestration executions +- `Get`, `List`, `Update`, `Pause`, `Resume`, and `Delete` +- propagated tags and context fields +- bounded retry configuration using deterministic activity failure injection + +## Prerequisites + +Set `DTS_CONNECTION_STRING` to a task hub you can use for samples: + +```bash +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +The target must support the Go scheduled-task hub protocol. The sample does not use Azure CLI, resource creation, fixed target instance IDs, or retry plus fixed instance ID combinations. + +## Run + +```bash +go run ./samples/scheduledtasks +``` + +## Expected result + +The program creates a schedule, waits for an initial target execution that succeeds after one scheduled retry, pauses and updates the schedule, verifies that the updated target does not run while paused, resumes it, observes an updated target execution, deletes the schedule, cleans up captured target instance IDs, and prints: + +```text +SAMPLE_OK scheduledtasks +``` + +only after validation and cleanup both succeed. + +Schedule context fields are read from the target orchestration's durable context, +not assumed to be present in the activity's worker-local Go context. + +## Cleanup + +The schedule ID is unique to the sample run. The sample deletes the schedule before worker shutdown, captures actual target orchestration IDs started by the scheduler, and purges only those owned target IDs with `dtssample.Cleanup`. diff --git a/samples/scheduledtasks/scheduledtasks.go b/samples/scheduledtasks/scheduledtasks.go new file mode 100644 index 00000000..7418286d --- /dev/null +++ b/samples/scheduledtasks/scheduledtasks.go @@ -0,0 +1,345 @@ +// Command scheduledtasks demonstrates recurring scheduled-task management and +// validation using the public Go scheduler surface. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + scheduledTargetName = "SampleScheduledTasksTarget" + scheduledActivityName = "SampleScheduledTasksAttempt" +) + +var scheduledAttempts sync.Map + +type scheduledInput struct { + Run string `json:"run"` + Phase string `json:"phase"` + FailFirst bool `json:"failFirst"` +} + +type scheduledOutput struct { + Run string `json:"run"` + Phase string `json:"phase"` + Attempt int32 `json:"attempt"` + ContextFields map[string]string `json:"contextFields"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK scheduledtasks") +} + +func run() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + createdFrom := time.Now().UTC().Add(-time.Second) + runID := string(dtssample.NewInstanceID("scheduledtasks")) + scheduleID := runID + "-schedule" + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(scheduledTargetName, scheduledTargetOrchestrator); err != nil { + return err + } + if err := registry.AddActivityN(scheduledActivityName, scheduledAttemptActivity); err != nil { + return err + } + if err := durabletaskscheduler.RegisterScheduledTasks(registry); err != nil { + return err + } + app, err := dtssample.Start(ctx, registry, durabletaskscheduler.WithScheduledTasks(), durabletaskclient.WithAutoWorkItemFilters()) + if err != nil { + return err + } + var targetIDs []api.InstanceID + var handle *durabletaskscheduler.ScheduleClient + defer func() { + var cleanupErr error + if handle != nil { + deleteCtx, stopDelete := context.WithTimeout(context.Background(), 15*time.Second) + cleanupErr = errors.Join(cleanupErr, handle.Delete(deleteCtx)) + stopDelete() + } + queryCtx, stopQuery := context.WithTimeout(context.Background(), 15*time.Second) + _, observed, queryErr := waitForObservedTargets(queryCtx, app.Client, createdFrom, runID) + targetIDs = appendMissingIDs(targetIDs, observed...) + stopQuery() + cleanupErr = errors.Join(cleanupErr, queryErr) + cleanupErr = errors.Join(cleanupErr, dtssample.Cleanup(app.Client, targetIDs...)) + err = errors.Join(err, cleanupErr, app.Shutdown()) + }() + + schedules := app.Client.ScheduledTasks() + handle, err = schedules.Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ + ScheduleID: scheduleID, + OrchestrationName: scheduledTargetName, + TypedOrchestrationInput: scheduledInput{Run: runID, Phase: "initial", FailFirst: true}, + Interval: time.Hour, + StartAt: time.Now().UTC().Add(-time.Second), + StartImmediatelyIfLate: true, + Tags: map[string]string{"sample": "scheduledtasks", "run": runID, "phase": "initial"}, + ContextFields: api.ContextFields{"tenant": "initial"}, + RetryPolicy: &durabletaskscheduler.ScheduleRetryPolicy{ + MaxAttempts: 2, + InitialRetryInterval: time.Second, + BackoffCoefficient: 1, + MaxRetryInterval: time.Second, + RetryTimeout: 15 * time.Second, + }, + }) + if err != nil { + return err + } + description, err := schedules.Get(ctx, scheduleID) + if err != nil { + return err + } + if description == nil || description.Status != durabletaskscheduler.ScheduleStatusActive { + return fmt.Errorf("created schedule description = %+v, want active", description) + } + if description.Tags["phase"] != "initial" || description.ContextFields["tenant"] != "initial" { + return fmt.Errorf("created schedule tags/context = %#v / %#v", description.Tags, description.ContextFields) + } + list, err := schedules.List(ctx, durabletaskscheduler.ScheduleQuery{ + ScheduleIDPrefix: scheduleID, + PageSize: 10, + }) + if err != nil { + return err + } + if len(list.Schedules) != 1 || list.Schedules[0].ScheduleID != scheduleID { + return fmt.Errorf("schedule list returned %+v, want %s", list, scheduleID) + } + + initial, observed, err := waitForScheduledCompletion(ctx, app.Client, createdFrom, runID, "initial") + targetIDs = appendMissingIDs(targetIDs, observed...) + if err != nil { + return err + } + var initialOutput scheduledOutput + if err := initial.ReadOutput(&initialOutput); err != nil { + return err + } + if initialOutput.Phase != "initial" || initialOutput.Attempt != 2 || + initialOutput.ContextFields["tenant"] != "initial" { + return fmt.Errorf("initial scheduled output = %#v", initialOutput) + } + + if err := handle.Pause(ctx); err != nil { + return err + } + paused, err := handle.Describe(ctx) + if err != nil { + return err + } + if paused.Status != durabletaskscheduler.ScheduleStatusPaused { + return fmt.Errorf("paused schedule status=%s, want paused", paused.Status) + } + updatedStart := time.Now().UTC().Add(-time.Second) + updatedInterval := 2 * time.Second + startImmediately := true + if err := handle.Update(ctx, durabletaskscheduler.ScheduleUpdateOptions{ + TypedOrchestrationInput: scheduledInput{Run: runID, Phase: "updated"}, + StartAt: &updatedStart, + Interval: &updatedInterval, + StartImmediatelyIfLate: &startImmediately, + Tags: map[string]string{"sample": "scheduledtasks", "run": runID, "phase": "updated"}, + ContextFields: api.ContextFields{"tenant": "updated"}, + }); err != nil { + return err + } + updated, err := handle.Describe(ctx) + if err != nil { + return err + } + var updatedInput scheduledInput + if err := updated.ReadInput(&updatedInput); err != nil { + return err + } + if updated.Status != durabletaskscheduler.ScheduleStatusPaused || + updatedInput.Phase != "updated" || + updated.Tags["phase"] != "updated" || + updated.ContextFields["tenant"] != "updated" { + return fmt.Errorf("updated paused schedule = %+v input=%#v", updated, updatedInput) + } + for until := time.Now().Add(2 * updatedInterval); time.Now().Before(until); { + observed, ids, err := waitForObservedTargets(ctx, app.Client, createdFrom, runID) + targetIDs = appendMissingIDs(targetIDs, ids...) + if err != nil { + return err + } + for _, target := range observed { + if target.Tags["phase"] == "updated" { + return errors.New("updated target started while its schedule was paused") + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } + if err := handle.Resume(ctx); err != nil { + return err + } + resumed, err := handle.Describe(ctx) + if err != nil { + return err + } + if resumed.Status != durabletaskscheduler.ScheduleStatusActive { + return fmt.Errorf("resumed schedule status=%s, want active", resumed.Status) + } + second, observed, err := waitForScheduledCompletion(ctx, app.Client, createdFrom, runID, "updated") + targetIDs = appendMissingIDs(targetIDs, observed...) + if err != nil { + return err + } + var secondOutput scheduledOutput + if err := second.ReadOutput(&secondOutput); err != nil { + return err + } + if secondOutput.Phase != "updated" || + secondOutput.ContextFields["tenant"] != "updated" || + secondOutput.Run != runID { + return fmt.Errorf("updated scheduled output = %#v", secondOutput) + } + + if err := handle.Delete(ctx); err != nil { + return err + } + handle = nil + deleted, err := schedules.Get(ctx, scheduleID) + if err != nil { + return err + } + if deleted != nil { + return fmt.Errorf("schedule %s still exists after delete: %+v", scheduleID, deleted) + } + _, observed, err = waitForObservedTargets(ctx, app.Client, createdFrom, runID) + targetIDs = appendMissingIDs(targetIDs, observed...) + return err +} + +func scheduledTargetOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input scheduledInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output scheduledOutput + if err := ctx.CallActivity(scheduledActivityName, task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + output.ContextFields = map[string]string(api.ContextFieldsFromContext(ctx.Context())) + return output, nil +} + +func scheduledAttemptActivity(ctx task.ActivityContext) (any, error) { + var input scheduledInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + key := input.Run + "/" + input.Phase + counterValue, _ := scheduledAttempts.LoadOrStore(key, new(atomic.Int32)) + attempt := counterValue.(*atomic.Int32).Add(1) + if input.FailFirst && attempt == 1 { + return nil, fmt.Errorf("deterministic first activity failure for %s", key) + } + return scheduledOutput{ + Run: input.Run, + Phase: input.Phase, + Attempt: attempt, + }, nil +} + +func waitForScheduledCompletion( + ctx context.Context, + client *durabletaskscheduler.Client, + createdFrom time.Time, + runID string, + phase string, +) (*api.OrchestrationMetadata, []api.InstanceID, error) { + deadline := time.Now().Add(45 * time.Second) + for { + observed, ids, err := waitForObservedTargets(ctx, client, createdFrom, runID) + if err != nil { + return nil, ids, err + } + for _, metadata := range observed { + if metadata.Name == scheduledTargetName && + metadata.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED && + metadata.Tags["phase"] == phase { + full, err := client.FetchOrchestrationMetadata(ctx, metadata.InstanceID, api.WithFetchPayloads(true)) + if err != nil { + return nil, ids, err + } + if full.SerializedOutput == "" { + return nil, ids, fmt.Errorf( + "completed scheduled target %s returned no output from FetchOrchestrationMetadata(..., api.WithFetchPayloads(true))", + metadata.InstanceID, + ) + } + return full, ids, nil + } + } + if time.Now().After(deadline) { + return nil, ids, fmt.Errorf("scheduled target phase %q did not complete; observed %d target(s)", phase, len(observed)) + } + select { + case <-ctx.Done(): + return nil, ids, ctx.Err() + case <-time.After(200 * time.Millisecond): + } + } +} + +func waitForObservedTargets( + ctx context.Context, + client *durabletaskscheduler.Client, + createdFrom time.Time, + runID string, +) ([]*api.OrchestrationMetadata, []api.InstanceID, error) { + query, err := client.QueryInstances(ctx, api.OrchestrationQuery{ + CreatedTimeFrom: createdFrom, + Tags: map[string]string{"sample": "scheduledtasks", "run": runID}, + PageSize: 100, + FetchInputsAndOutputs: true, + }) + if err != nil { + return nil, nil, err + } + ids := make([]api.InstanceID, 0, len(query.Orchestrations)) + targets := make([]*api.OrchestrationMetadata, 0, len(query.Orchestrations)) + for _, metadata := range query.Orchestrations { + if metadata.Name != scheduledTargetName { + continue + } + targets = append(targets, metadata) + ids = append(ids, metadata.InstanceID) + } + return targets, ids, nil +} + +func appendMissingIDs(base []api.InstanceID, additions ...api.InstanceID) []api.InstanceID { + for _, id := range additions { + if id != "" && !slices.Contains(base, id) { + base = append(base, id) + } + } + return base +} diff --git a/samples/suborchestrations/README.md b/samples/suborchestrations/README.md new file mode 100644 index 00000000..77c4c9c5 --- /dev/null +++ b/samples/suborchestrations/README.md @@ -0,0 +1,42 @@ +# Sub-orchestrations sample + +This sample demonstrates parent/child orchestration patterns in Durable Task Scheduler: + +- a parent orchestration scheduling multiple child orchestrations +- explicit child instance IDs derived from the owned parent ID +- combined child results returned to the parent +- user tags and explicit context fields propagated to child orchestration history +- child failure propagation to the parent +- recursive cleanup of the owned parent/child instance tree + +## Prerequisites + +Set `DTS_CONNECTION_STRING` to a task hub you can use for samples, for example a local emulator hub: + +```bash +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```bash +go run ./samples/suborchestrations +``` + +## Expected result + +The program starts one successful parent and one controlled-failure parent. It validates child outputs, parent and child histories, propagated context fields, and the expected failed-child history event. It prints: + +```text +SAMPLE_OK suborchestrations +``` + +only after validation and cleanup both succeed. + +The child reads its durable context fields from its orchestration context. +Values needed by an activity are passed explicitly as activity input; this sample +does not assume the activity's worker-local Go context inherits those fields. + +## Cleanup + +The sample tracks the parent and planned child instance IDs before scheduling. Before shutting down its worker, it calls `dtssample.Cleanup` with the owned IDs, using recursive purge for child orchestration state. It does not create schedules, task hubs, or other shared resources. diff --git a/samples/suborchestrations/suborchestrations.go b/samples/suborchestrations/suborchestrations.go new file mode 100644 index 00000000..4c7857f3 --- /dev/null +++ b/samples/suborchestrations/suborchestrations.go @@ -0,0 +1,293 @@ +// Command suborchestrations demonstrates parent/child orchestration composition +// with explicit child instance IDs, result aggregation, propagated tags/context, +// and child-failure propagation. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "maps" + "slices" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + subParentName = "SampleSubOrchestrationsParent" + subChildName = "SampleSubOrchestrationsChild" + subActivityName = "SampleSubOrchestrationsFormat" +) + +type subParentInput struct { + Children []string `json:"children"` + IncludeFailure bool `json:"includeFailure"` +} + +type subChildInput struct { + Name string `json:"name"` + Fail bool `json:"fail"` + Tenant string `json:"tenant,omitempty"` +} + +type subChildResult struct { + Name string `json:"name"` + Message string `json:"message"` + ContextFields map[string]string `json:"contextFields"` +} + +type subParentOutput struct { + CombinedResults []string `json:"combinedResults"` + ChildIDs []api.InstanceID `json:"childIds"` + ChildContexts []map[string]string `json:"childContexts"` + Children []subChildResult `json:"children"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK suborchestrations") +} + +func run() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN(subParentName, subParentOrchestrator); err != nil { + return err + } + if err := registry.AddOrchestratorN(subChildName, subChildOrchestrator); err != nil { + return err + } + if err := registry.AddActivityN(subActivityName, subFormatActivity); err != nil { + return err + } + + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + defer func() { + err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) + }() + + parentID := dtssample.NewInstanceID("suborchestrations-parent") + childIDs := plannedSubChildIDs(parentID, 2) + ownedIDs = append(ownedIDs, parentID) + ownedIDs = append(ownedIDs, childIDs...) + if err := verifySuccessfulParent(ctx, app.Client, parentID, childIDs); err != nil { + return err + } + + failureParentID := dtssample.NewInstanceID("suborchestrations-failure") + failureChildIDs := plannedSubChildIDs(failureParentID, 3) + ownedIDs = append(ownedIDs, failureParentID) + ownedIDs = append(ownedIDs, failureChildIDs...) + if err := verifyChildFailurePropagation(ctx, app.Client, failureParentID); err != nil { + return err + } + return nil +} + +func verifySuccessfulParent(ctx context.Context, client *durabletaskscheduler.Client, parentID api.InstanceID, childIDs []api.InstanceID) error { + _, err := client.ScheduleNewOrchestration( + ctx, + subParentName, + api.WithInstanceID(parentID), + api.WithInput(subParentInput{Children: []string{"alpha", "bravo"}}), + api.WithTags(map[string]string{"sample": "suborchestrations", "scenario": "success"}), + api.WithContextFields(api.ContextFields{"tenant": "sample", "request": "sub-orchestration"}), + ) + if err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, parentID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output subParentOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if !slices.Equal(output.CombinedResults, []string{"child alpha for sample", "child bravo for sample"}) { + return fmt.Errorf("combined child results = %#v", output.CombinedResults) + } + if !slices.Equal(output.ChildIDs, childIDs) { + return fmt.Errorf("child IDs = %#v, want %#v", output.ChildIDs, childIDs) + } + parentHistory, err := client.GetOrchestrationHistory(ctx, parentID, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + if err != nil { + return err + } + if countHistoryEvents(parentHistory, api.HistoryEventSubOrchestrationInstanceCreated) != len(childIDs) || + countHistoryEvents(parentHistory, api.HistoryEventSubOrchestrationInstanceCompleted) != len(childIDs) { + return fmt.Errorf("parent history does not contain expected child create/complete events") + } + for _, childID := range childIDs { + child, err := client.FetchOrchestrationMetadata(ctx, childID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(child); err != nil { + return err + } + history, err := client.GetOrchestrationHistory(ctx, childID, api.HistoryQuery{ExecutionID: child.ExecutionID}) + if err != nil { + return err + } + started := executionStarted(history) + if started == nil || started.Parent == nil || started.Parent.InstanceID != parentID { + return fmt.Errorf("child %s did not record parent %s in history", childID, parentID) + } + if started.ContextFields["tenant"] != "sample" || started.ContextFields["child-scope"] != "explicit" { + return fmt.Errorf("child %s context fields = %#v", childID, started.ContextFields) + } + } + fmt.Printf("verified parent %s and %d children\n", parentID, len(childIDs)) + return nil +} + +func verifyChildFailurePropagation(ctx context.Context, client *durabletaskscheduler.Client, parentID api.InstanceID) error { + _, err := client.ScheduleNewOrchestration( + ctx, + subParentName, + api.WithInstanceID(parentID), + api.WithInput(subParentInput{Children: []string{"ok-a", "ok-b"}, IncludeFailure: true}), + api.WithTags(map[string]string{"sample": "suborchestrations", "scenario": "failure"}), + ) + if err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, parentID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_FAILED { + return fmt.Errorf("failure propagation status = %s, want FAILED", metadata.RuntimeStatus) + } + if metadata.FailureDetails == nil || + !strings.Contains(metadata.FailureDetails.ErrorMessage, "child forced-error failed as requested") { + return fmt.Errorf("parent %s did not preserve the expected child failure: %v", parentID, metadata.FailureDetails) + } + history, err := client.GetOrchestrationHistory(ctx, parentID, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + if err != nil { + return err + } + if countHistoryEvents(history, api.HistoryEventSubOrchestrationInstanceFailed) != 1 { + return fmt.Errorf("parent history did not record exactly one failed child") + } + fmt.Printf("verified child failure propagation for %s\n", parentID) + return nil +} + +func subParentOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input subParentInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + childNames := append([]string(nil), input.Children...) + if input.IncludeFailure { + childNames = append(childNames, "forced-error") + } + output := subParentOutput{ + ChildIDs: make([]api.InstanceID, 0, len(childNames)), + ChildContexts: make([]map[string]string, 0, len(childNames)), + Children: make([]subChildResult, 0, len(childNames)), + } + parentFields := api.ContextFieldsFromContext(ctx.Context()) + for index, name := range childNames { + childID := api.InstanceID(fmt.Sprintf("%s-child-%02d", ctx.ID, index)) + output.ChildIDs = append(output.ChildIDs, childID) + childFields := api.ContextFields{"child-scope": "explicit"} + maps.Copy(childFields, parentFields) + var child subChildResult + err := ctx.CallSubOrchestrator( + subChildName, + task.WithSubOrchestrationInstanceID(string(childID)), + task.WithSubOrchestratorInput(subChildInput{Name: name, Fail: name == "forced-error"}), + task.WithSubOrchestrationTags(map[string]string{"sample": "suborchestrations", "child": name}), + task.WithSubOrchestrationContextFields(childFields), + ).Await(&child) + if err != nil { + return nil, err + } + output.Children = append(output.Children, child) + output.CombinedResults = append(output.CombinedResults, child.Message) + output.ChildContexts = append(output.ChildContexts, maps.Clone(child.ContextFields)) + } + return output, nil +} + +func subChildOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input subChildInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if input.Fail { + return nil, fmt.Errorf("child %s failed as requested", input.Name) + } + fields := api.ContextFieldsFromContext(ctx.Context()) + input.Tenant = fields["tenant"] + var result subChildResult + if err := ctx.CallActivity(subActivityName, task.WithActivityInput(input)).Await(&result); err != nil { + return nil, err + } + result.ContextFields = maps.Clone(fields) + return result, nil +} + +func subFormatActivity(ctx task.ActivityContext) (any, error) { + var input subChildInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return subChildResult{ + Name: input.Name, + Message: fmt.Sprintf("child %s for %s", input.Name, input.Tenant), + }, nil +} + +func plannedSubChildIDs(parentID api.InstanceID, count int) []api.InstanceID { + ids := make([]api.InstanceID, 0, count) + for index := range count { + ids = append(ids, api.InstanceID(fmt.Sprintf("%s-child-%02d", parentID, index))) + } + return ids +} + +func countHistoryEvents(history *api.OrchestrationHistory, eventType api.HistoryEventType) int { + count := 0 + if history == nil { + return count + } + for _, event := range history.Events { + if event.Type == eventType { + count++ + } + } + return count +} + +func executionStarted(history *api.OrchestrationHistory) *api.HistoryExecutionStartedEvent { + if history == nil { + return nil + } + for _, event := range history.Events { + if event.ExecutionStarted != nil { + return event.ExecutionStarted + } + } + return nil +} diff --git a/samples/versioning/README.md b/samples/versioning/README.md new file mode 100644 index 00000000..938ef179 --- /dev/null +++ b/samples/versioning/README.md @@ -0,0 +1,42 @@ +# Versioning sample + +This sample demonstrates Durable Task version-aware dispatch: + +- two worker registries for versions `1.0` and `2.0` +- client default version routing +- explicit orchestration name/version routing +- activity version inheritance from orchestrators +- controlled explicit unversioned fallback +- strict version mismatch as a bounded negative case +- `ContinueAsNew` migration from version `1.0` to `2.0` +- `CurrentOrOlder` routing where a version `2.0` worker accepts version `1.0` work + +Worker handling is verified from activity outputs and recorded orchestration/activity versions, not from orchestrator-side global state or I/O. + +## Prerequisites + +Set `DTS_CONNECTION_STRING` to a task hub you can use for samples: + +```bash +export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +``` + +## Run + +```bash +go run ./samples/versioning +``` + +## Expected result + +The program starts real versioned orchestrations, validates outputs and metadata for every routing scenario, asserts the strict mismatch failure, cleans up owned instances, and prints: + +```text +SAMPLE_OK versioning +``` + +only after validation and cleanup both succeed. + +## Cleanup + +Every scenario creates a unique owned instance ID with `dtssample.NewInstanceID`, records it before scheduling, stops all workers gracefully, and purges owned orchestration instances before shutting down the main app. diff --git a/samples/versioning/versioning.go b/samples/versioning/versioning.go new file mode 100644 index 00000000..d8a3058c --- /dev/null +++ b/samples/versioning/versioning.go @@ -0,0 +1,532 @@ +// Command versioning demonstrates version-aware orchestration and activity +// routing across two worker registries. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + versionGreetingName = "SampleVersioningGreeting" + versionActivityName = "SampleVersioningActivity" + versionMigrationName = "SampleVersioningMigration" + versionOlderName = "SampleVersioningOlderAccepted" + versionOlderActivityName = "SampleVersioningOlderActivity" + versionUnversionedName = "SampleVersioningUnversioned" + versionStrictOnlyName = "SampleVersioningStrictOnly" + unversionedActivityName = "SampleVersioningUnversionedActivity" +) + +type versionOutput struct { + Input string `json:"input"` + Worker string `json:"worker"` + OrchestrationVersion string `json:"orchestrationVersion"` + ActivityWorker string `json:"activityWorker"` + ActivityVersion string `json:"activityVersion"` + ActivityObservedFrom string `json:"activityObservedFrom"` +} + +type migrationState struct { + Steps []versionOutput `json:"steps"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK versioning") +} + +func run() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + baseOptions, err := dtssample.Options() + if err != nil { + return err + } + v1Options := cloneVersionOptions(baseOptions, &task.VersioningOptions{ + Version: "1.0", + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureFail, + }) + v1Registry := task.NewTaskRegistry() + if err := registerVersionOne(v1Registry); err != nil { + return err + } + app, err := dtssample.StartWithOptions( + ctx, + v1Options, + v1Registry, + durabletaskclient.WithWorkItemFilters(&durabletaskclient.WorkItemFilters{ + Orchestrations: []durabletaskclient.WorkItemFilter{ + {Name: versionGreetingName, Versions: []string{"1.0"}}, + {Name: versionMigrationName, Versions: []string{"1.0"}}, + {Name: versionStrictOnlyName, Versions: []string{"2.0"}}, + {Name: versionUnversionedName, Versions: []string{task.UnversionedTaskVersion}}, + }, + Activities: []durabletaskclient.WorkItemFilter{ + {Name: versionActivityName, Versions: []string{"1.0"}}, + {Name: unversionedActivityName, Versions: []string{task.UnversionedTaskVersion}}, + }, + RejectAllEntities: true, + }), + durabletaskclient.WithTaskExecutorOptions( + task.WithUnversionedOrchestratorNames(versionUnversionedName), + task.WithUnversionedActivityNames(unversionedActivityName), + ), + ) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + var extraWorkers []*durabletaskclient.TaskHubGrpcWorker + defer func() { + cleanupErr := dtssample.Cleanup(app.Client, ownedIDs...) + for _, worker := range extraWorkers { + cleanupErr = errors.Join(cleanupErr, shutdownWorker(worker)) + } + err = errors.Join(err, cleanupErr, app.Shutdown()) + }() + + v2Options := cloneVersionOptions(baseOptions, &task.VersioningOptions{ + Version: "2.0", + DefaultVersion: "2.0", + MatchStrategy: task.VersionMatchStrict, + FailureStrategy: task.VersionFailureFail, + }) + v2Registry := task.NewTaskRegistry() + if err := registerVersionTwo(v2Registry); err != nil { + return err + } + v2Worker, err := startVersionWorker(ctx, v2Options, v2Registry) + if err != nil { + return err + } + extraWorkers = append(extraWorkers, v2Worker) + + checks := []struct { + name string + run func(context.Context, *durabletaskscheduler.Client, *[]api.InstanceID) error + }{ + {"default-version-routing", verifyDefaultVersionRouting}, + {"explicit-version-routing", verifyExplicitVersionRouting}, + {"unversioned-fallback", verifyUnversionedFallback}, + {"strict-negative-routing", verifyStrictNegativeRouting}, + {"continue-as-new-migration", verifyContinueAsNewMigration}, + } + for _, check := range checks { + if err := check.run(ctx, app.Client, &ownedIDs); err != nil { + return fmt.Errorf("%s: %w", check.name, err) + } + fmt.Printf("verified %s\n", check.name) + } + + olderOptions := cloneVersionOptions(baseOptions, &task.VersioningOptions{ + Version: "2.0", + DefaultVersion: "2.0", + MatchStrategy: task.VersionMatchCurrentOrOlder, + FailureStrategy: task.VersionFailureFail, + }) + olderRegistry := task.NewTaskRegistry() + if err := olderRegistry.AddOrchestratorNVersion(versionOlderName, "1.0", versionedGreetingWithActivity("worker-v2-current-or-older", versionOlderActivityName)); err != nil { + return err + } + if err := olderRegistry.AddActivityNVersion(versionOlderActivityName, "1.0", versionedActivity("worker-v2-current-or-older")); err != nil { + return err + } + olderWorker, err := startVersionWorker(ctx, olderOptions, olderRegistry) + if err != nil { + return err + } + extraWorkers = append(extraWorkers, olderWorker) + if err := verifyCurrentOrOlderRouting(ctx, app.Client, &ownedIDs); err != nil { + return fmt.Errorf("current-or-older-routing: %w", err) + } + fmt.Println("verified current-or-older-routing") + return nil +} + +func registerVersionOne(registry *task.TaskRegistry) error { + if err := registry.AddOrchestratorNVersion(versionGreetingName, "1.0", versionedGreeting("worker-v1")); err != nil { + return err + } + if err := registry.AddActivityNVersion(versionActivityName, "1.0", versionedActivity("worker-v1")); err != nil { + return err + } + if err := registry.AddOrchestratorNVersion(versionMigrationName, "1.0", migrationV1); err != nil { + return err + } + if err := registry.AddOrchestratorN(versionUnversionedName, unversionedOrchestrator); err != nil { + return err + } + if err := registry.AddActivityN(unversionedActivityName, unversionedActivity); err != nil { + return err + } + return registry.AddOrchestratorNVersion(versionStrictOnlyName, "1.0", versionedGreeting("worker-v1")) +} + +func registerVersionTwo(registry *task.TaskRegistry) error { + if err := registry.AddOrchestratorNVersion(versionGreetingName, "2.0", versionedGreeting("worker-v2")); err != nil { + return err + } + if err := registry.AddActivityNVersion(versionActivityName, "2.0", versionedActivity("worker-v2")); err != nil { + return err + } + return registry.AddOrchestratorNVersion(versionMigrationName, "2.0", migrationV2) +} + +func verifyDefaultVersionRouting(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("versioning-default") + *ownedIDs = append(*ownedIDs, id) + output, metadata, err := runVersionGreeting(ctx, client, id, nil, "default") + if err != nil { + return err + } + if metadata.Version != "1.0" || output.Worker != "worker-v1" || output.ActivityVersion != "1.0" { + return fmt.Errorf("default routing output=%#v metadata version=%q", output, metadata.Version) + } + return nil +} + +func verifyExplicitVersionRouting(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + for _, test := range []struct { + version string + worker string + }{ + {"1.0", "worker-v1"}, + {"2.0", "worker-v2"}, + } { + id := dtssample.NewInstanceID("versioning-explicit-" + test.version) + *ownedIDs = append(*ownedIDs, id) + version := test.version + output, metadata, err := runVersionGreeting(ctx, client, id, func() api.NewOrchestrationOptions { + return api.WithVersion(version) + }, "explicit-"+test.version) + if err != nil { + return err + } + if metadata.Version != test.version || output.Worker != test.worker || output.ActivityVersion != test.version { + return fmt.Errorf("explicit %s routed to output=%#v metadata=%q", test.version, output, metadata.Version) + } + } + return nil +} + +func verifyUnversionedFallback(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("versioning-unversioned") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + versionUnversionedName, + api.WithInstanceID(id), + api.WithVersion(task.UnversionedTaskVersion), + api.WithInput("fallback"), + api.WithTags(map[string]string{"sample": "versioning", "scenario": "unversioned"}), + ); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output versionOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if metadata.Version != "" || output.Worker != "worker-v1-unversioned" || output.ActivityVersion != "" { + return fmt.Errorf("unversioned fallback output=%#v metadata=%q", output, metadata.Version) + } + return nil +} + +func verifyStrictNegativeRouting(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("versioning-strict-negative") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + versionStrictOnlyName, + api.WithInstanceID(id), + api.WithVersion("2.0"), + api.WithTags(map[string]string{"sample": "versioning", "scenario": "strict-negative"}), + ); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if metadata.RuntimeStatus != api.RUNTIME_STATUS_FAILED || + metadata.FailureDetails == nil || + !metadata.FailureDetails.Matches(api.ErrVersionMismatch) { + return fmt.Errorf("strict mismatch metadata=%+v", metadata) + } + return nil +} + +func verifyContinueAsNewMigration(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("versioning-migration") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + versionMigrationName, + api.WithInstanceID(id), + api.WithVersion("1.0"), + api.WithTags(map[string]string{"sample": "versioning", "scenario": "migration"}), + ); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + if metadata.Version != "2.0" { + return fmt.Errorf("migration completed as version %q, want 2.0", metadata.Version) + } + var output migrationState + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if len(output.Steps) != 2 || + output.Steps[0].Worker != "worker-v1" || + output.Steps[0].ActivityVersion != "1.0" || + output.Steps[1].Worker != "worker-v2" || + output.Steps[1].ActivityVersion != "2.0" { + return fmt.Errorf("migration output = %#v", output) + } + return nil +} + +func verifyCurrentOrOlderRouting(ctx context.Context, client *durabletaskscheduler.Client, ownedIDs *[]api.InstanceID) error { + id := dtssample.NewInstanceID("versioning-current-or-older") + *ownedIDs = append(*ownedIDs, id) + if _, err := client.ScheduleNewOrchestration( + ctx, + versionOlderName, + api.WithInstanceID(id), + api.WithVersion("1.0"), + api.WithInput("older"), + api.WithTags(map[string]string{"sample": "versioning", "scenario": "current-or-older"}), + ); err != nil { + return err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output versionOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if metadata.Version != "1.0" || output.Worker != "worker-v2-current-or-older" || output.ActivityVersion != "1.0" { + return fmt.Errorf("current-or-older output=%#v metadata=%q", output, metadata.Version) + } + return nil +} + +func runVersionGreeting( + ctx context.Context, + client *durabletaskscheduler.Client, + id api.InstanceID, + version func() api.NewOrchestrationOptions, + input string, +) (versionOutput, *api.OrchestrationMetadata, error) { + options := []api.NewOrchestrationOptions{ + api.WithInstanceID(id), + api.WithInput(input), + api.WithTags(map[string]string{"sample": "versioning"}), + } + if version != nil { + options = append(options, version()) + } + if _, err := client.ScheduleNewOrchestration(ctx, versionGreetingName, options...); err != nil { + return versionOutput{}, nil, err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return versionOutput{}, nil, err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return versionOutput{}, nil, err + } + var output versionOutput + if err := metadata.ReadOutput(&output); err != nil { + return versionOutput{}, nil, err + } + return output, metadata, nil +} + +func versionedGreeting(worker string) task.Orchestrator { + return versionedGreetingWithActivity(worker, versionActivityName) +} + +func versionedGreetingWithActivity(worker string, activityName string) task.Orchestrator { + return func(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var activity versionOutput + if err := ctx.CallActivity(activityName, task.WithActivityInput(input)).Await(&activity); err != nil { + return nil, err + } + return versionOutput{ + Input: input, + Worker: worker, + OrchestrationVersion: ctx.Version, + ActivityWorker: activity.ActivityWorker, + ActivityVersion: activity.ActivityVersion, + ActivityObservedFrom: activity.ActivityObservedFrom, + }, nil + } +} + +func versionedActivity(worker string) task.Activity { + return func(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + activityInfo, _ := api.ActivityContextInfoFromContext(ctx.Context()) + orchestrationInfo, _ := api.OrchestrationContextInfoFromContext(ctx.Context()) + return versionOutput{ + Input: input, + ActivityWorker: worker, + ActivityVersion: activityInfo.Version, + ActivityObservedFrom: orchestrationInfo.Version, + }, nil + } +} + +func migrationV1(ctx *task.OrchestrationContext) (any, error) { + var activity versionOutput + if err := ctx.CallActivity(versionActivityName, task.WithActivityInput("migration-v1")).Await(&activity); err != nil { + return nil, err + } + state := migrationState{Steps: []versionOutput{{ + Input: "migration-v1", + Worker: "worker-v1", + OrchestrationVersion: ctx.Version, + ActivityWorker: activity.ActivityWorker, + ActivityVersion: activity.ActivityVersion, + ActivityObservedFrom: activity.ActivityObservedFrom, + }}} + ctx.ContinueAsNew(state, task.WithContinueAsNewVersion("2.0")) + return nil, nil +} + +func migrationV2(ctx *task.OrchestrationContext) (any, error) { + var state migrationState + if err := ctx.GetInput(&state); err != nil { + return nil, err + } + var activity versionOutput + if err := ctx.CallActivity(versionActivityName, task.WithActivityInput("migration-v2")).Await(&activity); err != nil { + return nil, err + } + state.Steps = append(state.Steps, versionOutput{ + Input: "migration-v2", + Worker: "worker-v2", + OrchestrationVersion: ctx.Version, + ActivityWorker: activity.ActivityWorker, + ActivityVersion: activity.ActivityVersion, + ActivityObservedFrom: activity.ActivityObservedFrom, + }) + return state, nil +} + +func unversionedOrchestrator(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var activity versionOutput + if err := ctx.CallActivity( + unversionedActivityName, + task.WithActivityInput(input), + task.WithActivityVersion(task.UnversionedTaskVersion), + ).Await(&activity); err != nil { + return nil, err + } + return versionOutput{ + Input: input, + Worker: "worker-v1-unversioned", + OrchestrationVersion: ctx.Version, + ActivityWorker: activity.ActivityWorker, + ActivityVersion: activity.ActivityVersion, + ActivityObservedFrom: activity.ActivityObservedFrom, + }, nil +} + +func unversionedActivity(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + activityInfo, _ := api.ActivityContextInfoFromContext(ctx.Context()) + orchestrationInfo, _ := api.OrchestrationContextInfoFromContext(ctx.Context()) + return versionOutput{ + Input: input, + ActivityWorker: "worker-v1-unversioned", + ActivityVersion: activityInfo.Version, + ActivityObservedFrom: orchestrationInfo.Version, + }, nil +} + +func cloneVersionOptions( + base *durabletaskscheduler.Options, + versioning *task.VersioningOptions, +) *durabletaskscheduler.Options { + copy := *base + copy.Versioning = versioning + return © +} + +func startVersionWorker( + ctx context.Context, + options *durabletaskscheduler.Options, + registry *task.TaskRegistry, +) (*durabletaskclient.TaskHubGrpcWorker, error) { + worker, err := durabletaskscheduler.NewWorker( + options, + registry, + api.DefaultLogger(), + durabletaskclient.WithAutoWorkItemFilters(), + ) + if err != nil { + return nil, err + } + if err := worker.Start(ctx); err != nil { + return nil, err + } + return worker, nil +} + +func shutdownWorker(worker *durabletaskclient.TaskHubGrpcWorker) error { + if worker == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return worker.Shutdown(ctx) +} diff --git a/samples/worker/README.md b/samples/worker/README.md new file mode 100644 index 00000000..1b2f8c87 --- /dev/null +++ b/samples/worker/README.md @@ -0,0 +1,33 @@ +# Worker lifecycle + +This sample focuses on worker lifecycle behavior using the public DTS client and +worker APIs. It does not modify emulator/cloud networking or fake service +faults. + +## Run + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/worker +``` + +## What it proves + +- `Start` runs a worker in the background. +- Activity concurrency stays within the configured bound. +- `Shutdown` is started while an accepted activity is blocked; the sample proves + shutdown does not finish until that in-flight activity is released, then waits + for bounded graceful drain. +- A new owned worker can restart processing after a prior worker stops, and + the orchestration continuation after the drained activity completes after + restart. +- `Run` blocks until its context is canceled and exits cleanly. +- The compatibility listener on a management client demonstrates the borrowed + connection contract; new applications should prefer owned workers. + +Activities are at-least-once, so the drain scenario uses an idempotent output +and does not assert an exact delivery count. The recovery scenario is limited to +bounded worker process restart ordering. Controlled service restarts or +proxy fault injection are needed to validate network-fault recovery. `SAMPLE_OK +worker` appears only after all output assertions, shutdown checks, deadlines, +and exact-ID cleanup succeed. diff --git a/samples/worker/worker.go b/samples/worker/worker.go new file mode 100644 index 00000000..7ad926e6 --- /dev/null +++ b/samples/worker/worker.go @@ -0,0 +1,493 @@ +// Command worker demonstrates worker Start, Run, Shutdown, drain, restart, and limits. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "sync" + "sync/atomic" + "time" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +var activityGauge = &concurrencyGauge{} +var drainBarriers = newBarrierRegistry() + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK worker") +} + +func run() (err error) { + options, err := dtssample.Options() + if err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + client, err := durabletaskscheduler.NewClient(ctx, options, api.DefaultLogger()) + if err != nil { + return err + } + var ids []api.InstanceID + defer func() { err = errors.Join(err, dtssample.Cleanup(client, ids...), client.Close()) }() + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleWorkerEcho", workerEchoWorkflow); err != nil { + return err + } + if err := registry.AddOrchestratorN("SampleWorkerDelay", workerDelayWorkflow); err != nil { + return err + } + if err := registry.AddActivityN("SampleWorkerEchoActivity", workerEchoActivity); err != nil { + return err + } + if err := registry.AddActivityN("SampleWorkerDelayActivity", workerDelayActivity); err != nil { + return err + } + + concurrencyIDs := []api.InstanceID{ + dtssample.NewInstanceID("worker-concurrency-1"), + dtssample.NewInstanceID("worker-concurrency-2"), + dtssample.NewInstanceID("worker-concurrency-3"), + dtssample.NewInstanceID("worker-concurrency-4"), + } + ids = append(ids, concurrencyIDs...) + drainID := dtssample.NewInstanceID("worker-drain") + restartID := dtssample.NewInstanceID("worker-restart") + runID := dtssample.NewInstanceID("worker-run") + borrowedID := dtssample.NewInstanceID("worker-borrowed") + ids = append(ids, drainID, restartID, runID, borrowedID) + + if err := verifyStartConcurrencyAndDrain(ctx, options, client, registry, concurrencyIDs, drainID); err != nil { + return err + } + if err := verifyRestart(ctx, options, client, registry, restartID); err != nil { + return err + } + if err := verifyRun(ctx, options, client, registry, runID); err != nil { + return err + } + if err := verifyBorrowedConnectionListener(ctx, client, registry, borrowedID); err != nil { + return err + } + return nil +} + +func verifyStartConcurrencyAndDrain( + ctx context.Context, + options *durabletaskscheduler.Options, + client *durabletaskscheduler.Client, + registry *task.TaskRegistry, + concurrencyIDs []api.InstanceID, + drainID api.InstanceID, +) error { + activityGauge.Reset() + worker, err := newSampleWorker(options, registry, 2) + if err != nil { + return err + } + if err := worker.Start(ctx); err != nil { + return err + } + workerStopped := false + defer func() { + if !workerStopped { + shutdownWorker(worker) + } + }() + for _, id := range concurrencyIDs { + if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerDelay", + api.WithInstanceID(id), api.WithInput(workerDelayInput{Label: string(id), DelayMillis: 200})); err != nil { + return err + } + } + for _, id := range concurrencyIDs { + if err := waitForWorkerOutput(ctx, client, id, "delay:"+string(id)); err != nil { + return err + } + } + if maxInflight := activityGauge.Max(); maxInflight > 2 { + return fmt.Errorf("max in-flight activities = %d, want <= 2", maxInflight) + } + + barrier := drainBarriers.Add(string(drainID)) + defer drainBarriers.Remove(string(drainID)) + if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerDelay", + api.WithInstanceID(drainID), api.WithInput(workerDelayInput{ + Label: string(drainID), + BlockOnBarrier: true, + })); err != nil { + return err + } + if err := barrier.WaitStarted(ctx); err != nil { + return err + } + shutdownCtx, stopShutdown := context.WithTimeout(context.Background(), 10*time.Second) + defer stopShutdown() + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- worker.Shutdown(shutdownCtx) }() + select { + case err := <-shutdownDone: + if err != nil { + return fmt.Errorf("Shutdown completed with an error while activity was blocked: %w", err) + } + return errors.New("Shutdown completed before the accepted blocked activity was released") + case <-time.After(200 * time.Millisecond): + } + barrier.Release() + select { + case err := <-shutdownDone: + if err != nil { + return err + } + case <-shutdownCtx.Done(): + return fmt.Errorf("Shutdown did not drain the released activity before deadline: %w", shutdownCtx.Err()) + } + workerStopped = true + if worker.Running() { + return errors.New("worker still reports running after Shutdown") + } + + recovery, err := newSampleWorker(options, registry, 2) + if err != nil { + return err + } + if err := recovery.Start(ctx); err != nil { + return err + } + defer shutdownWorker(recovery) + if err := waitForWorkerOutput(ctx, client, drainID, "delay:"+string(drainID)); err != nil { + return fmt.Errorf("post-shutdown work did not survive worker restart: %w", err) + } + fmt.Printf("verified Start, bounded activity concurrency (max %d), in-flight drain, and restart continuation\n", activityGauge.Max()) + return nil +} + +func verifyRestart( + ctx context.Context, + options *durabletaskscheduler.Options, + client *durabletaskscheduler.Client, + registry *task.TaskRegistry, + id api.InstanceID, +) error { + worker, err := newSampleWorker(options, registry, 2) + if err != nil { + return err + } + if err := worker.Start(ctx); err != nil { + return err + } + defer shutdownWorker(worker) + if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerEcho", + api.WithInstanceID(id), api.WithInput("restart")); err != nil { + return err + } + if err := waitForWorkerOutput(ctx, client, id, "echo:restart"); err != nil { + return err + } + fmt.Println("verified a fresh worker restarts processing after shutdown") + return nil +} + +func verifyRun( + ctx context.Context, + options *durabletaskscheduler.Options, + client *durabletaskscheduler.Client, + registry *task.TaskRegistry, + id api.InstanceID, +) error { + worker, err := newSampleWorker(options, registry, 2) + if err != nil { + return err + } + runCtx, cancelRun := context.WithCancel(ctx) + done := make(chan error, 1) + go func() { done <- worker.Run(runCtx) }() + stopped := false + defer func() { + if !stopped { + cancelRun() + <-done + } + }() + if err := waitForWorkerRunning(ctx, worker); err != nil { + return err + } + if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerEcho", + api.WithInstanceID(id), api.WithInput("run")); err != nil { + return err + } + if err := waitForWorkerOutput(ctx, client, id, "echo:run"); err != nil { + return err + } + cancelRun() + select { + case err := <-done: + stopped = true + if err != nil { + return err + } + case <-time.After(5 * time.Second): + return errors.New("Run worker did not stop after its context was canceled") + } + fmt.Println("verified Run blocks and exits cleanly when its context is canceled") + return nil +} + +func verifyBorrowedConnectionListener( + ctx context.Context, + client *durabletaskscheduler.Client, + registry *task.TaskRegistry, + id api.InstanceID, +) error { + if err := client.StartWorkItemListener(ctx, registry, + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentActivityWorkItems(1)); err != nil { + return err + } + defer func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = client.StopWorkItemListener(shutdownCtx) + }() + if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerEcho", + api.WithInstanceID(id), api.WithInput("borrowed")); err != nil { + return err + } + if err := waitForWorkerOutput(ctx, client, id, "echo:borrowed"); err != nil { + return err + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := client.StopWorkItemListener(shutdownCtx); err != nil { + return err + } + fmt.Println("verified borrowed connection compatibility listener") + return nil +} + +func newSampleWorker( + options *durabletaskscheduler.Options, + registry *task.TaskRegistry, + maxActivities int, +) (*durabletaskclient.TaskHubGrpcWorker, error) { + return durabletaskscheduler.NewWorker(options, registry, api.DefaultLogger(), + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithMaxConcurrentActivityWorkItems(maxActivities), + ) +} + +func shutdownWorker(worker *durabletaskclient.TaskHubGrpcWorker) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = worker.Shutdown(ctx) +} + +func workerEchoWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output string + if err := ctx.CallActivity("SampleWorkerEchoActivity", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func workerDelayWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input workerDelayInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output string + if err := ctx.CallActivity("SampleWorkerDelayActivity", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func workerEchoActivity(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return "echo:" + input, nil +} + +func workerDelayActivity(ctx task.ActivityContext) (any, error) { + var input workerDelayInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + activityGauge.Enter() + defer activityGauge.Leave() + if input.BlockOnBarrier { + barrier := drainBarriers.Get(input.Label) + if barrier == nil { + return nil, fmt.Errorf("missing drain barrier for %s", input.Label) + } + barrier.MarkStarted() + if err := barrier.WaitReleased(ctx.Context()); err != nil { + return nil, err + } + } + select { + case <-ctx.Context().Done(): + return nil, ctx.Context().Err() + case <-time.After(time.Duration(input.DelayMillis) * time.Millisecond): + return "delay:" + input.Label, nil + } +} + +type workerDelayInput struct { + Label string `json:"label"` + DelayMillis int `json:"delayMillis"` + BlockOnBarrier bool `json:"blockOnBarrier,omitempty"` +} + +type concurrencyGauge struct { + current atomic.Int32 + max atomic.Int32 +} + +func (g *concurrencyGauge) Enter() { + current := g.current.Add(1) + for { + previous := g.max.Load() + if current <= previous || g.max.CompareAndSwap(previous, current) { + return + } + } +} + +func (g *concurrencyGauge) Leave() { + g.current.Add(-1) +} + +func (g *concurrencyGauge) Max() int32 { + return g.max.Load() +} + +func (g *concurrencyGauge) Reset() { + g.current.Store(0) + g.max.Store(0) +} + +type barrierRegistry struct { + mu sync.Mutex + barriers map[string]*activityBarrier +} + +func newBarrierRegistry() *barrierRegistry { + return &barrierRegistry{barriers: make(map[string]*activityBarrier)} +} + +func (r *barrierRegistry) Add(key string) *activityBarrier { + r.mu.Lock() + defer r.mu.Unlock() + barrier := newActivityBarrier() + r.barriers[key] = barrier + return barrier +} + +func (r *barrierRegistry) Get(key string) *activityBarrier { + r.mu.Lock() + defer r.mu.Unlock() + return r.barriers[key] +} + +func (r *barrierRegistry) Remove(key string) { + r.mu.Lock() + defer r.mu.Unlock() + delete(r.barriers, key) +} + +type activityBarrier struct { + startedOnce sync.Once + releaseOnce sync.Once + started chan struct{} + released chan struct{} +} + +func newActivityBarrier() *activityBarrier { + return &activityBarrier{ + started: make(chan struct{}), + released: make(chan struct{}), + } +} + +func (b *activityBarrier) MarkStarted() { + b.startedOnce.Do(func() { close(b.started) }) +} + +func (b *activityBarrier) Release() { + b.releaseOnce.Do(func() { close(b.released) }) +} + +func (b *activityBarrier) WaitStarted(ctx context.Context) error { + waitCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + select { + case <-b.started: + return nil + case <-waitCtx.Done(): + return fmt.Errorf("blocked activity was not accepted before deadline: %w", waitCtx.Err()) + } +} + +func (b *activityBarrier) WaitReleased(ctx context.Context) error { + select { + case <-b.released: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func waitForWorkerOutput(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, expected string) error { + waitCtx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + metadata, err := client.WaitForOrchestrationCompletion(waitCtx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output != expected { + return fmt.Errorf("%s output = %q, want %q", id, output, expected) + } + return nil +} + +func waitForWorkerRunning(ctx context.Context, worker *durabletaskclient.TaskHubGrpcWorker) error { + waitCtx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + ticker := time.NewTicker(25 * time.Millisecond) + defer ticker.Stop() + for { + if worker.Running() { + return nil + } + select { + case <-waitCtx.Done(): + return fmt.Errorf("worker did not report running: %w", waitCtx.Err()) + case <-ticker.C: + } + } +} From 884e457f9331a967b2f1e495f98cdcdca7871939 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Wed, 9 Sep 2026 19:43:03 -0700 Subject: [PATCH 08/26] add service integration samples and executable E2E catalogue Cover real blob storage, continuous history export, correlated OTLP traces, replay analysis, Azure identity, and isolated maintenance. Add a 22-family catalogue and a process-level runner with explicit service requirements and CI groups. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/samples-e2e.yml | 100 ++ CHANGELOG.md | 1 + README.md | 11 +- samples/README.md | 121 +++ samples/authentication/README.md | 37 + samples/authentication/authentication.go | 113 +++ samples/distributedtracing/README.md | 81 ++ .../distributedtracing/distributedtracing.go | 336 +++++-- .../distributedtracing_test.go | 32 + .../otel-collector-config.yaml | 17 + samples/exporthistory/README.md | 64 ++ samples/exporthistory/exporthistory.go | 852 +++++++++++++++--- samples/exporthistory/exporthistory_test.go | 87 ++ samples/largepayloads/README.md | 54 ++ samples/largepayloads/largepayloads.go | 332 +++++++ samples/largepayloads/largepayloads_test.go | 29 + samples/replayanalysis/README.md | 66 ++ samples/replayanalysis/replayanalysis.go | 109 +++ samples/replayanalysis/replayanalysis_test.go | 26 + samples/replayanalysis/testdata/bad/bad.go | 29 + samples/serviceoperations/README.md | 44 + .../serviceoperations/serviceoperations.go | 251 ++++++ .../serviceoperations_test.go | 27 + tests/samples/samples_test.go | 234 +++++ 24 files changed, 2862 insertions(+), 191 deletions(-) create mode 100644 .github/workflows/samples-e2e.yml create mode 100644 samples/README.md create mode 100644 samples/authentication/README.md create mode 100644 samples/authentication/authentication.go create mode 100644 samples/distributedtracing/README.md create mode 100644 samples/distributedtracing/distributedtracing_test.go create mode 100644 samples/distributedtracing/otel-collector-config.yaml create mode 100644 samples/exporthistory/README.md create mode 100644 samples/exporthistory/exporthistory_test.go create mode 100644 samples/largepayloads/README.md create mode 100644 samples/largepayloads/largepayloads.go create mode 100644 samples/largepayloads/largepayloads_test.go create mode 100644 samples/replayanalysis/README.md create mode 100644 samples/replayanalysis/replayanalysis.go create mode 100644 samples/replayanalysis/replayanalysis_test.go create mode 100644 samples/replayanalysis/testdata/bad/bad.go create mode 100644 samples/serviceoperations/README.md create mode 100644 samples/serviceoperations/serviceoperations.go create mode 100644 samples/serviceoperations/serviceoperations_test.go create mode 100644 tests/samples/samples_test.go diff --git a/.github/workflows/samples-e2e.yml b/.github/workflows/samples-e2e.yml new file mode 100644 index 00000000..2a7db11e --- /dev/null +++ b/.github/workflows/samples-e2e.yml @@ -0,0 +1,100 @@ +name: Sample E2E + +on: + pull_request: + paths: + - "samples/**" + - "tests/samples/**" + - "api/**" + - "task/**" + - "client/**" + - "durabletaskscheduler/**" + - "exporthistory/**" + - "payload/**" + - "internal/**" + - "cmd/orchestratorvet/**" + - "go.mod" + - "go.sum" + - ".github/workflows/samples-e2e.yml" + push: + branches: [main] + workflow_dispatch: + +permissions: + contents: read + +jobs: + samples: + runs-on: ubuntu-latest + timeout-minutes: 25 + strategy: + fail-fast: false + matrix: + group: [emulator, storage, telemetry] + env: + DTS_CONNECTION_STRING: "Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None" + DTS_SAMPLES_E2E: "1" + DTS_SAMPLES_GROUP: ${{ matrix.group }} + DTS_SAMPLES_RACE: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version: "1.25.x" + + - name: Start isolated DTS + run: | + docker run -d --name sample-dts \ + -p 127.0.0.1:8080:8080 -p 127.0.0.1:8082:8082 \ + -e DTS_TASK_HUB_NAMES=default \ + mcr.microsoft.com/dts/dts-emulator:latest@sha256:361323065a608d605f9d3ae56b854eb11f1c47fcc16845a37f948f03ca9c5fac + curl --fail --retry 30 --retry-delay 1 --retry-all-errors http://127.0.0.1:8082/ >/dev/null + + - name: Start isolated blob storage + if: matrix.group == 'storage' + shell: bash + run: | + key="$(openssl rand -base64 32)" + echo "::add-mask::$key" + export AZURITE_ACCOUNTS="sampleacct:$key" + docker run -d --name sample-azurite \ + -p 127.0.0.1:10000:10000 -e AZURITE_ACCOUNTS \ + mcr.microsoft.com/azure-storage/azurite:3.37.0 + connection="DefaultEndpointsProtocol=http;AccountName=sampleacct;AccountKey=$key;BlobEndpoint=http://127.0.0.1:10000/sampleacct;" + echo "::add-mask::$connection" + printf 'AZURE_STORAGE_CONNECTION_STRING=%s\nEXPORT_STORAGE_CONNECTION_STRING=%s\n' \ + "$connection" "$connection" >> "$GITHUB_ENV" + printf 'DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1\nDTS_SAMPLE_ISOLATED_TASKHUB=1\n' >> "$GITHUB_ENV" + timeout 60 bash -c 'until > "$GITHUB_ENV" + timeout 60 bash -c 'until /dev/null 2>&1; then docker logs sample-azurite; fi + if docker inspect sample-collector >/dev/null 2>&1; then docker logs sample-collector; fi + + - name: Stop owned services + if: always() + run: | + for service in sample-dts sample-azurite sample-collector; do + if docker inspect "$service" >/dev/null 2>&1; then docker rm -f "$service"; fi + done diff --git a/CHANGELOG.md b/CHANGELOG.md index 49084a56..db906722 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Added a runnable DTS feature-family sample catalogue with outcome assertions, owned-resource cleanup, and process-level E2E groups for DTS, blob storage, telemetry, Azure authentication, and isolated hub maintenance. - Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, distinct activity and sub-orchestration action trace contexts, and legacy entity-operation trace forwarding. - Added the top-level `durabletaskscheduler` transport package, a dedicated resilient gRPC worker, DTS emulator tests, and an environment-driven sample. - Added advanced management APIs for bounded instance queries/listing, restart, rewind, batch/filter purge polling, immediate termination, and task-hub lifecycle operations. diff --git a/README.md b/README.md index 49aa288f..73ccad00 100644 --- a/README.md +++ b/README.md @@ -136,6 +136,10 @@ For more information, read the [DTS transport guide and feature matrix](./durabl Each sample connects to the task hub in `DTS_CONNECTION_STRING`. Set that variable first. Then run `go run ./samples/`. +The [complete feature-to-sample catalogue](./samples/README.md) covers the SDK's +feature families and documents actual end-to-end execution groups. These are +useful starting points: + | Sample | Description | | --- | --- | | [durabletaskscheduler](./samples/durabletaskscheduler) | Connect to DTS and call activities in sequence. | @@ -146,10 +150,11 @@ Each sample connects to the task hub in `DTS_CONNECTION_STRING`. Set that variab | [exporthistory](./samples/exporthistory) | Export orchestration histories to Azure Blob Storage. | | [distributedtracing](./samples/distributedtracing) | Send trace data to an OpenTelemetry collector. | -Two samples need more steps: +Some samples need additional services: -- `distributedtracing` is a separate Go module. Run it with `cd samples/distributedtracing && go run .`. -- `exporthistory` also needs `EXPORT_STORAGE_CONNECTION_STRING`. It accepts the optional variable `EXPORT_CONTAINER`. +- `distributedtracing` is a separate Go module and uses a local OTLP collector. Run it with `cd samples/distributedtracing && go run .`. +- `exporthistory` and `largepayloads` need blob storage; see their READMEs for the storage connection variables. +- `authentication` requires a real Azure identity and DTS endpoint. Hub-wide administrative examples require a disposable hub and explicit acknowledgement. Most samples use the shared helper [`samples/internal/dtssample`](./samples/internal/dtssample). The helper reads `DTS_CONNECTION_STRING`, opens the client, and starts the worker. The `exporthistory` sample does not use the helper. That sample needs the client before it registers the export system tasks. diff --git a/samples/README.md b/samples/README.md new file mode 100644 index 00000000..c04efa2f --- /dev/null +++ b/samples/README.md @@ -0,0 +1,121 @@ +# Runnable DTS feature samples + +These examples use Durable Task Scheduler, not an embedded storage backend. +Each sample has its own README, a bounded run, observable outcome assertions, +and cleanup of its own resources. `SAMPLE_OK ` is printed only after +validation and cleanup succeed. A build, a skipped run, or an unsupported +operation is not evidence that a feature works end to end. + +## Feature coverage + +| Sample | Feature family | E2E group | +| --- | --- | --- | +| [durabletaskscheduler](durabletaskscheduler) | Orchestrations, activities, typed input/output, start/wait | emulator | +| [parallel](parallel) | Fan-out/fan-in, WhenAll/WhenAny, aggregation | emulator | +| [coroutines](coroutines) | Durable coroutines, wait groups, selection, cancellation scopes | emulator | +| [timers](timers) | Durable delays, scheduled starts, timer splitting, deterministic time/IDs | emulator | +| [externalevents](externalevents) | Typed and repeated events, timeouts, cross-instance events | emulator | +| [suborchestrations](suborchestrations) | Child workflows, results, IDs, metadata and failures | emulator | +| [retries](retries) | Controlled retries, retry handlers, typed/non-retriable failures | emulator | +| [continueasnew](continueasnew) | Checkpoints, event carryover, continuation and history/turn budgets | emulator | +| [management](management) | Progress, queries/paging, lifecycle, restart, ID reuse, ID-scoped purge | emulator | +| [scheduledtasks](scheduledtasks) | Actual scheduled execution and schedule lifecycle | emulator | +| [versioning](versioning) | Worker versions, inheritance, fallback, migration and routing | emulator | +| [entity](entity) | State, signals/calls, registration models, factories and locked transfers | emulator | +| [dataconverter](dataconverter) | Custom serialization and raw versus typed payloads | emulator | +| [largepayloads](largepayloads) | Blob externalization, hydration, compression and bounds | storage | +| [history](history) | Buffered/streaming history, execution identity, typed readers and limits | emulator | +| [observability](observability) | Tags/context, replay-safe logs and metric hooks | emulator | +| [worker](worker) | Concurrency, graceful drain, restart/recovery and connection ownership | emulator | +| [authentication](authentication) | Real Azure connection-string and programmatic identity authentication | azure | +| [distributedtracing](distributedtracing) | Application spans, caller propagation and actual OTLP receipt | telemetry | +| [exporthistory](exporthistory) | Export jobs and verification of downloaded JSON/JSONL objects | storage | +| [replayanalysis](replayanalysis) | Real analyzer diagnostics and safe runnable counterparts | emulator | +| [serviceoperations](serviceoperations) | Hub-wide filtered purge and empty-entity maintenance | admin | + +Protocol bookkeeping, legacy wire compatibility and unsupported sandbox worker +profiles are not separate runnable features. SDK regression tests cover the +internal contracts. The catalogue demonstrates user-facing behavior. + +## Run an example + +Go 1.25 or later is required. Point the normal examples at an isolated DTS emulator: + +```bash +export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' +go run ./samples/durabletaskscheduler +``` + +Run commands are relative to the repository root unless a sample README says +otherwise. The distributed-tracing sample remains a nested Go module: + +```bash +cd samples/distributedtracing +go run . +``` + +Examples use unique instance/entity/schedule/job IDs. Cleanup never searches a +shared hub for arbitrary data to delete. Activities may be delivered more than +once; failure injection is a teaching device, not a production idempotency store. + +## Execute the actual programs end to end + +The standard-library runner in `tests/samples` builds and runs each selected +entry point as a separate process. It verifies exit status and the sample's +validation receipt. It also checks that every runnable sample has a catalogue +entry and run instructions. + +```bash +DTS_SAMPLES_E2E=1 DTS_SAMPLES_GROUP=emulator \ + go test ./tests/samples -run 'TestSampleCatalogue|TestSamplesE2E' -count=1 -timeout=20m -v +``` + +Set `DTS_SAMPLES_RACE=1` to build the sample processes with the race detector. +The external-event sample receives piped input automatically. The replay-analysis +entry also runs the real analyzer against its safe and intentionally unsafe code. +There is no fake service or hand-built history substituting for these executions. + +| Group | Required configuration | +| --- | --- | +| `emulator` | `DTS_CONNECTION_STRING` pointing at isolated DTS | +| `storage` | Isolated DTS with `DTS_SAMPLE_ISOLATED_TASKHUB=1`, plus `AZURE_STORAGE_CONNECTION_STRING` (`EXPORT_STORAGE_CONNECTION_STRING` is also accepted); set `DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1` for loopback Azurite | +| `telemetry` | DTS, `OTEL_EXPORTER_OTLP_ENDPOINT`, and `OTEL_CAPTURE_FILE` written by the real collector | +| `azure` | `DTS_SAMPLES_AZURE_CONNECTION_STRING` and an authorized Azure Identity credential | +| `admin` | `DTS_SAMPLES_ADMIN_CONNECTION_STRING` for a **disposable `sample-*` hub** and `DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1` | +| `all` | All of the above; each process gets the appropriate DTS target | + +For example, run the same authentication executable against a real Azure hub: + +```bash +export DTS_SAMPLES_AZURE_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' +DTS_SAMPLES_E2E=1 DTS_SAMPLES_GROUP=azure \ + go test ./tests/samples -run TestSamplesE2E -count=1 -timeout=5m -v +``` + +When E2E is explicitly enabled, missing required configuration fails the selected +group instead of silently skipping it. Ordinary `go test ./...` leaves these +external-service runs disabled; that default skip does not count as E2E coverage. + +The `Sample E2E` workflow runs emulator, storage and telemetry groups on real +isolated services. It generates temporary Azurite credentials rather than +committing account keys. Cloud groups are deliberately separate: use an approved +identity and target, not credentials exposed to untrusted pull requests. + +## Capability and safety boundaries + +Rewind, skip-graceful termination and SDK task-hub lifecycle RPCs are intentionally +not supported. Provision and remove task hubs through Azure's control plane or +CLI. Filtered purge and instance listing have documented emulator limitations; +consult the [SDK feature matrix](../durabletaskscheduler/README.md#feature-matrix). + +The administrative sample verifies state changes, not just acknowledgements. +An unsupported API, a success response that leaves the expected state unchanged, +or a missing precondition remains failed/blocked coverage. Do not turn such +results into a success or a skipped test just to make a dashboard green. + +Hub-wide purge and entity maintenance are destructive. +The `admin` group must use a newly provisioned disposable hub, never the hub +used for ordinary shared examples. Remove that temporary control-plane resource +afterward through Azure's control plane. Storage/export examples must +use isolated destinations and task hubs so they cannot export or delete unrelated +application data. diff --git a/samples/authentication/README.md b/samples/authentication/README.md new file mode 100644 index 00000000..38817e11 --- /dev/null +++ b/samples/authentication/README.md @@ -0,0 +1,37 @@ +# Azure authentication + +This sample runs and verifies an orchestration and activity twice: once with the +connection string's identity configuration and once with a programmatically +supplied `DefaultAzureCredential`. Both the management client and worker must +successfully perform real data-plane work. An emulator cannot validate this sample. + +## Run + +Use an existing Azure DTS task hub and an identity authorized to run workers, +start/query orchestrations, and terminate/purge the sample's instances. +For local development, sign in with `az login` and select the correct tenant. + +```bash +export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' +go run ./samples/authentication +``` + +For managed or workload identity, run on the corresponding Azure host and +configure the standard `AZURE_*` identity environment variables. The programmatic +half always uses the DefaultAzureCredential chain; configuring a different +connection-string mode does not change that chain. See the +[DTS authentication reference](../../durabletaskscheduler/README.md#configuration) +for the supported credential modes and their settings. + +No account keys or access tokens belong in source code or logs. + +## Expected result and cleanup + +Both modes report their completed, uniquely named instance and the process ends +with `SAMPLE_OK authentication`. It checks the exact activity result, not merely +whether a connection or Hello request succeeded. + +The sample terminates/purges only its own instance IDs and descendants, then +stops its workers and closes its clients. It never resets or deletes the task hub. +Missing credentials, missing data-plane permissions, incorrect results, and +cleanup failures produce a nonzero exit. diff --git a/samples/authentication/authentication.go b/samples/authentication/authentication.go new file mode 100644 index 00000000..6631a212 --- /dev/null +++ b/samples/authentication/authentication.go @@ -0,0 +1,113 @@ +// Command authentication runs real DTS work using connection-string and +// programmatic Azure Identity credentials. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net/url" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK authentication") +} + +func run() error { + options, err := dtssample.Options() + if err != nil { + return err + } + endpoint, err := url.Parse(options.EndpointAddress) + if err != nil { + return err + } + if endpoint.Scheme != "https" || options.Authentication == durabletaskscheduler.AuthenticationNone { + return errors.New("authentication requires an HTTPS DTS endpoint and Azure Identity; an emulator run does not validate authentication") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + if err := runAuthenticated(ctx, options, "connection-string"); err != nil { + return err + } + + credential, err := azidentity.NewDefaultAzureCredential(&azidentity.DefaultAzureCredentialOptions{ + TenantID: options.TenantID, + }) + if err != nil { + return err + } + programmatic := durabletaskscheduler.NewOptionsWithCredential( + options.EndpointAddress, options.TaskHubName, credential, + ) + programmatic.ResourceID = options.ResourceID + return runAuthenticated(ctx, programmatic, "programmatic-credential") +} + +func runAuthenticated(ctx context.Context, options *durabletaskscheduler.Options, mode string) (err error) { + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleAuthentication", authenticatedWorkflow); err != nil { + return err + } + if err := registry.AddActivityN("SampleAuthenticationEcho", authenticatedEcho); err != nil { + return err + } + app, err := dtssample.StartWithOptions(ctx, options, registry) + if err != nil { + return fmt.Errorf("%s connection: %w", mode, err) + } + defer func() { err = errors.Join(err, app.Shutdown()) }() + id := dtssample.NewInstanceID("authentication") + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, id)) }() + if _, err := app.Client.ScheduleNewOrchestration(ctx, "SampleAuthentication", + api.WithInstanceID(id), api.WithInput(mode)); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output != "authenticated:"+mode { + return fmt.Errorf("%s output=%q, want authenticated:%s", mode, output, mode) + } + fmt.Printf("%s: authenticated client and worker completed %s\n", mode, id) + return nil +} + +func authenticatedWorkflow(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output string + if err := ctx.CallActivity("SampleAuthenticationEcho", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func authenticatedEcho(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return "authenticated:" + input, nil +} diff --git a/samples/distributedtracing/README.md b/samples/distributedtracing/README.md new file mode 100644 index 00000000..b1761cb4 --- /dev/null +++ b/samples/distributedtracing/README.md @@ -0,0 +1,81 @@ +# Distributed tracing sample + +Demonstrates application trace propagation into Durable Task Scheduler using +OTLP/HTTP and a real OpenTelemetry collector. + +## Features + +- Starts a caller span and schedules an orchestration under that trace. +- Uses a local loopback HTTP target instead of an external dependency. +- Exports application spans over OTLP/HTTP to a collector. +- Verifies the collector file exporter received the caller and local HTTP spans + under the same trace ID. +- Reads DTS history and verifies persisted trace context shares the caller trace. + +DTS orchestration/activity/timer spans are service-owned. This sample validates +the trace context recorded in history and does not claim that the local worker +duplicates service spans. + +## Prerequisites + +- `DTS_CONNECTION_STRING` for DTS. +- A real OpenTelemetry Collector reachable through `OTEL_EXPORTER_OTLP_ENDPOINT` + or `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`. +- `OTEL_CAPTURE_FILE`, mounted/readable by both the collector and this process. + +Collector config for local/CI runs is provided in +`otel-collector-config.yaml`. Use +`otel/opentelemetry-collector-contrib:0.160.0`, which includes the `file` +exporter. The trace file may live outside the repository, but the same host file +must be mounted into the collector and exposed to the sample through +`OTEL_CAPTURE_FILE`. + +Collector contract: inside the collector container, set +`OTEL_CAPTURE_FILE=/output/traces.json`. In the sample process, set +`OTEL_CAPTURE_FILE` to the corresponding host path. The sample reads that host +file back and requires each expected span name to belong to the current trace ID, +so stale data from earlier runs is ignored. + +## Run + +Start a collector in one shell: + +```bash +cd samples/distributedtracing +export HOST_OTEL_CAPTURE_FILE="${HOST_OTEL_CAPTURE_FILE:-$PWD/otel-traces.json}" +docker run --rm \ + -p 4317:4317 -p 4318:4318 \ + -v "$PWD/otel-collector-config.yaml:/etc/otelcol/config.yaml:ro" \ + -v "$(dirname "$HOST_OTEL_CAPTURE_FILE"):/output" \ + -e OTEL_CAPTURE_FILE="/output/$(basename "$HOST_OTEL_CAPTURE_FILE")" \ + otel/opentelemetry-collector-contrib:0.160.0 \ + --config=/etc/otelcol/config.yaml +``` + +Run the sample from another shell: + +```bash +cd samples/distributedtracing +export OTEL_CAPTURE_FILE="${HOST_OTEL_CAPTURE_FILE:-$PWD/otel-traces.json}" +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" +go run . +``` + +## Expected outcome + +The process exits 0 and prints: + +```text +SAMPLE_OK distributedtracing +``` + +## Cleanup + +The orchestration is recursively terminated/purged before worker shutdown. The +sample does not create cloud telemetry resources. + +## Required validation variants + +Integration validation should start the real OTLP collector with this config and +execute `go run .`. Do not clear an open collector file; unique trace IDs +disambiguate repeated runs. diff --git a/samples/distributedtracing/distributedtracing.go b/samples/distributedtracing/distributedtracing.go index 8322b6a0..b8598036 100644 --- a/samples/distributedtracing/distributedtracing.go +++ b/samples/distributedtracing/distributedtracing.go @@ -1,186 +1,348 @@ -// Command distributedtracing starts an application caller span, propagates its -// W3C trace context into Durable Task Scheduler, and exports application-process -// spans to a local OpenTelemetry collector over OTLP/HTTP. DTS emits orchestration, -// activity, and timer telemetry service-side. -// -// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" -// cd samples/distributedtracing && go run . +// Command distributedtracing propagates an application caller span into DTS and +// verifies that a real OTLP/HTTP collector received application spans. package main import ( "context" "encoding/json" + "errors" "fmt" + "io" "log" "net/http" + "net/http/httptest" + "os" + "strings" "time" "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/propagation" "go.opentelemetry.io/otel/sdk/resource" sdktrace "go.opentelemetry.io/otel/sdk/trace" oteltrace "go.opentelemetry.io/otel/trace" + "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +const callerSpanName = "schedule_distributed_trace_sample" + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK distributedtracing") } -func run() error { - // Tracing can be configured independently of the orchestration code. - tp, err := ConfigureOTLPTracing(context.Background()) +func run() (err error) { + traceFile := os.Getenv("OTEL_CAPTURE_FILE") + if traceFile == "" { + return errors.New("OTEL_CAPTURE_FILE is required so the sample can verify collector receipt") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + tp, err := ConfigureOTLPTracing(ctx) if err != nil { - return fmt.Errorf("failed to create tracer: %w", err) + return fmt.Errorf("create tracer: %w", err) } - defer func() { - if err := tp.Shutdown(context.Background()); err != nil { - log.Printf("Failed to stop tracer: %v", err) - } - }() + defer func() { err = errors.Join(err, shutdownTracer(tp)) }() + tracedTarget := httptest.NewServer(otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = io.WriteString(w, "ok") + }), "local_trace_target", otelhttp.WithSpanNameFormatter(func(string, *http.Request) string { + return "local_trace_target" + }))) + defer tracedTarget.Close() - // Create a new task registry and add the orchestrator and activities - r := task.NewTaskRegistry() - if err := r.AddOrchestratorN("DistributedTraceSampleOrchestrator", DistributedTraceSampleOrchestrator); err != nil { - return fmt.Errorf("failed to register orchestrator: %w", err) + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("DistributedTraceSampleOrchestrator", DistributedTraceSampleOrchestrator); err != nil { + return fmt.Errorf("register orchestrator: %w", err) } - if err := r.AddActivityN("DoWorkActivity", DoWorkActivity); err != nil { - return fmt.Errorf("failed to register activity: %w", err) + if err := registry.AddActivityN("DoWorkActivity", DoWorkActivity); err != nil { + return fmt.Errorf("register activity: %w", err) } - if err := r.AddActivityN("CallHttpEndpointActivity", CallHttpEndpointActivity); err != nil { - return fmt.Errorf("failed to register activity: %w", err) + if err := registry.AddActivityN("CallHttpEndpointActivity", CallHttpEndpointActivity); err != nil { + return fmt.Errorf("register activity: %w", err) } - // Connect a client and worker to the Durable Task Scheduler task hub - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer cancel() - app, err := dtssample.Start(ctx, r) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } + var ownedIDs []api.InstanceID + cleanupDone := false defer func() { - if err := app.Shutdown(); err != nil { - log.Printf("Failed to shut down: %v", err) + if !cleanupDone { + err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) } }() - // A sampled caller span is propagated to DTS when the orchestration is - // scheduled, allowing service-side spans to join the application's trace. callerCtx, callerSpan := otel.Tracer("durabletask-sample").Start( ctx, - "schedule_distributed_trace_sample", + callerSpanName, oteltrace.WithSpanKind(oteltrace.SpanKindClient), ) - defer callerSpan.End() - id, err := app.Client.ScheduleNewOrchestration(callerCtx, "DistributedTraceSampleOrchestrator") - if err != nil { - return fmt.Errorf("failed to schedule new orchestration: %w", err) + traceID := callerSpan.SpanContext().TraceID().String() + instanceID := dtssample.NewInstanceID("distributedtracing") + ownedIDs = append(ownedIDs, instanceID) + if _, err := app.Client.ScheduleNewOrchestration(callerCtx, "DistributedTraceSampleOrchestrator", + api.WithInstanceID(instanceID), + api.WithInput(tracedTarget.URL), + api.WithTags(map[string]string{"sample": "distributedtracing"})); err != nil { + callerSpan.End() + return fmt.Errorf("schedule orchestration: %w", err) } + callerSpan.End() - // Wait for the orchestration to complete - metadata, err := app.Client.WaitForOrchestrationCompletion(callerCtx, id) + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) if err != nil { - return fmt.Errorf("failed to wait for orchestration to complete: %w", err) + return fmt.Errorf("wait for orchestration: %w", err) + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + history, err := app.Client.GetOrchestrationHistory(ctx, instanceID, api.HistoryQuery{ExecutionID: metadata.ExecutionID}) + if err != nil { + return fmt.Errorf("read orchestration history: %w", err) + } + if err := requireHistoryTraceContext(history.Events, traceID); err != nil { + return err } - - // Print the results metadataEnc, err := json.MarshalIndent(metadata, "", " ") if err != nil { - return fmt.Errorf("failed to encode result to JSON: %w", err) + return fmt.Errorf("encode result to JSON: %w", err) } - log.Printf("Orchestration completed: %v", string(metadataEnc)) - return nil + log.Printf("Orchestration completed: %s", metadataEnc) + + cleanupErr := dtssample.Cleanup(app.Client, ownedIDs...) + appErr := app.Shutdown() + traceErr := shutdownTracer(tp) + cleanupDone = true + receiptCtx, receiptCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer receiptCancel() + receiptErr := waitForCollectorReceipt(receiptCtx, traceFile, traceID, []string{callerSpanName, "local_trace_target"}) + return errors.Join(cleanupErr, appErr, traceErr, receiptErr) } func ConfigureOTLPTracing(ctx context.Context) (*sdktrace.TracerProvider, error) { - exp, err := otlptracehttp.New( - ctx, - otlptracehttp.WithEndpoint("localhost:4318"), - otlptracehttp.WithInsecure(), - ) + options := []otlptracehttp.Option{otlptracehttp.WithTimeout(10 * time.Second)} + if os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") == "" && os.Getenv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT") == "" { + options = append(options, otlptracehttp.WithEndpointURL("http://localhost:4318/v1/traces")) + } + exp, err := otlptracehttp.New(ctx, options...) if err != nil { return nil, err } - - // NOTE: The simple span processor is not recommended for production. - // Instead, the batch span processor should be used for production. - processor := sdktrace.NewSimpleSpanProcessor(exp) - // processor := sdktrace.NewBatchSpanProcessor(exp) - tp := sdktrace.NewTracerProvider( - sdktrace.WithSpanProcessor(processor), + sdktrace.WithSpanProcessor(sdktrace.NewBatchSpanProcessor(exp)), sdktrace.WithSampler(sdktrace.AlwaysSample()), sdktrace.WithResource(resource.NewWithAttributes( "durabletask.io", - attribute.KeyValue{Key: "service.name", Value: attribute.StringValue("sample-app")}, + attribute.String("service.name", "distributedtracing-sample"), )), ) otel.SetTracerProvider(tp) + otel.SetTextMapPropagator(propagation.TraceContext{}) return tp, nil } -// DistributedTraceSampleOrchestrator is a simple orchestration that's intended to generate -// distributed trace output to the configured exporter. +func shutdownTracer(tp *sdktrace.TracerProvider) error { + if tp == nil { + return nil + } + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + return tp.Shutdown(ctx) +} + func DistributedTraceSampleOrchestrator(ctx *task.OrchestrationContext) (any, error) { - if err := ctx.CallActivity("DoWorkActivity", task.WithActivityInput(1*time.Second)).Await(nil); err != nil { + var targetURL string + if err := ctx.GetInput(&targetURL); err != nil { return nil, err } - if err := ctx.CreateTimer(2 * time.Second).Await(nil); err != nil { + if err := ctx.CallActivity("DoWorkActivity", task.WithActivityInput(100*time.Millisecond)).Await(nil); err != nil { return nil, err } - if err := ctx.CallActivity("CallHttpEndpointActivity", task.WithActivityInput("https://bing.com")).Await(nil); err != nil { + if err := ctx.CreateTimer(100 * time.Millisecond).Await(nil); err != nil { return nil, err } - return nil, nil + if err := ctx.CallActivity("CallHttpEndpointActivity", task.WithActivityInput(targetURL)).Await(nil); err != nil { + return nil, err + } + return "trace-complete", nil } -// DoWorkActivity is a no-op activity function that sleeps for a specified amount of time. func DoWorkActivity(ctx task.ActivityContext) (any, error) { var duration time.Duration if err := ctx.GetInput(&duration); err != nil { - return "", err + return nil, err } - - // Simulate doing work select { case <-time.After(duration): - // Ok case <-ctx.Context().Done(): return nil, ctx.Context().Err() } - - return nil, nil + return "worked", nil } func CallHttpEndpointActivity(ctx task.ActivityContext) (any, error) { - var url string - if err := ctx.GetInput(&url); err != nil { - return "", err + var targetURL string + if err := ctx.GetInput(&targetURL); err != nil { + return nil, err } + if err := callLocalTarget(ctx.Context(), targetURL); err != nil { + return nil, err + } + return "called", nil +} - // The OTel HTTP client records the outbound request in the worker process. - // DTS owns the service-side activity span. - req, err := http.NewRequestWithContext(ctx.Context(), http.MethodGet, url, nil) +func callLocalTarget(ctx context.Context, targetURL string) (err error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) if err != nil { - return nil, err + return err } - httpClient := &http.Client{ - Transport: otelhttp.NewTransport(http.DefaultTransport), + client := &http.Client{Transport: otelhttp.NewTransport(http.DefaultTransport)} + resp, err := client.Do(req) + if err != nil { + return err + } + defer func() { err = errors.Join(err, resp.Body.Close()) }() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("local target returned %s", resp.Status) } - resp, err := httpClient.Do(req) + body, err := io.ReadAll(resp.Body) if err != nil { - return nil, err + return err } - defer func() { - if err := resp.Body.Close(); err != nil { - log.Printf("Failed to close HTTP response body: %v", err) + if string(body) != "ok" { + return fmt.Errorf("local target body=%q, want ok", body) + } + return nil +} + +func requireHistoryTraceContext(events []*api.HistoryEvent, traceID string) error { + if traceID == "" { + return errors.New("caller trace ID is empty") + } + var sawStarted, sawActivity bool + for _, event := range events { + if event == nil { + continue } - }() - return nil, nil + if event.ExecutionStarted != nil && event.ExecutionStarted.ParentTraceContext != nil { + sawStarted = sawStarted || strings.Contains(event.ExecutionStarted.ParentTraceContext.TraceParent, traceID) + } + if event.TaskScheduled != nil && event.TaskScheduled.Name == "CallHttpEndpointActivity" && + event.TaskScheduled.ParentTraceContext != nil { + sawActivity = sawActivity || strings.Contains(event.TaskScheduled.ParentTraceContext.TraceParent, traceID) + } + } + if !sawStarted || !sawActivity { + return fmt.Errorf("history trace context missing caller trace: started=%v activity=%v", sawStarted, sawActivity) + } + return nil +} + +func waitForCollectorReceipt(ctx context.Context, path string, traceID string, spanNames []string) error { + var lastErr error + var missing []string + for { + body, err := os.ReadFile(path) + if err == nil { + missing, lastErr = missingSpansForTrace(string(body), traceID, spanNames) + if lastErr == nil && len(missing) == 0 { + return nil + } + } else { + lastErr = err + } + select { + case <-ctx.Done(): + if lastErr != nil { + return fmt.Errorf("collector trace file %s could not be verified: %w", path, lastErr) + } + return fmt.Errorf("collector trace file %s did not contain trace %s spans %v; missing %v: %w", + path, traceID, spanNames, missing, ctx.Err()) + case <-time.After(500 * time.Millisecond): + } + } +} + +func traceFileContains(body, traceID string, spanNames []string) bool { + missing, err := missingSpansForTrace(body, traceID, spanNames) + return err == nil && len(missing) == 0 +} + +func missingSpansForTrace(body, traceID string, spanNames []string) ([]string, error) { + spans, err := traceSpansFromCollectorFile(body) + if err != nil { + return spanNames, err + } + found := map[string]struct{}{} + for _, span := range spans { + if strings.EqualFold(span.traceID, traceID) { + found[span.name] = struct{}{} + } + } + var missing []string + for _, name := range spanNames { + if _, ok := found[name]; !ok { + missing = append(missing, name) + } + } + return missing, nil +} + +type collectedSpan struct { + traceID string + name string +} + +func traceSpansFromCollectorFile(body string) ([]collectedSpan, error) { + decoder := json.NewDecoder(strings.NewReader(body)) + var spans []collectedSpan + for { + var value any + err := decoder.Decode(&value) + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, err + } + collectSpans(value, &spans) + } + return spans, nil +} + +func collectSpans(value any, spans *[]collectedSpan) { + switch typed := value.(type) { + case map[string]any: + traceID := stringField(typed, "traceId", "traceID", "trace_id") + name := stringField(typed, "name") + if traceID != "" && name != "" { + *spans = append(*spans, collectedSpan{traceID: traceID, name: name}) + } + for _, child := range typed { + collectSpans(child, spans) + } + case []any: + for _, child := range typed { + collectSpans(child, spans) + } + } +} + +func stringField(fields map[string]any, names ...string) string { + for _, name := range names { + value, ok := fields[name].(string) + if ok { + return value + } + } + return "" } diff --git a/samples/distributedtracing/distributedtracing_test.go b/samples/distributedtracing/distributedtracing_test.go new file mode 100644 index 00000000..1af65dde --- /dev/null +++ b/samples/distributedtracing/distributedtracing_test.go @@ -0,0 +1,32 @@ +package main + +import "testing" + +func TestTraceFileContains(t *testing.T) { + body := `{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"abc123","name":"schedule_distributed_trace_sample"},{"traceId":"abc123","name":"local_trace_target"}]}]}]}` + if !traceFileContains(body, "abc123", []string{callerSpanName, "local_trace_target"}) { + t.Fatal("expected trace file body to contain required trace and spans") + } + if traceFileContains(body, "missing", []string{callerSpanName}) { + t.Fatal("unexpected trace match") + } + if traceFileContains(body, "abc123", []string{"missing"}) { + t.Fatal("unexpected span match") + } +} + +func TestTraceFileContainsRejectsStaleSpansFromOtherTraces(t *testing.T) { + body := ` +{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"current-trace","name":"schedule_distributed_trace_sample"}]}]}]} +{"resourceSpans":[{"scopeSpans":[{"spans":[{"traceId":"stale-trace","name":"local_trace_target"}]}]}]} +` + if traceFileContains(body, "current-trace", []string{callerSpanName, "local_trace_target"}) { + t.Fatal("expected spans split across traces to fail") + } +} + +func TestTraceSpansFromCollectorFileRejectsInvalidJSON(t *testing.T) { + if _, err := traceSpansFromCollectorFile(`{"traceId":"abc"`); err == nil { + t.Fatal("expected invalid collector JSON to fail") + } +} diff --git a/samples/distributedtracing/otel-collector-config.yaml b/samples/distributedtracing/otel-collector-config.yaml new file mode 100644 index 00000000..45f3c7b6 --- /dev/null +++ b/samples/distributedtracing/otel-collector-config.yaml @@ -0,0 +1,17 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + grpc: + endpoint: 0.0.0.0:4317 + +exporters: + file: + path: ${env:OTEL_CAPTURE_FILE} + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [file] diff --git a/samples/exporthistory/README.md b/samples/exporthistory/README.md new file mode 100644 index 00000000..d1ecd36c --- /dev/null +++ b/samples/exporthistory/README.md @@ -0,0 +1,64 @@ +# Export history sample + +Runs real source orchestrations, creates Durable Task Scheduler export-history +jobs, downloads the emitted Azure Blob objects, and verifies that each object +contains the expected source execution and durable events. + +## Features + +- Batch and continuous export jobs. +- Gzip-compressed JSONL and plain JSON history formats. +- `CreateJob`, `GetJob`, `ListJobs`, `Describe`, `Delete`, and batch recreate. +- Continuous scenarios complete an additional source orchestration after the + initial export and verify the same active job exports it on the next service + poll. +- Bounded wait for `ListInstanceIDs`; an index omission is a hard failure. +- Prefix-scoped blob cleanup and retained-generation cleanup. + +## Prerequisites + +- `DTS_CONNECTION_STRING` for an isolated hub. +- `DTS_SAMPLE_ISOLATED_TASKHUB=1`, explicitly acknowledging that this hub is + isolated. A name prefix alone is not proof that a hub is safe to export. +- `AZURE_STORAGE_CONNECTION_STRING`; `EXPORT_STORAGE_CONNECTION_STRING` remains + supported for compatibility. +- `DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1` for loopback Azurite HTTP endpoints. + +The export window cannot filter by instance ID prefix, so do not run this +sample against a shared cloud hub. + +## Run + +```bash +# from the repository root +go run ./samples/exporthistory -scenario=batch-jsonl +go run ./samples/exporthistory -scenario=batch-json +go run ./samples/exporthistory -scenario=continuous-jsonl +go run ./samples/exporthistory -scenario=continuous-json +``` + +`-scenario=all` runs all advertised variants and may take several minutes. The +two continuous variants intentionally wait for the service's default +approximately one-minute continuous-export poll after a late source completion, +so use a 6-8 minute budget for the all-variants command. + +## Expected outcome + +Each command exits 0 and the last line is: + +```text +SAMPLE_OK exporthistory +``` + +## Cleanup + +The sample deletes each export job, purges retained batch export generations, +recursively purges source orchestrations before worker shutdown, and deletes its +owned container. If `EXPORT_CONTAINER` is provided, only +run-owned `samples/exporthistory//` prefixes are deleted and the +container remains. + +## Required validation variants + +Integration validation should run the four individual commands above after DTS +and Azurite are ready. diff --git a/samples/exporthistory/exporthistory.go b/samples/exporthistory/exporthistory.go index 54717bf9..2c899414 100644 --- a/samples/exporthistory/exporthistory.go +++ b/samples/exporthistory/exporthistory.go @@ -1,187 +1,837 @@ -// Command exporthistory demonstrates the preview export history package: it -// runs a few orchestrations, then creates a batch export job that writes their -// durable histories to Azure Blob Storage as gzip-compressed JSONL. -// -// Configure it with a Durable Task Scheduler connection string and an Azure -// Storage connection string: -// -// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" -// export EXPORT_STORAGE_CONNECTION_STRING="UseDevelopmentStorage=true" -// export EXPORT_CONTAINER="history-exports" -// go run ./samples/exporthistory +// Command exporthistory exercises the public exporthistory package against +// known source orchestrations and verifies the downloaded Azure Blob output. package main import ( + "bytes" + "compress/gzip" "context" + "crypto/rand" + "encoding/hex" + "encoding/json" "errors" + "flag" "fmt" + "io" "log" + "net" + "net/url" "os" "strings" "time" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/microsoft/durabletask-go/api" durabletaskclient "github.com/microsoft/durabletask-go/client" "github.com/microsoft/durabletask-go/durabletaskscheduler" "github.com/microsoft/durabletask-go/exporthistory" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" "github.com/microsoft/durabletask-go/task" ) +type scenario struct { + name string + mode exporthistory.ExportMode + format exporthistory.ExportFormatKind + recreate bool +} + +type sourceInput struct { + Scenario string `json:"scenario"` + Index int `json:"index"` +} + +type sourceOutput struct { + Value string `json:"value"` +} + +type sourceExecution struct { + id api.InstanceID + executionID string + output string +} + +type sampleApp struct { + client *durabletaskscheduler.Client + worker *durabletaskclient.TaskHubGrpcWorker +} + +type storageSettings struct { + connectionString string + container string + deleteContainer bool + allowInsecureHTTP bool +} + func main() { if err := run(); err != nil { log.Fatal(err) } + fmt.Println("SAMPLE_OK exporthistory") } -func run() error { - connectionString := os.Getenv("DTS_CONNECTION_STRING") - if connectionString == "" { - return errors.New("DTS_CONNECTION_STRING is required") +func run() (err error) { + scenarioName := flag.String("scenario", "all", "scenario to run: all, batch-jsonl, batch-json, continuous-jsonl, continuous-json") + flag.Parse() + scenarios, err := selectScenarios(*scenarioName) + if err != nil { + return err } - storageConnectionString := os.Getenv("EXPORT_STORAGE_CONNECTION_STRING") - if storageConnectionString == "" { - return errors.New("EXPORT_STORAGE_CONNECTION_STRING is required") + settings, err := readStorageSettings() + if err != nil { + return err } - container := os.Getenv("EXPORT_CONTAINER") - if container == "" { - container = "history-exports" + blobClient, err := azblob.NewClientFromConnectionString(settings.connectionString, nil) + if err != nil { + return fmt.Errorf("create Azure Blob reader: %w", err) } - - options, err := durabletaskscheduler.NewOptionsFromConnectionString(connectionString) + store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ + ConnectionString: settings.connectionString, + ContainerName: settings.container, + AllowInsecureHTTP: settings.allowInsecureHTTP, + }) if err != nil { - return fmt.Errorf("invalid DTS connection string: %w", err) + return fmt.Errorf("create Azure Blob history store: %w", err) + } + var ownedPrefixes []string + cleanupStorage := func() error { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if settings.deleteContainer { + _, cleanupErr := blobClient.DeleteContainer(cleanupCtx, settings.container, nil) + if bloberror.HasCode(cleanupErr, bloberror.ContainerNotFound) { + return nil + } + return cleanupErr + } + var failures []error + for _, prefix := range ownedPrefixes { + failures = append(failures, deleteBlobsWithPrefix(cleanupCtx, blobClient, settings.container, prefix)) + } + return errors.Join(failures...) } - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + options, err := dtssample.Options() + if err != nil { + return err + } + if err := requireIsolatedTaskHub(options); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Minute) defer cancel() - logger := api.DefaultLogger() + app, err := startExportHistoryApp(ctx, options, store) + if err != nil { + err = errors.Join(err, cleanupStorage()) + return err + } + var ownedIDs []api.InstanceID + var retainedExportRuns []api.InstanceID + defer func() { + err = errors.Join( + err, + dtssample.Cleanup(app.client, retainedExportRuns...), + dtssample.Cleanup(app.client, ownedIDs...), + cleanupStorage(), + app.shutdown(), + ) + }() - // The export activities read orchestration metadata and history through the - // same task hub client the application uses. - client, err := durabletaskscheduler.NewClient(ctx, options, logger) + for _, scenario := range scenarios { + result, scenarioErr := runScenario(ctx, app.client, blobClient, settings.container, scenario) + if scenarioErr != nil { + return scenarioErr + } + ownedIDs = append(ownedIDs, result.sourceIDs...) + retainedExportRuns = append(retainedExportRuns, result.retainedRuns...) + ownedPrefixes = append(ownedPrefixes, result.prefix) + fmt.Printf("verified %s with %d exported histories\n", scenario.name, result.exported) + } + return nil +} + +type scenarioResult struct { + sourceIDs []api.InstanceID + retainedRuns []api.InstanceID + prefix string + exported int +} + +func runScenario( + ctx context.Context, + client *durabletaskscheduler.Client, + blobClient *azblob.Client, + container string, + scenario scenario, +) (result scenarioResult, err error) { + var sources []sourceExecution + var retainedRuns []api.InstanceID + var prefix string + defer func() { + if err != nil { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + err = errors.Join( + err, + dtssample.Cleanup(client, sourceIDs(sources)...), + dtssample.Cleanup(client, retainedRuns...), + deleteBlobsWithPrefix(cleanupCtx, blobClient, container, prefix), + ) + } + }() + from := time.Now().UTC().Add(-time.Second) + sources, err = runKnownSources(ctx, client, scenario.name, 3) + if err != nil { + return scenarioResult{}, err + } + if err := waitForListableInstances(ctx, client, from, sourceIDs(sources), 90*time.Second); err != nil { + return scenarioResult{}, err + } + format := exporthistory.ExportFormat{Kind: scenario.format} + suffix, err := compactID() + if err != nil { + return scenarioResult{}, err + } + jobID := strings.ReplaceAll("sample-"+scenario.name+"-"+suffix, "_", "-") + prefix = "samples/exporthistory/" + jobID + "/" + exportClient, err := exporthistory.NewClient(client.TaskHubGrpcClient, exporthistory.ClientOptions{ + ContainerName: container, + Prefix: prefix, + }) if err != nil { - return fmt.Errorf("failed to create the Durable Task Scheduler client: %w", err) + return scenarioResult{}, err } + job, err := exportClient.JobClient(jobID) + if err != nil { + return scenarioResult{}, err + } + deleteJob := true defer func() { - if closeErr := client.Close(); closeErr != nil { - log.Printf("failed to close the client: %v", closeErr) + if deleteJob { + cleanupCtx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + if cleanupErr := job.Delete(cleanupCtx); cleanupErr != nil { + err = errors.Join(err, fmt.Errorf("cleanup export job %s: %w", jobID, cleanupErr)) + } } }() + if _, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ + JobID: jobID, + Mode: scenario.mode, + CompletedTimeFrom: from, + CompletedTimeTo: completedTimeTo(scenario.mode), + Format: &format, + MaxInstancesPerBatch: 2, + }); err != nil { + return scenarioResult{}, err + } - store, err := exporthistory.NewAzureBlobHistoryStore(exporthistory.AzureBlobHistoryStoreOptions{ - ConnectionString: storageConnectionString, - ContainerName: container, - // Azurite serves plaintext HTTP on loopback; production endpoints are HTTPS. - AllowInsecureHTTP: strings.Contains(storageConnectionString, "http://"), - }) + description, err := waitForExport(ctx, job, scenario.mode, len(sources)) + if err != nil { + return scenarioResult{}, err + } + if err := verifyJobLifecycle(ctx, exportClient, jobID, description, scenario.mode); err != nil { + return scenarioResult{}, err + } + exported, err := waitForDownloadedHistories(ctx, blobClient, container, prefix, scenario.format, sources) if err != nil { - return fmt.Errorf("failed to create the export store: %w", err) + return scenarioResult{}, err } + exportedCount := len(exported) + + if scenario.mode == exporthistory.ExportModeContinuous { + lateSources, err := runKnownSources(ctx, client, scenario.name+"-late", 1) + sources = append(sources, lateSources...) + if err != nil { + return scenarioResult{}, err + } + if err := waitForListableInstances(ctx, client, from, sourceIDs(lateSources), 90*time.Second); err != nil { + return scenarioResult{}, err + } + lateDescription, err := waitForExport(ctx, job, scenario.mode, len(sources)) + if err != nil { + return scenarioResult{}, err + } + if lateDescription.Status != exporthistory.ExportJobStatusActive { + return scenarioResult{}, fmt.Errorf("continuous job %s status=%s after late export, want Active", + jobID, lateDescription.Status) + } + if lateDescription.OrchestratorInstanceID != description.OrchestratorInstanceID { + return scenarioResult{}, fmt.Errorf("continuous job %s changed run ID from %s to %s", + jobID, description.OrchestratorInstanceID, lateDescription.OrchestratorInstanceID) + } + if err := verifyJobLifecycle(ctx, exportClient, jobID, lateDescription, scenario.mode); err != nil { + return scenarioResult{}, err + } + exported, err = waitForDownloadedHistories(ctx, blobClient, container, prefix, scenario.format, sources) + if err != nil { + return scenarioResult{}, err + } + exportedCount = len(exported) + description = lateDescription + } + + result = scenarioResult{sourceIDs: sourceIDs(sources), prefix: prefix, exported: exportedCount} + if scenario.recreate { + firstRunID := api.InstanceID(description.OrchestratorInstanceID) + if firstRunID == "" { + return scenarioResult{}, errors.New("completed export job did not report its run orchestration ID") + } + retainedRuns = append(retainedRuns, firstRunID) + result.retainedRuns = append(result.retainedRuns, firstRunID) + if err := job.Create(ctx, exporthistory.JobCreationOptions{ + JobID: jobID, + Mode: scenario.mode, + CompletedTimeFrom: from, + CompletedTimeTo: completedTimeTo(scenario.mode), + Format: &format, + MaxInstancesPerBatch: 2, + }); err != nil { + return scenarioResult{}, fmt.Errorf("recreate job %s: %w", jobID, err) + } + recreated, err := waitForExport(ctx, job, scenario.mode, len(sources)) + if err != nil { + return scenarioResult{}, err + } + if recreated.OrchestratorInstanceID == description.OrchestratorInstanceID { + return scenarioResult{}, errors.New("recreated job reused the previous run ID") + } + retained, err := client.FetchOrchestrationMetadata(ctx, firstRunID) + if err != nil { + return scenarioResult{}, fmt.Errorf("previous export generation was not retained: %w", err) + } + if retained.RuntimeStatus != api.RUNTIME_STATUS_COMPLETED { + return scenarioResult{}, fmt.Errorf("previous export generation status=%s, want COMPLETED", retained.RuntimeStatus) + } + if _, err := waitForDownloadedHistories(ctx, blobClient, container, prefix, scenario.format, sources); err != nil { + return scenarioResult{}, err + } + } + + if err := job.Delete(ctx); err != nil { + return scenarioResult{}, fmt.Errorf("delete job %s: %w", jobID, err) + } + if _, err := exportClient.GetJob(ctx, jobID); !errors.Is(err, exporthistory.ErrJobNotFound) { + return scenarioResult{}, fmt.Errorf("expected deleted job %s to be missing, got %v", jobID, err) + } + deleteJob = false + return result, nil +} + +func startExportHistoryApp( + ctx context.Context, + options *durabletaskscheduler.Options, + store exporthistory.Store, +) (*sampleApp, error) { registry := task.NewTaskRegistry() - if err := registry.AddOrchestratorN("Greeting", greeting); err != nil { - return err + if err := registry.AddOrchestratorN("ExportHistoryKnownSource", exportHistoryKnownSource); err != nil { + return nil, err } - if err := registry.AddActivityN("SayHello", sayHello); err != nil { - return err + if err := registry.AddActivityN("ExportHistoryEcho", exportHistoryEcho); err != nil { + return nil, err + } + logger := api.DefaultLogger() + client, err := durabletaskscheduler.NewClient(ctx, options, logger) + if err != nil { + return nil, fmt.Errorf("create DTS client: %w", err) } if err := exporthistory.Register(registry, exporthistory.WorkerOptions{ Source: client.TaskHubGrpcClient, Store: store, }); err != nil { - return err + return nil, errors.Join(err, client.Close()) } - worker, err := durabletaskscheduler.NewWorker(options, registry, logger, durabletaskclient.WithAutoWorkItemFilters(), - // Keeps the unversioned export system tasks routable when the - // application enables strict worker versioning. + durabletaskclient.WithMaxConcurrentOrchestrationWorkItems(4), + durabletaskclient.WithMaxConcurrentActivityWorkItems(8), exporthistory.WithExportHistory(), ) if err != nil { - return fmt.Errorf("failed to create the worker: %w", err) + return nil, errors.Join(fmt.Errorf("create DTS worker: %w", err), client.Close()) } if err := worker.Start(ctx); err != nil { - return fmt.Errorf("failed to start the worker: %w", err) + return nil, errors.Join(fmt.Errorf("start DTS worker: %w", err), client.Close()) } - defer func() { - shutdownCtx, shutdownCancel := context.WithTimeout(context.Background(), 30*time.Second) - defer shutdownCancel() - if shutdownErr := worker.Shutdown(shutdownCtx); shutdownErr != nil { - log.Printf("failed to shut down the worker: %v", shutdownErr) + return &sampleApp{client: client, worker: worker}, nil +} + +func (a *sampleApp) shutdown() error { + if a == nil { + return nil + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + return errors.Join(a.worker.Shutdown(shutdownCtx), a.client.Close()) +} + +func runKnownSources( + ctx context.Context, + client *durabletaskscheduler.Client, + scenario string, + count int, +) ([]sourceExecution, error) { + sources := make([]sourceExecution, 0, count) + for i := 0; i < count; i++ { + id := dtssample.NewInstanceID("exporthistory-" + scenario) + sources = append(sources, sourceExecution{id: id}) + if _, err := client.ScheduleNewOrchestration(ctx, "ExportHistoryKnownSource", + api.WithInstanceID(id), + api.WithInput(sourceInput{Scenario: scenario, Index: i}), + api.WithTags(map[string]string{"sample": "exporthistory", "scenario": scenario})); err != nil { + return sources, err } - }() + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return sources, err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return sources, err + } + var output sourceOutput + if err := metadata.ReadOutput(&output); err != nil { + return sources, err + } + want := fmt.Sprintf("history:%s:%d", scenario, i) + if output.Value != want { + return sources, fmt.Errorf("%s output=%q, want %q", id, output.Value, want) + } + sources[len(sources)-1] = sourceExecution{id: id, executionID: metadata.ExecutionID, output: output.Value} + } + return sources, nil +} - from := time.Now().UTC().Add(-time.Minute) - for _, name := range []string{"Ana", "Bo", "Cyd"} { - id, scheduleErr := client.ScheduleNewOrchestration(ctx, "Greeting", api.WithInput(name)) - if scheduleErr != nil { - return fmt.Errorf("failed to schedule an orchestration: %w", scheduleErr) +func exportHistoryKnownSource(ctx *task.OrchestrationContext) (any, error) { + var input sourceInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.CreateTimer(25 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var output sourceOutput + if err := ctx.CallActivity("ExportHistoryEcho", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func exportHistoryEcho(ctx task.ActivityContext) (any, error) { + var input sourceInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return sourceOutput{Value: fmt.Sprintf("history:%s:%d", input.Scenario, input.Index)}, nil +} + +func waitForExport( + ctx context.Context, + job *exporthistory.JobClient, + mode exporthistory.ExportMode, + wantExported int, +) (*exporthistory.ExportJobDescription, error) { + for { + description, err := job.Describe(ctx) + if err != nil { + return nil, err + } + if description.Status == exporthistory.ExportJobStatusFailed { + return nil, fmt.Errorf("export job %s failed: %s", description.JobID, description.LastError) } - metadata, waitErr := client.WaitForOrchestrationCompletion(ctx, id) - if waitErr != nil { - return fmt.Errorf("failed to wait for %s: %w", id, waitErr) + if description.ExportedInstances >= int64(wantExported) && description.ScannedInstances > 0 { + if mode == exporthistory.ExportModeContinuous || description.Status == exporthistory.ExportJobStatusCompleted { + return description, nil + } + } + if mode == exporthistory.ExportModeBatch && description.Status == exporthistory.ExportJobStatusCompleted { + return nil, fmt.Errorf("batch job completed after scanning %d/exporting %d, want at least %d", + description.ScannedInstances, description.ExportedInstances, wantExported) + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(time.Second): } - log.Printf("orchestration %s finished with status %s", id, metadata.RuntimeStatus) } +} - exportClient, err := exporthistory.NewClient(client.TaskHubGrpcClient, exporthistory.ClientOptions{ - ContainerName: container, - }) +func verifyJobLifecycle( + ctx context.Context, + client *exporthistory.Client, + jobID string, + description *exporthistory.ExportJobDescription, + mode exporthistory.ExportMode, +) error { + got, err := client.GetJob(ctx, jobID) if err != nil { - return fmt.Errorf("failed to create the export client: %w", err) + return err + } + if got.JobID != jobID || got.Status != description.Status { + return fmt.Errorf("GetJob returned %+v, want job %s status %s", got, jobID, description.Status) + } + listed, err := client.ListJobs(ctx, exporthistory.ExportJobQuery{JobIDPrefix: jobID, PageSize: 10}) + if err != nil { + return err + } + if len(listed.Jobs) != 1 || listed.Jobs[0].JobID != jobID { + return fmt.Errorf("ListJobs returned %d matches for %s", len(listed.Jobs), jobID) + } + if description.Config == nil || description.Config.Mode != mode { + return fmt.Errorf("job %s description missing mode %s", jobID, mode) + } + return nil +} + +func waitForDownloadedHistories( + ctx context.Context, + client *azblob.Client, + container string, + prefix string, + format exporthistory.ExportFormatKind, + sources []sourceExecution, +) (map[string][]api.HistoryEvent, error) { + var lastErr error + for { + exported, err := downloadHistories(ctx, client, container, prefix, format) + if err == nil { + err = verifyHistories(exported, sources) + } + if err == nil { + return exported, nil + } + lastErr = err + select { + case <-ctx.Done(): + return nil, fmt.Errorf("downloaded histories under %s never matched sources: %w; last error: %v", prefix, ctx.Err(), lastErr) + case <-time.After(time.Second): + } } +} - job, err := exportClient.CreateJob(ctx, exporthistory.JobCreationOptions{ - Mode: exporthistory.ExportModeBatch, - CompletedTimeFrom: from, - CompletedTimeTo: time.Now().UTC(), +func downloadHistories( + ctx context.Context, + client *azblob.Client, + container string, + prefix string, + format exporthistory.ExportFormatKind, +) (map[string][]api.HistoryEvent, error) { + exported := map[string][]api.HistoryEvent{} + pager := client.NewListBlobsFlatPager(container, &azblob.ListBlobsFlatOptions{ + Prefix: &prefix, + Include: azblob.ListBlobsInclude{Metadata: true}, }) + for pager.More() { + page, err := pager.NextPage(ctx) + if err != nil { + return nil, err + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + instanceID := metadataValue(item.Metadata, "instanceId") + if instanceID == "" { + return nil, fmt.Errorf("blob %s has no instanceId metadata", *item.Name) + } + response, err := client.DownloadStream(ctx, container, *item.Name, nil) + if err != nil { + return nil, err + } + body, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if readErr != nil || closeErr != nil { + return nil, errors.Join(readErr, closeErr) + } + events, err := decodeHistoryObject(body, format) + if err != nil { + return nil, fmt.Errorf("decode %s: %w", *item.Name, err) + } + exported[instanceID] = events + } + } + return exported, nil +} + +func verifyHistories(exported map[string][]api.HistoryEvent, sources []sourceExecution) error { + for _, source := range sources { + events, ok := exported[string(source.id)] + if !ok { + return fmt.Errorf("source %s was not exported", source.id) + } + var sawStarted, sawCompleted, sawActivity bool + for _, event := range events { + switch event.Type { + case api.HistoryEventExecutionStarted: + if event.ExecutionStarted != nil && event.ExecutionStarted.ExecutionID == source.executionID { + sawStarted = true + } + case api.HistoryEventExecutionCompleted: + if event.ExecutionCompleted != nil { + var output sourceOutput + if err := json.Unmarshal([]byte(event.ExecutionCompleted.SerializedResult), &output); err != nil { + return fmt.Errorf("decode completed output for %s: %w", source.id, err) + } + sawCompleted = output.Value == source.output + } + case api.HistoryEventTaskScheduled: + if event.TaskScheduled != nil && event.TaskScheduled.Name == "ExportHistoryEcho" { + sawActivity = true + } + } + } + if !sawStarted || !sawCompleted || !sawActivity { + return fmt.Errorf("history for %s missing started=%v completed=%v activity=%v", source.id, sawStarted, sawCompleted, sawActivity) + } + } + return nil +} + +func decodeHistoryObject(body []byte, format exporthistory.ExportFormatKind) ([]api.HistoryEvent, error) { + if format == exporthistory.ExportFormatJSON { + var events []api.HistoryEvent + if err := json.Unmarshal(body, &events); err != nil { + return nil, err + } + return events, nil + } + reader, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { - return fmt.Errorf("failed to create the export job: %w", err) + return nil, err + } + decompressed, readErr := io.ReadAll(reader) + closeErr := reader.Close() + if readErr != nil || closeErr != nil { + return nil, errors.Join(readErr, closeErr) + } + var events []api.HistoryEvent + for _, line := range strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") { + if line == "" { + continue + } + var event api.HistoryEvent + if err := json.Unmarshal([]byte(line), &event); err != nil { + return nil, err + } + events = append(events, event) } - log.Printf("created export job %s", job.ID()) + return events, nil +} - // Batch jobs complete on their own once the window is drained. +func waitForListableInstances( + ctx context.Context, + client *durabletaskscheduler.Client, + from time.Time, + want []api.InstanceID, + timeout time.Duration, +) error { + deadline := time.Now().Add(timeout) for { - description, describeErr := job.Describe(ctx) - if describeErr != nil { - return fmt.Errorf("failed to describe the export job: %w", describeErr) - } - log.Printf("export job %s is %s (scanned %d, exported %d)", - description.JobID, description.Status, description.ScannedInstances, description.ExportedInstances) - switch description.Status { - case exporthistory.ExportJobStatusCompleted: - log.Printf("history exported to container %q under prefix %q", - description.Config.Destination.Container, description.Config.Destination.Prefix) + seen := map[api.InstanceID]bool{} + token := "" + for pages := 0; pages < 100; pages++ { + result, err := client.ListInstanceIDs(ctx, api.InstanceIDQuery{ + RuntimeStatus: exporthistory.TerminalStatuses(), + CompletedTimeFrom: from, + PageSize: 100, + ContinuationToken: token, + }) + if err != nil { + return err + } + for _, id := range result.InstanceIDs { + seen[id] = true + } + if result.ContinuationToken == "" { + break + } + token = result.ContinuationToken + } + missing := missingIDs(seen, want) + if len(missing) == 0 { return nil - case exporthistory.ExportJobStatusFailed: - return fmt.Errorf("export job failed: %s", description.LastError) + } + if time.Now().After(deadline) { + return fmt.Errorf("DTS ListInstanceIDs did not expose source IDs %v within %s", missing, timeout) } select { case <-ctx.Done(): return ctx.Err() - case <-time.After(2 * time.Second): + case <-time.After(time.Second): } } } -func greeting(ctx *task.OrchestrationContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err +func readStorageSettings() (storageSettings, error) { + connectionString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + if connectionString == "" { + connectionString = os.Getenv("EXPORT_STORAGE_CONNECTION_STRING") } - var message string - if err := ctx.CallActivity("SayHello", task.WithActivityInput(name)).Await(&message); err != nil { - return nil, err + if connectionString == "" { + return storageSettings{}, errors.New("AZURE_STORAGE_CONNECTION_STRING is required; EXPORT_STORAGE_CONNECTION_STRING is accepted for compatibility") } - return message, nil + allowHTTP, err := allowInsecureStorage(connectionString) + if err != nil { + return storageSettings{}, err + } + container := strings.TrimSpace(os.Getenv("EXPORT_CONTAINER")) + deleteContainer := false + if container == "" { + container, err = randomContainerName("dtgoexport") + if err != nil { + return storageSettings{}, err + } + deleteContainer = true + } + return storageSettings{connectionString: connectionString, container: container, deleteContainer: deleteContainer, allowInsecureHTTP: allowHTTP}, nil } -func sayHello(ctx task.ActivityContext) (any, error) { - var name string - if err := ctx.GetInput(&name); err != nil { - return nil, err +func requireIsolatedTaskHub(options *durabletaskscheduler.Options) error { + if os.Getenv("DTS_SAMPLE_ISOLATED_TASKHUB") == "1" { + return nil + } + return fmt.Errorf("exporthistory requires an isolated task hub because export windows cannot filter by instance ID prefix; set DTS_SAMPLE_ISOLATED_TASKHUB=1 for a private hub (current hub %q)", options.TaskHubName) +} + +func allowInsecureStorage(connectionString string) (bool, error) { + if !strings.Contains(strings.ToLower(connectionString), "http://") && + !strings.Contains(strings.ToLower(connectionString), "usedevelopmentstorage=true") { + return false, nil + } + if os.Getenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE") != "1" { + return false, errors.New("plain HTTP storage endpoints require DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1 and loopback Azurite endpoints") + } + if strings.Contains(strings.ToLower(connectionString), "usedevelopmentstorage=true") { + return true, nil + } + endpoints := connectionStringEndpoints(connectionString) + if len(endpoints) == 0 { + return false, errors.New("plaintext storage connection strings must include explicit loopback service endpoints") + } + for _, endpoint := range endpoints { + if strings.HasPrefix(strings.ToLower(endpoint), "http://") && !isLoopbackEndpoint(endpoint) { + return false, fmt.Errorf("refusing non-loopback plaintext storage endpoint %q", endpoint) + } + } + return true, nil +} + +func selectScenarios(name string) ([]scenario, error) { + all := []scenario{ + {name: "batch-jsonl", mode: exporthistory.ExportModeBatch, format: exporthistory.ExportFormatJSONL, recreate: true}, + {name: "batch-json", mode: exporthistory.ExportModeBatch, format: exporthistory.ExportFormatJSON}, + {name: "continuous-jsonl", mode: exporthistory.ExportModeContinuous, format: exporthistory.ExportFormatJSONL}, + {name: "continuous-json", mode: exporthistory.ExportModeContinuous, format: exporthistory.ExportFormatJSON}, + } + if name == "all" { + return all, nil + } + for _, candidate := range all { + if candidate.name == name { + return []scenario{candidate}, nil + } + } + return nil, fmt.Errorf("unknown scenario %q", name) +} + +func completedTimeTo(mode exporthistory.ExportMode) time.Time { + if mode == exporthistory.ExportModeBatch { + return time.Now().UTC() + } + return time.Time{} +} + +func sourceIDs(sources []sourceExecution) []api.InstanceID { + ids := make([]api.InstanceID, 0, len(sources)) + for _, source := range sources { + ids = append(ids, source.id) + } + return ids +} + +func missingIDs(seen map[api.InstanceID]bool, want []api.InstanceID) []api.InstanceID { + var missing []api.InstanceID + for _, id := range want { + if !seen[id] { + missing = append(missing, id) + } + } + return missing +} + +func randomContainerName(prefix string) (string, error) { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", err + } + return prefix + hex.EncodeToString(suffix[:]), nil +} + +func compactID() (string, error) { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generate job suffix: %w", err) + } + return hex.EncodeToString(suffix[:]), nil +} + +func metadataValue(metadata map[string]*string, key string) string { + for name, value := range metadata { + if strings.EqualFold(name, key) && value != nil { + return *value + } + } + return "" +} + +func deleteBlobsWithPrefix(ctx context.Context, client *azblob.Client, container, prefix string) error { + if prefix == "" { + return nil + } + var failures []error + pager := client.NewListBlobsFlatPager(container, &azblob.ListBlobsFlatOptions{Prefix: &prefix}) + for pager.More() { + page, err := pager.NextPage(ctx) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return nil + } + if err != nil { + return err + } + for _, item := range page.Segment.BlobItems { + if item.Name == nil { + continue + } + if _, err := client.DeleteBlob(ctx, container, *item.Name, nil); err != nil && + !bloberror.HasCode(err, bloberror.BlobNotFound, bloberror.ContainerNotFound) { + failures = append(failures, fmt.Errorf("delete blob %s: %w", *item.Name, err)) + } + } + } + return errors.Join(failures...) +} + +func connectionStringEndpoints(connectionString string) []string { + var endpoints []string + for _, part := range strings.Split(connectionString, ";") { + key, value, ok := strings.Cut(part, "=") + if ok && strings.HasSuffix(strings.ToLower(strings.TrimSpace(key)), "endpoint") { + endpoints = append(endpoints, strings.TrimSpace(value)) + } + } + return endpoints +} + +func isLoopbackEndpoint(endpoint string) bool { + parsed, err := url.Parse(endpoint) + if err != nil { + return false + } + host := parsed.Hostname() + if strings.EqualFold(host, "localhost") { + return true } - return "Hello, " + name + "!", nil + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() } diff --git a/samples/exporthistory/exporthistory_test.go b/samples/exporthistory/exporthistory_test.go new file mode 100644 index 00000000..563b67ae --- /dev/null +++ b/samples/exporthistory/exporthistory_test.go @@ -0,0 +1,87 @@ +package main + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/exporthistory" + "google.golang.org/grpc" +) + +type failedScheduleConnection struct{ grpc.ClientConnInterface } + +func (failedScheduleConnection) Invoke(context.Context, string, any, any, ...grpc.CallOption) error { + return context.Canceled +} + +func TestSourceIDsSurviveAnAmbiguousSchedulingFailure(t *testing.T) { + client := &durabletaskscheduler.Client{ + TaskHubGrpcClient: durabletaskclient.NewTaskHubGrpcClient(failedScheduleConnection{}, api.DefaultLogger()), + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + sources, err := runKnownSources(ctx, client, "cleanup", 3) + if !errors.Is(err, context.Canceled) || len(sources) != 1 || + !strings.HasPrefix(string(sources[0].id), "sample-exporthistory-cleanup-") { + t.Fatalf("intended cleanup ID was lost: sources=%v err=%v", sources, err) + } +} + +func TestExportRequiresExplicitIsolationAcknowledgement(t *testing.T) { + t.Setenv("DTS_SAMPLE_ISOLATED_TASKHUB", "") + options := durabletaskscheduler.NewOptions("https://example.invalid", "sample-shared") + if requireIsolatedTaskHub(options) == nil { + t.Fatal("a sample-looking name is not proof of isolation") + } + t.Setenv("DTS_SAMPLE_ISOLATED_TASKHUB", "1") + if err := requireIsolatedTaskHub(options); err != nil { + t.Fatal(err) + } +} + +func TestDecodeHistoryObjectJSONAndJSONL(t *testing.T) { + events := []api.HistoryEvent{{Type: api.HistoryEventExecutionStarted}, {Type: api.HistoryEventExecutionCompleted}} + jsonBody, err := json.Marshal(events) + if err != nil { + t.Fatal(err) + } + decoded, err := decodeHistoryObject(jsonBody, exporthistory.ExportFormatJSON) + if err != nil || len(decoded) != 2 { + t.Fatalf("decode JSON len=%d err=%v", len(decoded), err) + } + + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write([]byte(`{"type":"ExecutionStarted"}` + "\n" + `{"type":"ExecutionCompleted"}` + "\n")); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + decoded, err = decodeHistoryObject(compressed.Bytes(), exporthistory.ExportFormatJSONL) + if err != nil || len(decoded) != 2 { + t.Fatalf("decode JSONL len=%d err=%v", len(decoded), err) + } +} + +func TestSelectScenarios(t *testing.T) { + all, err := selectScenarios("all") + if err != nil || len(all) != 4 { + t.Fatalf("all scenarios len=%d err=%v", len(all), err) + } + one, err := selectScenarios("continuous-json") + if err != nil || len(one) != 1 || one[0].name != "continuous-json" { + t.Fatalf("continuous-json scenarios=%v err=%v", one, err) + } + if _, err := selectScenarios("missing"); err == nil { + t.Fatal("expected unknown scenario to fail") + } +} diff --git a/samples/largepayloads/README.md b/samples/largepayloads/README.md new file mode 100644 index 00000000..3c912d1c --- /dev/null +++ b/samples/largepayloads/README.md @@ -0,0 +1,54 @@ +# Large payloads sample + +Demonstrates Durable Task Scheduler large-payload externalization and hydration +with the public `payload.AzureBlobStore` and `durabletaskscheduler` APIs. + +## Features + +- Forces a 64-byte externalization threshold and sends a payload larger than it. +- Uses Azure Blob `blob:v2` references and optional gzip compression. +- Verifies the orchestration output hash and downloads the actual blob objects. +- Creates a unique owned container every run and deletes it during cleanup. + +## Prerequisites + +- `DTS_CONNECTION_STRING` for an isolated DTS task hub. +- `AZURE_STORAGE_CONNECTION_STRING`; `EXPORT_STORAGE_CONNECTION_STRING` is also + accepted for compatibility. +- For local Azurite over plaintext HTTP, set `DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1`. + The sample rejects non-loopback plaintext endpoints. + +Do not commit emulator keys. Local automation should generate the Azurite +account key at runtime. + +## Run + +```bash +# from the repository root +go run ./samples/largepayloads +``` + +Optional variants: + +```bash +LARGEPAYLOADS_DISABLE_GZIP=1 go run ./samples/largepayloads +``` + +## Expected outcome + +The process exits with status 0 and prints: + +```text +SAMPLE_OK largepayloads +``` + +## Cleanup + +The orchestration is recursively terminated/purged before shutdown. The unique +owned container is deleted. The sample does not accept an existing container, so +it never deletes another writer's blobs. + +## Required validation variants + +Integration validation should run the default gzip-enabled command and the +`LARGEPAYLOADS_DISABLE_GZIP=1` variant against DTS and Azurite. diff --git a/samples/largepayloads/largepayloads.go b/samples/largepayloads/largepayloads.go new file mode 100644 index 00000000..5cbb8682 --- /dev/null +++ b/samples/largepayloads/largepayloads.go @@ -0,0 +1,332 @@ +// Command largepayloads verifies Azure Blob backed large-payload +// externalization and hydration against a live Durable Task Scheduler hub. +package main + +import ( + "bytes" + "compress/gzip" + "context" + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "net" + "net/url" + "os" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +const ( + payloadThresholdBytes = 64 + payloadMaxBytes = 4 * 1024 * 1024 +) + +type sampleInput struct { + Content string `json:"content"` + SHA256 string `json:"sha256"` +} + +type sampleOutput struct { + Content string `json:"content"` + SHA256 string `json:"sha256"` + Length int `json:"length"` +} + +type storageSettings struct { + connectionString string + container string + allowInsecureHTTP bool +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK largepayloads") +} + +func run() (err error) { + settings, err := readStorageSettings() + if err != nil { + return err + } + blobClient, err := azblob.NewClientFromConnectionString(settings.connectionString, nil) + if err != nil { + return fmt.Errorf("create Azure Blob reader: %w", err) + } + compressionEnabled := compressionSetting() + store, err := payload.NewAzureBlobStore(payload.AzureBlobStoreOptions{ + ConnectionString: settings.connectionString, + Container: settings.container, + AllowInsecureHTTP: settings.allowInsecureHTTP, + CompressionEnabled: &compressionEnabled, + MaxPayloadBytes: payloadMaxBytes, + }) + if err != nil { + return fmt.Errorf("create Azure Blob payload store: %w", err) + } + cleanupStorage := func() error { + cleanupCtx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + _, cleanupErr := blobClient.DeleteContainer(cleanupCtx, settings.container, nil) + if bloberror.HasCode(cleanupErr, bloberror.ContainerNotFound) { + return nil + } + return cleanupErr + } + + options, err := dtssample.Options() + if err != nil { + return err + } + options.LargePayloads = &api.LargePayloadOptions{ + Store: store, + Resolver: store, + ThresholdBytes: payloadThresholdBytes, + MaxPayloadBytes: payloadMaxBytes, + } + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("LargePayloadsRoundTrip", largePayloadsRoundTrip); err != nil { + return err + } + if err := registry.AddActivityN("ValidateLargePayload", validateLargePayload); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + app, err := dtssample.StartWithOptions(ctx, options, registry, durabletaskclient.WithAutoWorkItemFilters()) + if err != nil { + err = errors.Join(err, cleanupStorage()) + return err + } + var ownedIDs []api.InstanceID + defer func() { + err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), cleanupStorage(), app.Shutdown()) + }() + + content := strings.Repeat("large-payloads-sample-", 512) + hash := sha256.Sum256([]byte(content)) + input := sampleInput{Content: content, SHA256: hex.EncodeToString(hash[:])} + instanceID := dtssample.NewInstanceID("largepayloads") + ownedIDs = append(ownedIDs, instanceID) + if _, err := app.Client.ScheduleNewOrchestration(ctx, "LargePayloadsRoundTrip", + api.WithInstanceID(instanceID), + api.WithInput(input), + api.WithTags(map[string]string{"sample": "largepayloads"})); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output sampleOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if output.Content != content || output.SHA256 != input.SHA256 || output.Length != len(content) { + return fmt.Errorf("roundtrip mismatch: length=%d hash=%s", output.Length, output.SHA256) + } + + created, err := listBlobNames(ctx, blobClient, settings.container) + if err != nil { + return err + } + if len(created) < 3 { + return fmt.Errorf("expected at least 3 externalized payload blobs, found %d (%v)", len(created), created) + } + if err := verifyStoredPayloads(ctx, blobClient, settings.container, created, input.SHA256, compressionEnabled); err != nil { + return err + } + fmt.Printf("verified %d Azure Blob payload object(s) in container %s\n", len(created), settings.container) + return nil +} + +func largePayloadsRoundTrip(ctx *task.OrchestrationContext) (any, error) { + var input sampleInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + var output sampleOutput + if err := ctx.CallActivity("ValidateLargePayload", task.WithActivityInput(input)).Await(&output); err != nil { + return nil, err + } + return output, nil +} + +func validateLargePayload(ctx task.ActivityContext) (any, error) { + var input sampleInput + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + hash := sha256.Sum256([]byte(input.Content)) + if got := hex.EncodeToString(hash[:]); got != input.SHA256 { + return nil, fmt.Errorf("activity received hash %s, want %s", got, input.SHA256) + } + return sampleOutput{Content: input.Content, SHA256: input.SHA256, Length: len(input.Content)}, nil +} + +func readStorageSettings() (storageSettings, error) { + connectionString := os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + if connectionString == "" { + connectionString = os.Getenv("EXPORT_STORAGE_CONNECTION_STRING") + } + if connectionString == "" { + return storageSettings{}, errors.New("AZURE_STORAGE_CONNECTION_STRING is required; EXPORT_STORAGE_CONNECTION_STRING is accepted for compatibility") + } + allowHTTP, err := allowInsecureStorage(connectionString) + if err != nil { + return storageSettings{}, err + } + container, err := randomContainerName("dtgolarge") + if err != nil { + return storageSettings{}, err + } + return storageSettings{ + connectionString: connectionString, + container: container, + allowInsecureHTTP: allowHTTP, + }, nil +} + +func compressionSetting() bool { + value := strings.ToLower(strings.TrimSpace(os.Getenv("LARGEPAYLOADS_DISABLE_GZIP"))) + return value != "1" && value != "true" +} + +func allowInsecureStorage(connectionString string) (bool, error) { + if !strings.Contains(strings.ToLower(connectionString), "http://") && + !strings.Contains(strings.ToLower(connectionString), "usedevelopmentstorage=true") { + return false, nil + } + if os.Getenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE") != "1" { + return false, errors.New("plain HTTP storage endpoints require DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1 and loopback Azurite endpoints") + } + if strings.Contains(strings.ToLower(connectionString), "usedevelopmentstorage=true") { + return true, nil + } + endpoints := connectionStringEndpoints(connectionString) + if len(endpoints) == 0 { + return false, errors.New("plaintext storage connection strings must include explicit loopback service endpoints") + } + for _, endpoint := range endpoints { + if strings.HasPrefix(strings.ToLower(endpoint), "http://") && !isLoopbackEndpoint(endpoint) { + return false, fmt.Errorf("refusing non-loopback plaintext storage endpoint %q", endpoint) + } + } + return true, nil +} + +func connectionStringEndpoints(connectionString string) []string { + var endpoints []string + for _, part := range strings.Split(connectionString, ";") { + key, value, ok := strings.Cut(part, "=") + if ok && strings.HasSuffix(strings.ToLower(strings.TrimSpace(key)), "endpoint") { + endpoints = append(endpoints, strings.TrimSpace(value)) + } + } + return endpoints +} + +func isLoopbackEndpoint(endpoint string) bool { + parsed, err := url.Parse(endpoint) + if err != nil { + return false + } + host := parsed.Hostname() + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +func randomContainerName(prefix string) (string, error) { + var suffix [8]byte + if _, err := rand.Read(suffix[:]); err != nil { + return "", fmt.Errorf("generate container name: %w", err) + } + return prefix + hex.EncodeToString(suffix[:]), nil +} + +func listBlobNames(ctx context.Context, client *azblob.Client, container string) (map[string]struct{}, error) { + names := map[string]struct{}{} + pager := client.NewListBlobsFlatPager(container, nil) + for pager.More() { + page, err := pager.NextPage(ctx) + if bloberror.HasCode(err, bloberror.ContainerNotFound) { + return names, nil + } + if err != nil { + return nil, fmt.Errorf("list blobs in %s: %w", container, err) + } + for _, item := range page.Segment.BlobItems { + if item.Name != nil { + names[*item.Name] = struct{}{} + } + } + } + return names, nil +} + +func verifyStoredPayloads( + ctx context.Context, + client *azblob.Client, + container string, + names map[string]struct{}, + wantHash string, + wantGzip bool, +) error { + sawHash := false + for name := range names { + response, err := client.DownloadStream(ctx, container, name, nil) + if err != nil { + return fmt.Errorf("download payload blob %s: %w", name, err) + } + body, readErr := io.ReadAll(response.Body) + closeErr := response.Body.Close() + if readErr != nil || closeErr != nil { + return errors.Join(readErr, closeErr) + } + isGzip := response.ContentEncoding != nil && strings.EqualFold(*response.ContentEncoding, "gzip") + if isGzip != wantGzip { + return fmt.Errorf("payload blob %s gzip=%t, want %t", name, isGzip, wantGzip) + } + if isGzip { + reader, err := gzip.NewReader(bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("decode gzip payload blob %s: %w", name, err) + } + body, readErr = io.ReadAll(reader) + closeErr = reader.Close() + if readErr != nil || closeErr != nil { + return errors.Join(readErr, closeErr) + } + } + if len(body) <= payloadThresholdBytes { + return fmt.Errorf("payload blob %s contains only %d bytes, threshold is %d", name, len(body), payloadThresholdBytes) + } + if strings.Contains(string(body), wantHash) { + sawHash = true + } + } + if !sawHash { + return fmt.Errorf("downloaded payload blobs did not contain expected hash %s", wantHash) + } + return nil +} diff --git a/samples/largepayloads/largepayloads_test.go b/samples/largepayloads/largepayloads_test.go new file mode 100644 index 00000000..010df51a --- /dev/null +++ b/samples/largepayloads/largepayloads_test.go @@ -0,0 +1,29 @@ +package main + +import "testing" + +func TestAllowInsecureStorageRequiresLoopbackOptIn(t *testing.T) { + t.Setenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE", "") + if _, err := allowInsecureStorage("BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;AccountName=a;AccountKey=b"); err == nil { + t.Fatal("expected plaintext storage without opt-in to fail") + } + + t.Setenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE", "1") + if _, err := allowInsecureStorage("BlobEndpoint=http://example.com:10000/account;AccountName=a;AccountKey=b"); err == nil { + t.Fatal("expected non-loopback plaintext storage to fail") + } + if allow, err := allowInsecureStorage("BlobEndpoint=http://127.0.0.1:10000/account;AccountName=a;AccountKey=b"); err != nil || !allow { + t.Fatalf("loopback plaintext storage allow=%v err=%v, want true nil", allow, err) + } +} + +func TestReadStorageSettingsAlwaysCreatesOwnedContainer(t *testing.T) { + t.Setenv("AZURE_STORAGE_CONNECTION_STRING", "DefaultEndpointsProtocol=https;AccountName=acct;AccountKey=key;EndpointSuffix=core.windows.net") + settings, err := readStorageSettings() + if err != nil { + t.Fatal(err) + } + if settings.container == "" { + t.Fatalf("container=%q, want generated owned container", settings.container) + } +} diff --git a/samples/replayanalysis/README.md b/samples/replayanalysis/README.md new file mode 100644 index 00000000..2f7114ed --- /dev/null +++ b/samples/replayanalysis/README.md @@ -0,0 +1,66 @@ +# Replay analysis sample + +Shows replay-safe orchestration patterns and how to verify them with +`cmd/orchestratorvet`. + +## Features + +- Good runnable orchestration uses deterministic counterparts: + `ctx.CurrentTimeUtc`, `ctx.NewGuid`, `ctx.CreateTimer`, and activities for + side effects. +- Intentional bad fixtures live under `testdata/bad`, outside normal + `go test ./...` package discovery. +- Integration validation can build the existing vet tool and run positive/negative checks + without adding a module or root dependency. + +## Prerequisites + +- `DTS_CONNECTION_STRING` for DTS. +- Existing `cmd/orchestratorvet` sources. + +## Run + +```bash +# from the repository root +go run ./samples/replayanalysis +``` + +Build and run the analyzer: + +```bash +# from the repository root +(cd cmd/orchestratorvet && go build -o ../../samples/replayanalysis/orchestratorvet-local .) +go vet -vettool="$PWD/samples/replayanalysis/orchestratorvet-local" ./samples/replayanalysis +! go vet -vettool="$PWD/samples/replayanalysis/orchestratorvet-local" ./samples/replayanalysis/testdata/bad +rm -f samples/replayanalysis/orchestratorvet-local +``` + +The negative command must report hazards such as wall-clock time, HTTP I/O, +goroutines/channels/select, random UUIDs, and stdout logging. + +## Suggested fixes + +To preview suggested fixes, copy the bad fixture to a scratch file inside the +workspace and run `go vet -fix` against that copy. Do not modify repository +fixtures during integration validation. + +## Expected outcome + +The runnable sample exits 0 and prints: + +```text +SAMPLE_OK replayanalysis +``` + +The good vet command exits 0. The bad fixture vet command exits nonzero with the +expected diagnostics. + +## Cleanup + +The sample recursively terminates/purges its orchestration before worker +shutdown. It creates no external resources. + +## Required validation variants + +Integration validation should run the `go run`, positive vet, and negative vet +commands above. diff --git a/samples/replayanalysis/replayanalysis.go b/samples/replayanalysis/replayanalysis.go new file mode 100644 index 00000000..2cbb4c91 --- /dev/null +++ b/samples/replayanalysis/replayanalysis.go @@ -0,0 +1,109 @@ +// Command replayanalysis pairs a replay-safe orchestration with instructions +// for checking intentionally unsafe fixtures using cmd/orchestratorvet. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +type replayOutput struct { + StartedAt time.Time `json:"startedAt"` + DurableID string `json:"durableId"` + ActivityOut string `json:"activityOut"` +} + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK replayanalysis") +} + +func run() (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("ReplayAnalysisSafe", replayAnalysisSafe); err != nil { + return err + } + if err := registry.AddActivityN("ReplayAnalysisActivity", replayAnalysisActivity); err != nil { + return err + } + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + var ownedIDs []api.InstanceID + defer func() { err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), app.Shutdown()) }() + + instanceID := dtssample.NewInstanceID("replayanalysis") + ownedIDs = append(ownedIDs, instanceID) + if _, err := app.Client.ScheduleNewOrchestration(ctx, "ReplayAnalysisSafe", + api.WithInstanceID(instanceID), + api.WithInput("safe-counterpart"), + api.WithTags(map[string]string{"sample": "replayanalysis"})); err != nil { + return err + } + metadata, err := app.Client.WaitForOrchestrationCompletion(ctx, instanceID, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return err + } + var output replayOutput + if err := metadata.ReadOutput(&output); err != nil { + return err + } + if err := validateReplayOutput(output); err != nil { + return err + } + return nil +} + +func replayAnalysisSafe(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + startedAt := ctx.CurrentTimeUtc + durableID := ctx.NewGuid() + if err := ctx.CreateTimer(50 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + var activityOut string + if err := ctx.CallActivity("ReplayAnalysisActivity", task.WithActivityInput(input+"|"+durableID)).Await(&activityOut); err != nil { + return nil, err + } + return replayOutput{StartedAt: startedAt, DurableID: durableID, ActivityOut: activityOut}, nil +} + +func replayAnalysisActivity(ctx task.ActivityContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return "activity:" + input, nil +} + +func validateReplayOutput(output replayOutput) error { + if output.StartedAt.IsZero() { + return errors.New("orchestration did not return durable current time") + } + if output.DurableID == "" { + return errors.New("orchestration did not return deterministic GUID") + } + if output.ActivityOut != "activity:safe-counterpart|"+output.DurableID { + return fmt.Errorf("activity output=%q does not contain the returned deterministic ID", output.ActivityOut) + } + return nil +} diff --git a/samples/replayanalysis/replayanalysis_test.go b/samples/replayanalysis/replayanalysis_test.go new file mode 100644 index 00000000..dcdfc29f --- /dev/null +++ b/samples/replayanalysis/replayanalysis_test.go @@ -0,0 +1,26 @@ +package main + +import ( + "testing" + "time" +) + +func TestValidateReplayOutput(t *testing.T) { + err := validateReplayOutput(replayOutput{ + StartedAt: time.Unix(1, 0).UTC(), + DurableID: "guid", + ActivityOut: "activity:safe-counterpart|guid", + }) + if err != nil { + t.Fatalf("validateReplayOutput failed: %v", err) + } + if err := validateReplayOutput(replayOutput{}); err == nil { + t.Fatal("expected empty output to fail") + } + if err := validateReplayOutput(replayOutput{ + StartedAt: time.Unix(1, 0).UTC(), DurableID: "guid", + ActivityOut: "activity:safe-counterpart|different-guid", + }); err == nil { + t.Fatal("activity and orchestration identifiers must agree") + } +} diff --git a/samples/replayanalysis/testdata/bad/bad.go b/samples/replayanalysis/testdata/bad/bad.go new file mode 100644 index 00000000..0d232231 --- /dev/null +++ b/samples/replayanalysis/testdata/bad/bad.go @@ -0,0 +1,29 @@ +package bad + +import ( + "fmt" + "net/http" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/task" +) + +func Register(registry *task.TaskRegistry) error { + return registry.AddOrchestratorN("ReplayAnalysisUnsafe", replayAnalysisUnsafe) +} + +func replayAnalysisUnsafe(ctx *task.OrchestrationContext) (any, error) { + fmt.Println("printing from an orchestrator is replay-unsafe") + _, _ = http.Get("http://example.invalid") + ch := make(chan string, 1) + go func() { + ch <- uuid.NewString() + }() + select { + case value := <-ch: + return time.Now().String() + value, nil + default: + return time.Now().String(), nil + } +} diff --git a/samples/serviceoperations/README.md b/samples/serviceoperations/README.md new file mode 100644 index 00000000..3f59a095 --- /dev/null +++ b/samples/serviceoperations/README.md @@ -0,0 +1,44 @@ +# Isolated hub maintenance + +**This sample performs hub-wide purge and entity maintenance. Never point it at +a shared, production, or otherwise valuable hub.** Provision a new, disposable +hub whose name starts with `sample-`, and remove that resource through Azure's +control plane after the experiment. +The sample does not provision Azure resources or change permissions. + +## Run + +Use a DTS service that actually implements the operations below: + +```bash +export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=sample-;Authentication=DefaultAzure' +export DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1 +go run ./samples/serviceoperations +``` + +Both the explicit acknowledgement and the disposable-name prefix are required. +Filters and entity maintenance act on the whole hub, so unique instance IDs alone +are not adequate protection. + +## What it proves + +- **Filtered purge:** three completed subjects must become unreadable. +- **Entity maintenance:** empty state must be absent afterward and a nonempty + entity must remain unchanged. If a transient record exists before cleanup, + the sample also requires a reported removal and verifies that record disappears. + +Checks are bounded and failures are reported by operation. Ordinary instance +cleanup uses only the generated IDs; hub-wide operations are intentionally +restricted to the disposable hub. + +`SAMPLE_OK serviceoperations` is printed only if every operation and cleanup +succeeds. Unsupported APIs and success responses with no observable state change +produce a nonzero exit, not coverage. The current DTS emulator has known limitations; +see the [feature matrix](../../durabletaskscheduler/README.md#feature-matrix). +Some services eagerly remove empty entity records. The output explicitly reports +that case: it validates idempotency and live-state preservation, not a removal +that did not happen. It never claims a nonzero removal count for an empty set. + +Rewind, skip-graceful termination, and SDK task-hub create/delete are intentionally +not supported. Provision and remove task hubs through Azure's control plane or +CLI, not SDK lifecycle RPCs. diff --git a/samples/serviceoperations/serviceoperations.go b/samples/serviceoperations/serviceoperations.go new file mode 100644 index 00000000..8e45a81d --- /dev/null +++ b/samples/serviceoperations/serviceoperations.go @@ -0,0 +1,251 @@ +// Command serviceoperations verifies hub-wide maintenance in a disposable task hub. +package main + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } + fmt.Println("SAMPLE_OK serviceoperations") +} + +func run() (err error) { + options, err := dtssample.Options() + if err != nil { + return err + } + if os.Getenv("DTS_SAMPLE_ALLOW_HUB_MAINTENANCE") != "1" || + !strings.HasPrefix(options.TaskHubName, "sample-") { + return errors.New("administration requires DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1 and a disposable task hub whose name starts with sample-; never use a shared hub") + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + registry := task.NewTaskRegistry() + if err := registry.AddOrchestratorN("SampleAdminEcho", adminEcho); err != nil { + return err + } + if err := registry.AddEntityN("SampleAdminEntity", adminEntity); err != nil { + return err + } + app, err := dtssample.StartWithOptions(ctx, options, registry) + if err != nil { + return err + } + defer func() { err = errors.Join(err, app.Shutdown()) }() + + var failures []error + for _, check := range []struct { + name string + run func(context.Context) error + }{ + {"filtered purge", func(ctx context.Context) error { return verifyFilteredPurge(ctx, app.Client) }}, + {"entity maintenance", func(ctx context.Context) error { return verifyEntityMaintenance(ctx, app.Client) }}, + } { + checkCtx, stop := context.WithTimeout(ctx, 40*time.Second) + checkErr := check.run(checkCtx) + stop() + if checkErr != nil { + failures = append(failures, fmt.Errorf("%s: %w", check.name, checkErr)) + } else { + fmt.Printf("verified %s\n", check.name) + } + } + return errors.Join(failures...) +} + +func adminEcho(ctx *task.OrchestrationContext) (any, error) { + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + return input, nil +} + +func adminEntity(ctx *task.EntityContext) (any, error) { + switch ctx.Operation { + case "set": + return nil, ctx.SetState(1) + case "delete": + ctx.DeleteState() + return nil, nil + default: + return nil, fmt.Errorf("unknown operation %q", ctx.Operation) + } +} + +func completeEcho(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID) (*api.OrchestrationMetadata, error) { + if _, err := client.ScheduleNewOrchestration(ctx, "SampleAdminEcho", + api.WithInstanceID(id), api.WithInput("echo")); err != nil { + return nil, err + } + metadata, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return nil, err + } + if err := dtssample.RequireCompleted(metadata); err != nil { + return nil, err + } + var output string + if err := metadata.ReadOutput(&output); err != nil { + return nil, err + } + if output != "echo" { + return nil, fmt.Errorf("echo output=%q, want echo", output) + } + return metadata, nil +} + +func verifyFilteredPurge(ctx context.Context, client *durabletaskscheduler.Client) (err error) { + var ids []api.InstanceID + defer func() { err = errors.Join(err, dtssample.Cleanup(client, ids...)) }() + var first, last time.Time + for range 3 { + id := dtssample.NewInstanceID("filtered-purge") + ids = append(ids, id) + metadata, err := completeEcho(ctx, client, id) + if err != nil { + return err + } + if first.IsZero() || metadata.CreatedAt.Before(first) { + first = metadata.CreatedAt + } + if metadata.CreatedAt.After(last) { + last = metadata.CreatedAt + } + } + result, err := client.PurgeInstances(ctx, api.PurgeInstancesRequest{ + Filter: &api.PurgeInstanceFilter{ + CreatedTimeFrom: first.Add(-time.Second), CreatedTimeTo: last.Add(time.Second), + RuntimeStatus: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, + }, + }) + if err != nil { + return err + } + if !result.IsComplete || result.DeletedInstanceCount < len(ids) { + return fmt.Errorf("filtered purge did not remove the subjects: %+v", result) + } + for _, id := range ids { + if _, err := client.FetchOrchestrationMetadata(ctx, id); !errors.Is(err, api.ErrInstanceNotFound) { + return fmt.Errorf("purged instance %s remains readable or lookup failed: %v", id, err) + } + } + return nil +} + +func verifyEntityMaintenance(ctx context.Context, client *durabletaskscheduler.Client) (err error) { + id := api.NewEntityID("SampleAdminEntity", string(dtssample.NewInstanceID("entity-maintenance"))) + live := api.NewEntityID("SampleAdminEntity", string(dtssample.NewInstanceID("entity-preserved"))) + needsDelete := true + defer func() { + cleanupCtx, stop := context.WithTimeout(context.Background(), 15*time.Second) + defer stop() + for _, target := range []api.EntityID{live, id} { + if target == id && !needsDelete { + continue + } + cleanupErr := client.SignalEntity(cleanupCtx, target, "delete") + if cleanupErr == nil { + _, cleanupErr = waitForEntity(cleanupCtx, client, target, func(m *api.EntityMetadata) bool { + return m == nil || !m.HasState + }) + } + err = errors.Join(err, cleanupErr) + } + }() + if err := client.SignalEntity(ctx, live, "set"); err != nil { + return err + } + if _, err := waitForEntity(ctx, client, live, func(m *api.EntityMetadata) bool { + return m != nil && m.HasState && m.SerializedState == "1" + }); err != nil { + return err + } + if err := client.SignalEntity(ctx, id, "set"); err != nil { + return err + } + if _, err := waitForEntity(ctx, client, id, func(m *api.EntityMetadata) bool { + return m != nil && m.HasState && m.SerializedState == "1" + }); err != nil { + return err + } + if err := client.SignalEntity(ctx, id, "delete"); err != nil { + return err + } + _, err = waitForEntity(ctx, client, id, func(m *api.EntityMetadata) bool { + return m == nil || !m.HasState + }) + if err != nil { + return err + } + needsDelete = false + query := api.EntityQuery{InstanceIDStartsWith: id.String(), IncludeTransient: true} + before, err := client.QueryEntities(ctx, query) + if err != nil { + return err + } + removed := int32(0) + for { + cleaned, err := client.CleanEntityStorage(ctx) + if err != nil { + return err + } + removed += cleaned.EmptyEntitiesRemoved + after, err := client.QueryEntities(ctx, query) + if err != nil { + return err + } + if len(after.Entities) == 0 && (len(before.Entities) == 0 || removed > 0) { + preserved, err := client.GetEntity(ctx, live) + if err != nil { + return err + } + if preserved == nil || !preserved.HasState || preserved.SerializedState != "1" { + return errors.New("entity maintenance changed the nonempty entity") + } + if len(before.Entities) == 0 { + fmt.Println("empty entity was already removed by the service; verified maintenance idempotency and preservation of live state") + } else { + fmt.Printf("maintenance removed retained empty state; reported removals=%d\n", removed) + } + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("maintenance left %d transient records and reported %d removals: %w", + len(after.Entities), removed, ctx.Err()) + case <-time.After(200 * time.Millisecond): + } + } +} + +func waitForEntity(ctx context.Context, client *durabletaskscheduler.Client, id api.EntityID, ready func(*api.EntityMetadata) bool) (*api.EntityMetadata, error) { + for { + metadata, err := client.GetEntity(ctx, id) + if err != nil { + return nil, err + } + if ready(metadata) { + return metadata, nil + } + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-time.After(100 * time.Millisecond): + } + } +} diff --git a/samples/serviceoperations/serviceoperations_test.go b/samples/serviceoperations/serviceoperations_test.go new file mode 100644 index 00000000..91775300 --- /dev/null +++ b/samples/serviceoperations/serviceoperations_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "strings" + "testing" +) + +func TestRejectsUnsafeHubBeforeConnecting(t *testing.T) { + for _, test := range []struct { + name string + hub string + ack string + }{ + {name: "no acknowledgement", hub: "sample-disposable"}, + {name: "shared hub", hub: "tests", ack: "1"}, + {name: "invalid acknowledgement", hub: "sample-disposable", ack: "true"}, + } { + t.Run(test.name, func(t *testing.T) { + t.Setenv("DTS_CONNECTION_STRING", "Endpoint=http://127.0.0.1:1;TaskHub="+test.hub+";Authentication=None") + t.Setenv("DTS_SAMPLE_ALLOW_HUB_MAINTENANCE", test.ack) + err := run() + if err == nil || !strings.Contains(err.Error(), "disposable task hub") { + t.Fatalf("expected safety rejection before any connection: %v", err) + } + }) + } +} diff --git a/tests/samples/samples_test.go b/tests/samples/samples_test.go new file mode 100644 index 00000000..783f2782 --- /dev/null +++ b/tests/samples/samples_test.go @@ -0,0 +1,234 @@ +package samples_test + +import ( + "context" + "go/parser" + "go/token" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" +) + +type sample struct { + name string + group string + input string + timeout time.Duration +} + +// Keep this execution list aligned with the feature map in samples/README.md. +var catalogue = []sample{ + {name: "durabletaskscheduler", group: "emulator"}, + {name: "parallel", group: "emulator"}, + {name: "coroutines", group: "emulator"}, + {name: "timers", group: "emulator"}, + {name: "externalevents", group: "emulator", input: "Taylor\n"}, + {name: "suborchestrations", group: "emulator"}, + {name: "retries", group: "emulator"}, + {name: "continueasnew", group: "emulator"}, + {name: "management", group: "emulator"}, + {name: "scheduledtasks", group: "emulator"}, + {name: "versioning", group: "emulator"}, + {name: "entity", group: "emulator"}, + {name: "dataconverter", group: "emulator"}, + {name: "largepayloads", group: "storage"}, + {name: "history", group: "emulator"}, + {name: "observability", group: "emulator"}, + {name: "worker", group: "emulator"}, + {name: "authentication", group: "azure", timeout: 3 * time.Minute}, + {name: "distributedtracing", group: "telemetry"}, + {name: "exporthistory", group: "storage", timeout: 8 * time.Minute}, + {name: "replayanalysis", group: "emulator"}, + {name: "serviceoperations", group: "admin", timeout: 6 * time.Minute}, +} + +func repositoryRoot(t *testing.T) string { + t.Helper() + root, err := filepath.Abs("../..") + if err != nil { + t.Fatal(err) + } + return root +} + +func TestSampleCatalogue(t *testing.T) { + root := repositoryRoot(t) + listed := make(map[string]bool, len(catalogue)) + for _, entry := range catalogue { + if listed[entry.name] { + t.Fatalf("duplicate sample %q", entry.name) + } + listed[entry.name] = true + if _, err := os.Stat(filepath.Join(root, "samples", entry.name, "README.md")); err != nil { + t.Errorf("%s needs run instructions: %v", entry.name, err) + } + } + directories, err := os.ReadDir(filepath.Join(root, "samples")) + if err != nil { + t.Fatal(err) + } + for _, directory := range directories { + if !directory.IsDir() || directory.Name() == "internal" || directory.Name() == "testdata" { + continue + } + files, err := os.ReadDir(filepath.Join(root, "samples", directory.Name())) + if err != nil { + t.Fatal(err) + } + runnable := false + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".go") || strings.HasSuffix(file.Name(), "_test.go") { + continue + } + path := filepath.Join(root, "samples", directory.Name(), file.Name()) + parsed, err := parser.ParseFile(token.NewFileSet(), path, nil, parser.PackageClauseOnly) + if err != nil { + t.Fatal(err) + } + runnable = runnable || parsed.Name.Name == "main" + } + if runnable && !listed[directory.Name()] { + t.Errorf("runnable sample %q has no E2E catalogue entry", directory.Name()) + } + if runnable { + delete(listed, directory.Name()) + } + } + for name := range listed { + t.Errorf("catalogue entry %q has no runnable package", name) + } +} + +func TestSamplesE2E(t *testing.T) { + if os.Getenv("DTS_SAMPLES_E2E") != "1" { + t.Skip("set DTS_SAMPLES_E2E=1 to execute real sample programs; this skip is not E2E coverage") + } + group := os.Getenv("DTS_SAMPLES_GROUP") + if group == "" { + group = "all" + } + switch group { + case "all", "emulator", "storage", "telemetry", "azure", "admin": + default: + t.Fatalf("unknown DTS_SAMPLES_GROUP %q", group) + } + root := repositoryRoot(t) + for _, entry := range catalogue { + if group != "all" && entry.group != group { + continue + } + t.Run(entry.name, func(t *testing.T) { + environment := sampleEnvironment(t, entry.group) + binary := filepath.Join(t.TempDir(), entry.name) + if runtime.GOOS == "windows" { + binary += ".exe" + } + buildArgs := []string{"build", "-mod=readonly", "-o", binary} + if os.Getenv("DTS_SAMPLES_RACE") == "1" { + buildArgs = append(buildArgs, "-race") + } + buildArgs = append(buildArgs, ".") + buildCtx, stopBuild := context.WithTimeout(t.Context(), 3*time.Minute) + build := exec.CommandContext(buildCtx, "go", buildArgs...) + build.Dir = filepath.Join(root, "samples", entry.name) + build.Env = append(environment, "GOWORK=off") + output, err := build.CombinedOutput() + stopBuild() + if err != nil { + t.Fatalf("build %s: %v\n%s", entry.name, err, output) + } + + timeout := entry.timeout + if timeout == 0 { + timeout = 2 * time.Minute + } + runCtx, stopRun := context.WithTimeout(t.Context(), timeout) + command := exec.CommandContext(runCtx, binary) + command.Dir = build.Dir + command.Env = environment + command.Stdin = strings.NewReader(entry.input) + output, err = command.CombinedOutput() + stopRun() + t.Logf("%s\n%s", entry.name, output) + if err != nil { + t.Fatalf("sample %s did not validate end to end: %v", entry.name, err) + } + if !strings.Contains(string(output), "SAMPLE_OK "+entry.name) { + t.Fatalf("%s exited without its validation receipt", entry.name) + } + if entry.name == "replayanalysis" { + checkReplayAnalysis(t, root) + } + }) + } +} + +func sampleEnvironment(t *testing.T, group string) []string { + t.Helper() + connectionVariable := "DTS_CONNECTION_STRING" + switch group { + case "azure": + connectionVariable = "DTS_SAMPLES_AZURE_CONNECTION_STRING" + case "admin": + connectionVariable = "DTS_SAMPLES_ADMIN_CONNECTION_STRING" + if requiredEnvironment(t, "DTS_SAMPLE_ALLOW_HUB_MAINTENANCE") != "1" { + t.Fatal("admin E2E requires explicit hub-maintenance acknowledgement") + } + case "telemetry": + requiredEnvironment(t, "OTEL_EXPORTER_OTLP_ENDPOINT") + requiredEnvironment(t, "OTEL_CAPTURE_FILE") + } + environment := append(os.Environ(), "DTS_CONNECTION_STRING="+requiredEnvironment(t, connectionVariable)) + if group == "storage" { + if requiredEnvironment(t, "DTS_SAMPLE_ISOLATED_TASKHUB") != "1" { + t.Fatal("storage E2E includes whole-window history export and requires an isolated task hub") + } + connection := os.Getenv("AZURE_STORAGE_CONNECTION_STRING") + if connection == "" { + connection = requiredEnvironment(t, "EXPORT_STORAGE_CONNECTION_STRING") + } + environment = append(environment, "AZURE_STORAGE_CONNECTION_STRING="+connection, + "EXPORT_STORAGE_CONNECTION_STRING="+connection) + } + return environment +} + +func requiredEnvironment(t *testing.T, name string) string { + t.Helper() + value := os.Getenv(name) + if value == "" { + t.Fatalf("%s is required for the selected E2E group; missing configuration is not a pass", name) + } + return value +} + +func checkReplayAnalysis(t *testing.T, root string) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) + defer cancel() + tool := filepath.Join(t.TempDir(), "orchestratorvet") + if runtime.GOOS == "windows" { + tool += ".exe" + } + build := exec.CommandContext(ctx, "go", "build", "-mod=readonly", "-o", tool, ".") + build.Dir = filepath.Join(root, "cmd", "orchestratorvet") + build.Env = append(os.Environ(), "GOWORK=off") + if output, err := build.CombinedOutput(); err != nil { + t.Fatalf("build replay analyzer: %v\n%s", err, output) + } + good := exec.CommandContext(ctx, "go", "vet", "-vettool="+tool, "./samples/replayanalysis") + good.Dir = root + if output, err := good.CombinedOutput(); err != nil { + t.Fatalf("safe replay sample has diagnostics: %v\n%s", err, output) + } + bad := exec.CommandContext(ctx, "go", "vet", "-vettool="+tool, "./samples/replayanalysis/testdata/bad") + bad.Dir = root + output, err := bad.CombinedOutput() + if err == nil || !strings.Contains(string(output), "not deterministic") { + t.Fatalf("unsafe fixture must produce a replay diagnostic, got %v\n%s", err, output) + } +} From 16b9643bb013ec7291d7355c978e18b48cb7f4ab Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:24:05 -0700 Subject: [PATCH 09/26] validate stored compression independently of HTTP decoding Check blob properties for the storage compression setting while accepting transport-decompressed downloads. Cover both compressed response forms and reject incorrect stored encoding. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/largepayloads/largepayloads.go | 12 +++- samples/largepayloads/largepayloads_test.go | 74 ++++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/samples/largepayloads/largepayloads.go b/samples/largepayloads/largepayloads.go index 5cbb8682..766b2829 100644 --- a/samples/largepayloads/largepayloads.go +++ b/samples/largepayloads/largepayloads.go @@ -294,6 +294,15 @@ func verifyStoredPayloads( ) error { sawHash := false for name := range names { + properties, err := client.ServiceClient().NewContainerClient(container).NewBlobClient(name).GetProperties(ctx, nil) + if err != nil { + return fmt.Errorf("read payload blob %s properties: %w", name, err) + } + // A download may be transparently decompressed by the HTTP transport. + storedGzip := properties.ContentEncoding != nil && strings.EqualFold(*properties.ContentEncoding, "gzip") + if storedGzip != wantGzip { + return fmt.Errorf("payload blob %s stored gzip=%t, want %t", name, storedGzip, wantGzip) + } response, err := client.DownloadStream(ctx, container, name, nil) if err != nil { return fmt.Errorf("download payload blob %s: %w", name, err) @@ -304,9 +313,6 @@ func verifyStoredPayloads( return errors.Join(readErr, closeErr) } isGzip := response.ContentEncoding != nil && strings.EqualFold(*response.ContentEncoding, "gzip") - if isGzip != wantGzip { - return fmt.Errorf("payload blob %s gzip=%t, want %t", name, isGzip, wantGzip) - } if isGzip { reader, err := gzip.NewReader(bytes.NewReader(body)) if err != nil { diff --git a/samples/largepayloads/largepayloads_test.go b/samples/largepayloads/largepayloads_test.go index 010df51a..0edabf29 100644 --- a/samples/largepayloads/largepayloads_test.go +++ b/samples/largepayloads/largepayloads_test.go @@ -1,6 +1,78 @@ package main -import "testing" +import ( + "bytes" + "compress/gzip" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" +) + +func TestStoredCompressionIsIndependentOfDownloadDecompression(t *testing.T) { + const hash = "expected-payload-hash" + content := []byte(strings.Repeat(hash, 10)) + var compressed bytes.Buffer + writer := gzip.NewWriter(&compressed) + if _, err := writer.Write(content); err != nil { + t.Fatal(err) + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name string + storedGzip bool + wantGzip bool + disableDecompression bool + }{ + {name: "automatically decompressed gzip", storedGzip: true, wantGzip: true}, + {name: "raw gzip", storedGzip: true, wantGzip: true, disableDecompression: true}, + {name: "uncompressed"}, + {name: "reject unexpected plain storage", wantGzip: true}, + {name: "reject unexpected compressed storage", storedGzip: true}, + } { + t.Run(test.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + body := content + if test.storedGzip { + body = compressed.Bytes() + w.Header().Set("Content-Encoding", "gzip") + } + w.Header().Set("Content-Length", strconv.Itoa(len(body))) + w.WriteHeader(http.StatusOK) + if r.Method != http.MethodHead { + if _, err := w.Write(body); err != nil { + t.Error(err) + } + } + })) + defer server.Close() + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.DisableCompression = test.disableDecompression + defer transport.CloseIdleConnections() + client, err := azblob.NewClientWithNoCredential(server.URL, &azblob.ClientOptions{ + ClientOptions: azcore.ClientOptions{Transport: &http.Client{Transport: transport}}, + }) + if err != nil { + t.Fatal(err) + } + err = verifyStoredPayloads(t.Context(), client, "container", + map[string]struct{}{"blob": {}}, hash, test.wantGzip) + if test.storedGzip == test.wantGzip { + if err != nil { + t.Fatal(err) + } + } else if err == nil || !strings.Contains(err.Error(), "stored gzip=") { + t.Fatalf("expected stored-compression mismatch, got %v", err) + } + }) + } +} func TestAllowInsecureStorageRequiresLoopbackOptIn(t *testing.T) { t.Setenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE", "") From c0fde4e6355eca77f7aa6d579fbc591593e63bf3 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:44:58 -0700 Subject: [PATCH 10/26] simplify samples to a flat named catalogue Remove execution groups and special Azure/admin connection routing. Run examples with their normal environment and select E2E subtests with Go flags; keep per-sample prerequisite and safety checks. Simplify CI to one isolated service setup and update the sample index. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/samples-e2e.yml | 16 ++--- CHANGELOG.md | 2 +- README.md | 2 +- samples/README.md | 112 ++++++++++++++++-------------- tests/samples/samples_test.go | 98 ++++++-------------------- 5 files changed, 89 insertions(+), 141 deletions(-) diff --git a/.github/workflows/samples-e2e.yml b/.github/workflows/samples-e2e.yml index 2a7db11e..71415fac 100644 --- a/.github/workflows/samples-e2e.yml +++ b/.github/workflows/samples-e2e.yml @@ -27,14 +27,9 @@ jobs: samples: runs-on: ubuntu-latest timeout-minutes: 25 - strategy: - fail-fast: false - matrix: - group: [emulator, storage, telemetry] env: DTS_CONNECTION_STRING: "Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None" DTS_SAMPLES_E2E: "1" - DTS_SAMPLES_GROUP: ${{ matrix.group }} DTS_SAMPLES_RACE: "1" steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -51,7 +46,6 @@ jobs: curl --fail --retry 30 --retry-delay 1 --retry-all-errors http://127.0.0.1:8082/ >/dev/null - name: Start isolated blob storage - if: matrix.group == 'storage' shell: bash run: | key="$(openssl rand -base64 32)" @@ -62,13 +56,11 @@ jobs: mcr.microsoft.com/azure-storage/azurite:3.37.0 connection="DefaultEndpointsProtocol=http;AccountName=sampleacct;AccountKey=$key;BlobEndpoint=http://127.0.0.1:10000/sampleacct;" echo "::add-mask::$connection" - printf 'AZURE_STORAGE_CONNECTION_STRING=%s\nEXPORT_STORAGE_CONNECTION_STRING=%s\n' \ - "$connection" "$connection" >> "$GITHUB_ENV" + printf 'AZURE_STORAGE_CONNECTION_STRING=%s\n' "$connection" >> "$GITHUB_ENV" printf 'DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1\nDTS_SAMPLE_ISOLATED_TASKHUB=1\n' >> "$GITHUB_ENV" timeout 60 bash -c 'until - + go test ./tests/samples -run 'TestSampleCatalogue|TestSamplesE2E' + -skip '^TestSamplesE2E/(authentication|serviceoperations)$' + -count=1 -timeout=20m -v - name: Service diagnostics if: failure() diff --git a/CHANGELOG.md b/CHANGELOG.md index feba3a27..a3cadb2c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- Added a runnable DTS feature-family sample catalogue with outcome assertions, owned-resource cleanup, and process-level E2E groups for DTS, blob storage, telemetry, Azure authentication, and isolated hub maintenance. +- Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag. - Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, distinct activity and sub-orchestration action trace contexts, and legacy entity-operation trace forwarding. - Added the top-level `durabletaskscheduler` transport package, a dedicated resilient gRPC worker, DTS emulator tests, and an environment-driven sample. - Added advanced management APIs for bounded instance queries/listing, restart, and batch/filter purge polling. diff --git a/README.md b/README.md index 6aace722..3e957b6f 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ For more information, read the [DTS transport guide and feature matrix](./durabl Each sample connects to the task hub in `DTS_CONNECTION_STRING`. Set that variable first. Then run `go run ./samples/`. The [complete feature-to-sample catalogue](./samples/README.md) covers the SDK's -feature families and documents actual end-to-end execution groups. These are +feature families and documents how to run and validate each example. These are useful starting points: | Sample | Description | diff --git a/samples/README.md b/samples/README.md index c04efa2f..45c149e3 100644 --- a/samples/README.md +++ b/samples/README.md @@ -8,30 +8,30 @@ operation is not evidence that a feature works end to end. ## Feature coverage -| Sample | Feature family | E2E group | +| Sample | What it demonstrates | Additional prerequisites | | --- | --- | --- | -| [durabletaskscheduler](durabletaskscheduler) | Orchestrations, activities, typed input/output, start/wait | emulator | -| [parallel](parallel) | Fan-out/fan-in, WhenAll/WhenAny, aggregation | emulator | -| [coroutines](coroutines) | Durable coroutines, wait groups, selection, cancellation scopes | emulator | -| [timers](timers) | Durable delays, scheduled starts, timer splitting, deterministic time/IDs | emulator | -| [externalevents](externalevents) | Typed and repeated events, timeouts, cross-instance events | emulator | -| [suborchestrations](suborchestrations) | Child workflows, results, IDs, metadata and failures | emulator | -| [retries](retries) | Controlled retries, retry handlers, typed/non-retriable failures | emulator | -| [continueasnew](continueasnew) | Checkpoints, event carryover, continuation and history/turn budgets | emulator | -| [management](management) | Progress, queries/paging, lifecycle, restart, ID reuse, ID-scoped purge | emulator | -| [scheduledtasks](scheduledtasks) | Actual scheduled execution and schedule lifecycle | emulator | -| [versioning](versioning) | Worker versions, inheritance, fallback, migration and routing | emulator | -| [entity](entity) | State, signals/calls, registration models, factories and locked transfers | emulator | -| [dataconverter](dataconverter) | Custom serialization and raw versus typed payloads | emulator | -| [largepayloads](largepayloads) | Blob externalization, hydration, compression and bounds | storage | -| [history](history) | Buffered/streaming history, execution identity, typed readers and limits | emulator | -| [observability](observability) | Tags/context, replay-safe logs and metric hooks | emulator | -| [worker](worker) | Concurrency, graceful drain, restart/recovery and connection ownership | emulator | -| [authentication](authentication) | Real Azure connection-string and programmatic identity authentication | azure | -| [distributedtracing](distributedtracing) | Application spans, caller propagation and actual OTLP receipt | telemetry | -| [exporthistory](exporthistory) | Export jobs and verification of downloaded JSON/JSONL objects | storage | -| [replayanalysis](replayanalysis) | Real analyzer diagnostics and safe runnable counterparts | emulator | -| [serviceoperations](serviceoperations) | Hub-wide filtered purge and empty-entity maintenance | admin | +| [durabletaskscheduler](durabletaskscheduler) | Orchestrations, activities, typed input/output, start/wait | None | +| [parallel](parallel) | Fan-out/fan-in, WhenAll/WhenAny, aggregation | None | +| [coroutines](coroutines) | Durable coroutines, wait groups, selection, cancellation scopes | None | +| [timers](timers) | Durable delays, scheduled starts, timer splitting, deterministic time/IDs | None | +| [externalevents](externalevents) | Typed and repeated events, timeouts, cross-instance events | Piped or interactive input | +| [suborchestrations](suborchestrations) | Child workflows, results, IDs, metadata and failures | None | +| [retries](retries) | Controlled retries, retry handlers, typed/non-retriable failures | None | +| [continueasnew](continueasnew) | Checkpoints, event carryover, continuation and history/turn budgets | None | +| [management](management) | Progress, queries/paging, lifecycle, restart, ID reuse, ID-scoped purge | None | +| [scheduledtasks](scheduledtasks) | Actual scheduled execution and schedule lifecycle | Go-only task hub | +| [versioning](versioning) | Worker versions, inheritance, fallback, migration and routing | None | +| [entity](entity) | State, signals/calls, registration models, factories and locked transfers | None | +| [dataconverter](dataconverter) | Custom serialization and raw versus typed payloads | None | +| [largepayloads](largepayloads) | Blob externalization, hydration, compression and bounds | Azure Blob Storage or Azurite | +| [history](history) | Buffered/streaming history, execution identity, typed readers and limits | None | +| [observability](observability) | Tags/context, replay-safe logs and metric hooks | None | +| [worker](worker) | Concurrency, graceful drain, restart/recovery and connection ownership | None | +| [authentication](authentication) | Azure connection-string and programmatic identity authentication | Azure DTS and an authorized identity | +| [distributedtracing](distributedtracing) | Application spans, caller propagation and actual OTLP receipt | OTLP collector and readable trace capture | +| [exporthistory](exporthistory) | Export jobs and downloaded JSON/JSONL objects | Blob storage and an explicitly isolated Go-only hub | +| [replayanalysis](replayanalysis) | Analyzer diagnostics and safe runnable counterparts | Local `cmd/orchestratorvet` sources | +| [serviceoperations](serviceoperations) | Hub-wide filtered purge and empty-entity maintenance | Disposable hub and explicit maintenance acknowledgement | Protocol bookkeeping, legacy wire compatibility and unsupported sandbox worker profiles are not separate runnable features. SDK regression tests cover the @@ -39,11 +39,13 @@ internal contracts. The catalogue demonstrates user-facing behavior. ## Run an example -Go 1.25 or later is required. Point the normal examples at an isolated DTS emulator: +Go 1.25 or later is required. Choose an example and configure `DTS_CONNECTION_STRING`. +Most examples can use either the emulator or Azure DTS: ```bash export DTS_CONNECTION_STRING='Endpoint=http://127.0.0.1:8080;TaskHub=default;Authentication=None' go run ./samples/durabletaskscheduler +go run ./samples/retries ``` Run commands are relative to the repository root unless a sample README says @@ -60,46 +62,48 @@ once; failure injection is a teaching device, not a production idempotency store ## Execute the actual programs end to end -The standard-library runner in `tests/samples` builds and runs each selected -entry point as a separate process. It verifies exit status and the sample's -validation receipt. It also checks that every runnable sample has a catalogue -entry and run instructions. +The runner in `tests/samples` builds and executes the same programs, with their +normal environment variables. Select samples using Go's standard `-run` flag: ```bash -DTS_SAMPLES_E2E=1 DTS_SAMPLES_GROUP=emulator \ - go test ./tests/samples -run 'TestSampleCatalogue|TestSamplesE2E' -count=1 -timeout=20m -v +DTS_SAMPLES_E2E=1 \ + go test ./tests/samples -run '^TestSamplesE2E/retries$' -count=1 -v + +DTS_SAMPLES_E2E=1 \ + go test ./tests/samples -run '^TestSamplesE2E/(timers|parallel)$' -count=1 -v ``` Set `DTS_SAMPLES_RACE=1` to build the sample processes with the race detector. The external-event sample receives piped input automatically. The replay-analysis -entry also runs the real analyzer against its safe and intentionally unsafe code. -There is no fake service or hand-built history substituting for these executions. - -| Group | Required configuration | -| --- | --- | -| `emulator` | `DTS_CONNECTION_STRING` pointing at isolated DTS | -| `storage` | Isolated DTS with `DTS_SAMPLE_ISOLATED_TASKHUB=1`, plus `AZURE_STORAGE_CONNECTION_STRING` (`EXPORT_STORAGE_CONNECTION_STRING` is also accepted); set `DTS_SAMPLE_ALLOW_INSECURE_STORAGE=1` for loopback Azurite | -| `telemetry` | DTS, `OTEL_EXPORTER_OTLP_ENDPOINT`, and `OTEL_CAPTURE_FILE` written by the real collector | -| `azure` | `DTS_SAMPLES_AZURE_CONNECTION_STRING` and an authorized Azure Identity credential | -| `admin` | `DTS_SAMPLES_ADMIN_CONNECTION_STRING` for a **disposable `sample-*` hub** and `DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1` | -| `all` | All of the above; each process gets the appropriate DTS target | +sample also runs the real analyzer against safe and intentionally unsafe code. +The runner preserves timeouts, checks exit status and the sample's validation +receipt, and checks that every runnable sample has an entry and README. -For example, run the same authentication executable against a real Azure hub: +Each sample validates its own prerequisites, identically whether invoked through +`go run` or the E2E runner. For example: ```bash -export DTS_SAMPLES_AZURE_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' -DTS_SAMPLES_E2E=1 DTS_SAMPLES_GROUP=azure \ - go test ./tests/samples -run TestSamplesE2E -count=1 -timeout=5m -v +export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' +go run ./samples/authentication + +DTS_SAMPLES_E2E=1 \ + go test ./tests/samples -run '^TestSamplesE2E/authentication$' -count=1 -timeout=5m -v ``` -When E2E is explicitly enabled, missing required configuration fails the selected -group instead of silently skipping it. Ordinary `go test ./...` leaves these -external-service runs disabled; that default skip does not count as E2E coverage. +There is no target switching in the runner: every selected sample inherits the +same `DTS_CONNECTION_STRING`. Run maintenance separately on a disposable hub. +Follow the individual READMEs for storage, telemetry and safety settings; the +runner does not invent aliases, supply credentials or relax those checks. + +Missing required configuration fails the selected example rather than silently +skipping it. Ordinary `go test ./...` leaves external-service runs disabled; that +default skip does not count as E2E coverage. -The `Sample E2E` workflow runs emulator, storage and telemetry groups on real -isolated services. It generates temporary Azurite credentials rather than -committing account keys. Cloud groups are deliberately separate: use an approved -identity and target, not credentials exposed to untrusted pull requests. +The `Sample E2E` workflow provisions isolated DTS, Azurite and an OTLP collector, +then runs the local examples using Go's test selection flags. It explicitly +excludes `authentication` and `serviceoperations`, which need approved Azure +targets. Temporary Azurite credentials are generated at runtime; cloud credentials +are not exposed to untrusted pull requests. ## Capability and safety boundaries @@ -114,8 +118,8 @@ or a missing precondition remains failed/blocked coverage. Do not turn such results into a success or a skipped test just to make a dashboard green. Hub-wide purge and entity maintenance are destructive. -The `admin` group must use a newly provisioned disposable hub, never the hub -used for ordinary shared examples. Remove that temporary control-plane resource +The `serviceoperations` example must use a newly provisioned disposable hub, never +the hub used for ordinary shared examples. Remove that temporary control-plane resource afterward through Azure's control plane. Storage/export examples must use isolated destinations and task hubs so they cannot export or delete unrelated application data. diff --git a/tests/samples/samples_test.go b/tests/samples/samples_test.go index 783f2782..0234a559 100644 --- a/tests/samples/samples_test.go +++ b/tests/samples/samples_test.go @@ -15,35 +15,34 @@ import ( type sample struct { name string - group string input string timeout time.Duration } // Keep this execution list aligned with the feature map in samples/README.md. var catalogue = []sample{ - {name: "durabletaskscheduler", group: "emulator"}, - {name: "parallel", group: "emulator"}, - {name: "coroutines", group: "emulator"}, - {name: "timers", group: "emulator"}, - {name: "externalevents", group: "emulator", input: "Taylor\n"}, - {name: "suborchestrations", group: "emulator"}, - {name: "retries", group: "emulator"}, - {name: "continueasnew", group: "emulator"}, - {name: "management", group: "emulator"}, - {name: "scheduledtasks", group: "emulator"}, - {name: "versioning", group: "emulator"}, - {name: "entity", group: "emulator"}, - {name: "dataconverter", group: "emulator"}, - {name: "largepayloads", group: "storage"}, - {name: "history", group: "emulator"}, - {name: "observability", group: "emulator"}, - {name: "worker", group: "emulator"}, - {name: "authentication", group: "azure", timeout: 3 * time.Minute}, - {name: "distributedtracing", group: "telemetry"}, - {name: "exporthistory", group: "storage", timeout: 8 * time.Minute}, - {name: "replayanalysis", group: "emulator"}, - {name: "serviceoperations", group: "admin", timeout: 6 * time.Minute}, + {name: "durabletaskscheduler"}, + {name: "parallel"}, + {name: "coroutines"}, + {name: "timers"}, + {name: "externalevents", input: "Taylor\n"}, + {name: "suborchestrations"}, + {name: "retries"}, + {name: "continueasnew"}, + {name: "management"}, + {name: "scheduledtasks"}, + {name: "versioning"}, + {name: "entity"}, + {name: "dataconverter"}, + {name: "largepayloads"}, + {name: "history"}, + {name: "observability"}, + {name: "worker"}, + {name: "authentication", timeout: 3 * time.Minute}, + {name: "distributedtracing"}, + {name: "exporthistory", timeout: 8 * time.Minute}, + {name: "replayanalysis"}, + {name: "serviceoperations", timeout: 6 * time.Minute}, } func repositoryRoot(t *testing.T) string { @@ -107,22 +106,10 @@ func TestSamplesE2E(t *testing.T) { if os.Getenv("DTS_SAMPLES_E2E") != "1" { t.Skip("set DTS_SAMPLES_E2E=1 to execute real sample programs; this skip is not E2E coverage") } - group := os.Getenv("DTS_SAMPLES_GROUP") - if group == "" { - group = "all" - } - switch group { - case "all", "emulator", "storage", "telemetry", "azure", "admin": - default: - t.Fatalf("unknown DTS_SAMPLES_GROUP %q", group) - } root := repositoryRoot(t) for _, entry := range catalogue { - if group != "all" && entry.group != group { - continue - } t.Run(entry.name, func(t *testing.T) { - environment := sampleEnvironment(t, entry.group) + environment := os.Environ() binary := filepath.Join(t.TempDir(), entry.name) if runtime.GOOS == "windows" { binary += ".exe" @@ -167,45 +154,6 @@ func TestSamplesE2E(t *testing.T) { } } -func sampleEnvironment(t *testing.T, group string) []string { - t.Helper() - connectionVariable := "DTS_CONNECTION_STRING" - switch group { - case "azure": - connectionVariable = "DTS_SAMPLES_AZURE_CONNECTION_STRING" - case "admin": - connectionVariable = "DTS_SAMPLES_ADMIN_CONNECTION_STRING" - if requiredEnvironment(t, "DTS_SAMPLE_ALLOW_HUB_MAINTENANCE") != "1" { - t.Fatal("admin E2E requires explicit hub-maintenance acknowledgement") - } - case "telemetry": - requiredEnvironment(t, "OTEL_EXPORTER_OTLP_ENDPOINT") - requiredEnvironment(t, "OTEL_CAPTURE_FILE") - } - environment := append(os.Environ(), "DTS_CONNECTION_STRING="+requiredEnvironment(t, connectionVariable)) - if group == "storage" { - if requiredEnvironment(t, "DTS_SAMPLE_ISOLATED_TASKHUB") != "1" { - t.Fatal("storage E2E includes whole-window history export and requires an isolated task hub") - } - connection := os.Getenv("AZURE_STORAGE_CONNECTION_STRING") - if connection == "" { - connection = requiredEnvironment(t, "EXPORT_STORAGE_CONNECTION_STRING") - } - environment = append(environment, "AZURE_STORAGE_CONNECTION_STRING="+connection, - "EXPORT_STORAGE_CONNECTION_STRING="+connection) - } - return environment -} - -func requiredEnvironment(t *testing.T, name string) string { - t.Helper() - value := os.Getenv(name) - if value == "" { - t.Fatalf("%s is required for the selected E2E group; missing configuration is not a pass", name) - } - return value -} - func checkReplayAnalysis(t *testing.T, root string) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 2*time.Minute) From ad6daf944649fc5c8b8e13259467f5c682afa662 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 09:56:22 -0700 Subject: [PATCH 11/26] remove sample-specific maintenance opt-in requirements Use the standard connection-string setup without a mandatory hub-name prefix or acknowledgement variable. Retain a brief hub-wide cleanup warning and remove tests for the deleted restrictions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 2 +- samples/README.md | 9 +++---- samples/serviceoperations/README.md | 23 +++++++--------- .../serviceoperations/serviceoperations.go | 14 ++-------- .../serviceoperations_test.go | 27 ------------------- 5 files changed, 16 insertions(+), 59 deletions(-) delete mode 100644 samples/serviceoperations/serviceoperations_test.go diff --git a/README.md b/README.md index 3e957b6f..96cc0d01 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ Some samples need additional services: - `distributedtracing` is a separate Go module and uses a local OTLP collector. Run it with `cd samples/distributedtracing && go run .`. - `exporthistory` and `largepayloads` need blob storage; see their READMEs for the storage connection variables. -- `authentication` requires a real Azure identity and DTS endpoint. Hub-wide administrative examples require a disposable hub and explicit acknowledgement. +- `authentication` requires a real Azure identity and DTS endpoint. `serviceoperations` performs hub-wide cleanup; read its warning before running it. Most samples use the shared helper [`samples/internal/dtssample`](./samples/internal/dtssample). The helper reads `DTS_CONNECTION_STRING`, opens the client, and starts the worker. The `exporthistory` sample does not use the helper. That sample needs the client before it registers the export system tasks. diff --git a/samples/README.md b/samples/README.md index 45c149e3..09e8dc9f 100644 --- a/samples/README.md +++ b/samples/README.md @@ -31,7 +31,7 @@ operation is not evidence that a feature works end to end. | [distributedtracing](distributedtracing) | Application spans, caller propagation and actual OTLP receipt | OTLP collector and readable trace capture | | [exporthistory](exporthistory) | Export jobs and downloaded JSON/JSONL objects | Blob storage and an explicitly isolated Go-only hub | | [replayanalysis](replayanalysis) | Analyzer diagnostics and safe runnable counterparts | Local `cmd/orchestratorvet` sources | -| [serviceoperations](serviceoperations) | Hub-wide filtered purge and empty-entity maintenance | Disposable hub and explicit maintenance acknowledgement | +| [serviceoperations](serviceoperations) | Hub-wide filtered purge and empty-entity maintenance | Deletes data; read the sample's warning | Protocol bookkeeping, legacy wire compatibility and unsupported sandbox worker profiles are not separate runnable features. SDK regression tests cover the @@ -91,7 +91,7 @@ DTS_SAMPLES_E2E=1 \ ``` There is no target switching in the runner: every selected sample inherits the -same `DTS_CONNECTION_STRING`. Run maintenance separately on a disposable hub. +same `DTS_CONNECTION_STRING`. Run maintenance separately: it performs hub-wide cleanup. Follow the individual READMEs for storage, telemetry and safety settings; the runner does not invent aliases, supply credentials or relax those checks. @@ -118,8 +118,7 @@ or a missing precondition remains failed/blocked coverage. Do not turn such results into a success or a skipped test just to make a dashboard green. Hub-wide purge and entity maintenance are destructive. -The `serviceoperations` example must use a newly provisioned disposable hub, never -the hub used for ordinary shared examples. Remove that temporary control-plane resource -afterward through Azure's control plane. Storage/export examples must +Read the `serviceoperations` warning before running it against a task hub. +Storage/export examples must use isolated destinations and task hubs so they cannot export or delete unrelated application data. diff --git a/samples/serviceoperations/README.md b/samples/serviceoperations/README.md index 3f59a095..3e2889c8 100644 --- a/samples/serviceoperations/README.md +++ b/samples/serviceoperations/README.md @@ -1,24 +1,19 @@ -# Isolated hub maintenance +# Hub maintenance -**This sample performs hub-wide purge and entity maintenance. Never point it at -a shared, production, or otherwise valuable hub.** Provision a new, disposable -hub whose name starts with `sample-`, and remove that resource through Azure's -control plane after the experiment. -The sample does not provision Azure resources or change permissions. +> **Warning:** This sample performs hub-wide cleanup and can delete data outside +> its own instances. Use a test hub, not production or valuable application data. ## Run Use a DTS service that actually implements the operations below: ```bash -export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=sample-;Authentication=DefaultAzure' -export DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1 +export DTS_CONNECTION_STRING='Endpoint=https://;TaskHub=;Authentication=DefaultAzure' go run ./samples/serviceoperations ``` -Both the explicit acknowledgement and the disposable-name prefix are required. -Filters and entity maintenance act on the whole hub, so unique instance IDs alone -are not adequate protection. +No special hub name or additional opt-in variable is required. The sample does +not provision Azure resources or change permissions. ## What it proves @@ -27,9 +22,9 @@ are not adequate protection. entity must remain unchanged. If a transient record exists before cleanup, the sample also requires a reported removal and verifies that record disappears. -Checks are bounded and failures are reported by operation. Ordinary instance -cleanup uses only the generated IDs; hub-wide operations are intentionally -restricted to the disposable hub. +Checks are bounded and failures are reported by operation. Cleanup of the +sample's instances uses their generated IDs; the demonstrated filter and entity +maintenance APIs act across the selected hub. `SAMPLE_OK serviceoperations` is printed only if every operation and cleanup succeeds. Unsupported APIs and success responses with no observable state change diff --git a/samples/serviceoperations/serviceoperations.go b/samples/serviceoperations/serviceoperations.go index 8e45a81d..3c3617b6 100644 --- a/samples/serviceoperations/serviceoperations.go +++ b/samples/serviceoperations/serviceoperations.go @@ -1,4 +1,4 @@ -// Command serviceoperations verifies hub-wide maintenance in a disposable task hub. +// Command serviceoperations verifies hub-wide purge and entity maintenance. package main import ( @@ -6,8 +6,6 @@ import ( "errors" "fmt" "log" - "os" - "strings" "time" "github.com/microsoft/durabletask-go/api" @@ -24,14 +22,6 @@ func main() { } func run() (err error) { - options, err := dtssample.Options() - if err != nil { - return err - } - if os.Getenv("DTS_SAMPLE_ALLOW_HUB_MAINTENANCE") != "1" || - !strings.HasPrefix(options.TaskHubName, "sample-") { - return errors.New("administration requires DTS_SAMPLE_ALLOW_HUB_MAINTENANCE=1 and a disposable task hub whose name starts with sample-; never use a shared hub") - } ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) defer cancel() registry := task.NewTaskRegistry() @@ -41,7 +31,7 @@ func run() (err error) { if err := registry.AddEntityN("SampleAdminEntity", adminEntity); err != nil { return err } - app, err := dtssample.StartWithOptions(ctx, options, registry) + app, err := dtssample.Start(ctx, registry) if err != nil { return err } diff --git a/samples/serviceoperations/serviceoperations_test.go b/samples/serviceoperations/serviceoperations_test.go deleted file mode 100644 index 91775300..00000000 --- a/samples/serviceoperations/serviceoperations_test.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import ( - "strings" - "testing" -) - -func TestRejectsUnsafeHubBeforeConnecting(t *testing.T) { - for _, test := range []struct { - name string - hub string - ack string - }{ - {name: "no acknowledgement", hub: "sample-disposable"}, - {name: "shared hub", hub: "tests", ack: "1"}, - {name: "invalid acknowledgement", hub: "sample-disposable", ack: "true"}, - } { - t.Run(test.name, func(t *testing.T) { - t.Setenv("DTS_CONNECTION_STRING", "Endpoint=http://127.0.0.1:1;TaskHub="+test.hub+";Authentication=None") - t.Setenv("DTS_SAMPLE_ALLOW_HUB_MAINTENANCE", test.ack) - err := run() - if err == nil || !strings.Contains(err.Error(), "disposable task hub") { - t.Fatalf("expected safety rejection before any connection: %v", err) - } - }) - } -} From 1a3db831f8801d1f7907acb839938e55610ae8c2 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:23:48 -0700 Subject: [PATCH 12/26] fix: restore rewind with Python-aligned recovery Return replacement history before replay for terminal rewind requests, preserving successful work and renewing execution identity. Restore the enqueue API, cover recursive recovery and transport contracts, and document the shared retry-timer limitation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 3 +- README.md | 12 +- api/management.go | 12 + client/client_grpc_management.go | 27 ++ client/client_grpc_management_test.go | 8 + client/client_grpc_rewind_test.go | 117 ++++++++ client/grpc_worker_rewind_test.go | 101 +++++++ durabletaskscheduler/README.md | 38 ++- samples/rewind/rewind.go | 132 +++++++++ task/executor.go | 3 + task/rewind.go | 86 ++++++ task/rewind_test.go | 168 ++++++++++++ tests/durabletaskscheduler/dts_rewind_test.go | 256 ++++++++++++++++++ 13 files changed, 959 insertions(+), 4 deletions(-) create mode 100644 client/client_grpc_rewind_test.go create mode 100644 client/grpc_worker_rewind_test.go create mode 100644 samples/rewind/rewind.go create mode 100644 task/rewind.go create mode 100644 task/rewind_test.go create mode 100644 tests/durabletaskscheduler/dts_rewind_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 203cf736..61d254c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, distinct activity and sub-orchestration action trace contexts, and legacy entity-operation trace forwarding. - Added the top-level `durabletaskscheduler` transport package, a dedicated resilient gRPC worker, DTS emulator tests, and an environment-driven sample. - Added advanced management APIs for bounded instance queries/listing, restart, and batch/filter purge polling. +- Restored `RewindInstance`, `api.RewindOptions`, and `api.WithRewindReason` with request validation and typed errors. Added Python-aligned worker history replacement for real failed-activity and recursive child recovery, preserving successful work and renewing execution identity. Rewind returns after enqueueing, not recovery; activity retry-policy histories with retry timers remain unsupported, matching the pinned Python algorithm. Added a focused rewind sample. - Added orchestration tags to scheduling, metadata, queries, sub-orchestrations, continue-as-new, and restart. - Added explicit worker capability advertisement and orchestration/activity name/version filters with local fallback enforcement. - Added pluggable large-payload store/resolver support with size limits, SHA-256 integrity validation, memory/file implementations, and opt-in DTS capability advertisement. @@ -36,7 +37,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed -- Removed the unsupported SDK methods `RewindInstance`, `SkipGracefulOrchestrationTerminations`, `CreateTaskHub`, and `DeleteTaskHub`, along with `api.RewindOptions`, `api.WithRewindReason`, `api.CreateTaskHubOptions`, `api.WithRecreateTaskHub`, and the lifecycle-only `client.ErrTaskHubExists` sentinel. Provision and delete task hubs through the Azure control plane or Azure CLI. `RestartInstance`, normal termination, instance purge, entity maintenance, and passive decoding of rewind history records remain supported; `client.ErrTaskHubNotFound` still identifies a configured task hub that does not exist. +- Removed the unsupported SDK methods `SkipGracefulOrchestrationTerminations`, `CreateTaskHub`, and `DeleteTaskHub`, along with `api.CreateTaskHubOptions`, `api.WithRecreateTaskHub`, and the lifecycle-only `client.ErrTaskHubExists` sentinel. Provision and delete task hubs through the Azure control plane or Azure CLI. `RestartInstance`, normal termination, instance purge, and entity maintenance remain supported; `client.ErrTaskHubNotFound` still identifies a configured task hub that does not exist. - **Removed the sqlite and PostgreSQL storage backends** (`backend/sqlite` and `backend/postgres`) and the `modernc.org/sqlite` and `github.com/jackc/pgx/v5` dependencies. Durable Task Scheduler is now the only supported runtime, and this repository is the DTS Go SDK rather than an embeddable engine plus storage providers. Applications that hosted an embedded task hub must move to `durabletaskscheduler.NewClient` and `durabletaskscheduler.NewWorker`. - **Removed the standalone local gRPC server** (root `main.go` and its `Dockerfile`), along with the `--port`/`--host`/`--db` commands and the `.NET` client-SDK integration-test instructions that targeted it. - Removed the `samples/sequence` and `samples/heterogeneous` samples. The activity sequence is demonstrated by `samples/durabletaskscheduler`, and the heterogeneous sample only demonstrated routing between embedded task executor implementations, which has no Durable Task Scheduler equivalent. diff --git a/README.md b/README.md index 10cdc754..0375804f 100644 --- a/README.md +++ b/README.md @@ -330,14 +330,24 @@ The client also does these operations, if the connected service supplies them: - Query the instances with a limit. List the instance IDs. - Restart an orchestration. +- Rewind a failed orchestration without repeating successful activities. - Purge in a batch or with a filter. - Terminate an orchestration. - Read the tags and the worker capabilities. Provision and delete task hubs through the Azure control plane or Azure CLI. -The SDK does not expose task-hub lifecycle, rewind, or skip-graceful-termination +The SDK does not expose task-hub lifecycle or skip-graceful-termination operations. +`RewindInstance(ctx, id, api.WithRewindReason("dependency repaired"))` only +enqueues recovery. Observe a **new execution ID** and its final status separately; +an immediate completion wait can still return the previous failed execution. +The worker replaces failed history using the same protocol as the Python SDK, +and DTS recursively rewinds failed children. Activity retry-policy histories +containing retry timers are not supported by the pinned Python rewind algorithm +or this implementation. See the [rewind sample](./samples/rewind) and +[rewind behavior and limitations](./durabletaskscheduler/README.md#rewind). + To read a long history, use `StreamOrchestrationHistory`. This method reads the history one part at a time. If you buffer the history instead, the SDK applies a validated event cap. ## Durable entities diff --git a/api/management.go b/api/management.go index b8065255..312f4541 100644 --- a/api/management.go +++ b/api/management.go @@ -6,6 +6,7 @@ import ( "time" "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/types/known/wrapperspb" ) const ( @@ -63,6 +64,17 @@ func WithRestartNewInstanceID(restartWithNewInstanceID bool) RestartOptions { } } +// RewindOptions configures a request to rewind a failed orchestration. +type RewindOptions func(*protos.RewindInstanceRequest) error + +// WithRewindReason records the reason for rewinding an orchestration. +func WithRewindReason(reason string) RewindOptions { + return func(req *protos.RewindInstanceRequest) error { + req.Reason = wrapperspb.String(reason) + return nil + } +} + type PurgeInstanceFilter struct { CreatedTimeFrom time.Time CreatedTimeTo time.Time diff --git a/client/client_grpc_management.go b/client/client_grpc_management.go index 6e850001..ac83c611 100644 --- a/client/client_grpc_management.go +++ b/client/client_grpc_management.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "slices" + "strings" "time" "github.com/microsoft/durabletask-go/api" @@ -209,6 +210,32 @@ func (c *TaskHubGrpcClient) RestartInstance(ctx context.Context, id api.Instance return api.InstanceID(resp.GetInstanceId()), nil } +// RewindInstance enqueues recovery of a failed orchestration. The service +// replaces its failed history with a new execution, preserving successful work +// and recursively rewinding failed sub-orchestrations. +// +// A nil error means the request was enqueued, not that recovery completed. +// Observe a new execution and its terminal status separately. Canceling ctx +// cancels enqueueing; it does not cancel a rewind already accepted by the service. +func (c *TaskHubGrpcClient) RewindInstance(ctx context.Context, id api.InstanceID, opts ...api.RewindOptions) error { + if id == api.EmptyInstanceID { + return api.WrapInvalidArgument(errors.New("rewind instance ID cannot be empty")) + } + if strings.HasPrefix(string(id), "@") { + return api.WrapInvalidArgument(errors.New("rewind does not support entity instance IDs")) + } + req := &protos.RewindInstanceRequest{InstanceId: string(id)} + for _, configure := range opts { + if err := configure(req); err != nil { + return fmt.Errorf("failed to configure rewind request: %w", api.WrapInvalidArgument(err)) + } + } + if _, err := c.client.RewindInstance(ctx, req); err != nil { + return clientRPCError(ctx, "failed to rewind orchestration instance", err) + } + return nil +} + func (c *TaskHubGrpcClient) PurgeInstances(ctx context.Context, request api.PurgeInstancesRequest) (*api.PurgeInstancesResult, error) { if err := request.Validate(); err != nil { return nil, err diff --git a/client/client_grpc_management_test.go b/client/client_grpc_management_test.go index d7d65ed4..9418f98c 100644 --- a/client/client_grpc_management_test.go +++ b/client/client_grpc_management_test.go @@ -20,6 +20,14 @@ type managementServer struct { protos.UnimplementedTaskHubSidecarServiceServer queryErr error + rewind func(context.Context, *protos.RewindInstanceRequest) (*protos.RewindInstanceResponse, error) +} + +func (s *managementServer) RewindInstance(ctx context.Context, req *protos.RewindInstanceRequest) (*protos.RewindInstanceResponse, error) { + if s.rewind == nil { + return nil, status.Error(codes.Unimplemented, "rewind is not implemented") + } + return s.rewind(ctx, req) } func (s *managementServer) QueryInstances( diff --git a/client/client_grpc_rewind_test.go b/client/client_grpc_rewind_test.go new file mode 100644 index 00000000..9c40e347 --- /dev/null +++ b/client/client_grpc_rewind_test.go @@ -0,0 +1,117 @@ +package client + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestRewindInstanceRequest(t *testing.T) { + for _, test := range []struct { + name string + reason *wrapperspb.StringValue + }{ + {"omitted reason", nil}, + {"empty reason", wrapperspb.String("")}, + {"reason", wrapperspb.String("dependency repaired")}, + } { + t.Run(test.name, func(t *testing.T) { + requests := make(chan *protos.RewindInstanceRequest, 1) + client := startQueryClient(t, &managementServer{ + rewind: func(_ context.Context, req *protos.RewindInstanceRequest) (*protos.RewindInstanceResponse, error) { + requests <- req + return &protos.RewindInstanceResponse{}, nil + }, + }) + var options []api.RewindOptions + if test.reason != nil { + options = append(options, api.WithRewindReason(test.reason.Value)) + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + // The fake service implements no metadata/wait RPC. Rewind must only enqueue. + require.NoError(t, client.RewindInstance(ctx, "failed-instance", options...)) + req := <-requests + require.Equal(t, "failed-instance", req.InstanceId) + require.Equal(t, test.reason, req.Reason) + }) + } +} + +func TestRewindInstanceValidation(t *testing.T) { + // Invalid requests must not reach the transport. + client := &TaskHubGrpcClient{} + for _, id := range []api.InstanceID{"", "@counter@key", "@", "@not-an-orchestration"} { + require.ErrorIs(t, client.RewindInstance(context.Background(), id), api.ErrInvalidArgument) + } + optionErr := errors.New("invalid rewind option") + err := client.RewindInstance(context.Background(), "instance", func(*protos.RewindInstanceRequest) error { + return optionErr + }) + require.ErrorIs(t, err, api.ErrInvalidArgument) + require.ErrorIs(t, err, optionErr) +} + +func TestRewindInstanceRPCErrors(t *testing.T) { + for _, test := range []struct { + code codes.Code + want error + }{ + {codes.InvalidArgument, api.ErrInvalidArgument}, + {codes.NotFound, api.ErrInstanceNotFound}, + {codes.FailedPrecondition, api.ErrInvalidState}, + {codes.Unimplemented, api.ErrFeatureNotSupported}, + {codes.Canceled, context.Canceled}, + {codes.DeadlineExceeded, context.DeadlineExceeded}, + } { + t.Run(test.code.String(), func(t *testing.T) { + client := startQueryClient(t, &managementServer{ + rewind: func(context.Context, *protos.RewindInstanceRequest) (*protos.RewindInstanceResponse, error) { + return nil, status.Error(test.code, "rewind rejected") + }, + }) + err := client.RewindInstance(context.Background(), "instance") + require.ErrorIs(t, err, test.want) + require.Equal(t, test.code, status.Code(err)) + }) + } +} + +func TestRewindInstanceCancellation(t *testing.T) { + entered := make(chan struct{}) + client := startQueryClient(t, &managementServer{ + rewind: func(ctx context.Context, _ *protos.RewindInstanceRequest) (*protos.RewindInstanceResponse, error) { + close(entered) + <-ctx.Done() + return nil, status.FromContextError(ctx.Err()).Err() + }, + }) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { result <- client.RewindInstance(ctx, "instance") }() + select { + case <-entered: + case <-ctx.Done(): + t.Fatal("rewind did not reach the service") + } + cancel() + err := <-result + require.ErrorIs(t, err, context.Canceled) + require.Equal(t, codes.Canceled, status.Code(err)) +} + +func TestRewindInstanceAlreadyCanceled(t *testing.T) { + client := startQueryClient(t, &managementServer{}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + require.ErrorIs(t, client.RewindInstance(ctx, "instance"), context.Canceled) +} diff --git a/client/grpc_worker_rewind_test.go b/client/grpc_worker_rewind_test.go new file mode 100644 index 00000000..52d81ce9 --- /dev/null +++ b/client/grpc_worker_rewind_test.go @@ -0,0 +1,101 @@ +package client + +import ( + "context" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/largepayload" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/payload" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestWorkerRewindHistoryTransport(t *testing.T) { + for _, streamed := range []bool{false, true} { + name := "inline" + if streamed { + name = "streamed" + } + t.Run(name, func(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{Store: store, Resolver: store, ThresholdBytes: 8} + raw := wrapperspb.String(`"preserved large input"`) + token, err := largepayload.Externalize(context.Background(), options, raw) + require.NoError(t, err) + require.NotEqual(t, raw.Value, token.Value) + start := helpers.NewExecutionStartedEvent("workflow", "instance", token, nil, nil, nil) + oldID := start.GetExecutionStarted().GetOrchestrationInstance().GetExecutionId().GetValue() + history := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), start, + helpers.NewTaskScheduledEvent(0, "bad", nil, token, nil), + helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ErrorMessage: "failed"}), + {EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{OrchestrationStatus: api.RUNTIME_STATUS_FAILED}, + }}, + } + client := &fakeSchedulerClient{history: []*protos.HistoryChunk{ + {Events: history[:2]}, {Events: history[2:]}, + }} + worker := newFakeWorker(t, client, WithWorkerLargePayloads(options)) + request := &protos.OrchestratorRequest{ + InstanceId: "instance", ExecutionId: wrapperspb.String(oldID), PastEvents: history, + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + {EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}}, + }, + RequiresHistoryStreaming: streamed, + } + if streamed { + request.PastEvents = nil + } + worker.processOrchestration(context.Background(), client, "completion-token", request) + require.Zero(t, client.orchestrationAbandons) + require.Len(t, client.orchestrationCompletions, 1) + response := client.orchestrationCompletions[0] + require.Equal(t, "completion-token", response.CompletionToken) + require.Nil(t, response.CustomStatus) + require.Len(t, response.Actions, 1) + clean := response.Actions[0].GetRewindOrchestration().GetNewHistory() + require.Len(t, clean, 4) + rewritten := clean[1].GetExecutionStarted() + require.NotEqual(t, oldID, rewritten.GetOrchestrationInstance().GetExecutionId().GetValue()) + require.NotEqual(t, raw.Value, rewritten.GetInput().GetValue(), "replacement history must be externalized before send") + hydrated, err := largepayload.Hydrate(context.Background(), options, rewritten.Input) + require.NoError(t, err) + require.Equal(t, raw.Value, hydrated.Value) + require.NotNil(t, clean[3].GetExecutionRewound()) + }) + } +} + +func TestWorkerRewindOversizedHistoryFailsExplicitly(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client, WithMaxOrchestratorCompletionBytes(minOrchestratorCompletionBytes)) + worker.processOrchestration(context.Background(), client, "token", &protos.OrchestratorRequest{ + InstanceId: "instance", + PastEvents: []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("workflow", "instance", nil, nil, nil, nil), + {EventType: &protos.HistoryEvent_GenericEvent{GenericEvent: &protos.GenericEvent{ + Data: wrapperspb.String(strings.Repeat("x", 2*minOrchestratorCompletionBytes)), + }}}, + {EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{OrchestrationStatus: api.RUNTIME_STATUS_FAILED}, + }}, + }, + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + {EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}}, + }, + }) + require.Len(t, client.orchestrationCompletions, 1) + response := client.orchestrationCompletions[0] + require.Len(t, response.Actions, 1) + completion := response.Actions[0].GetCompleteOrchestration() + require.Equal(t, api.RUNTIME_STATUS_FAILED, completion.GetOrchestrationStatus()) + require.Equal(t, string(api.ErrorTypeOrchestratorResponseTooLarge), completion.GetFailureDetails().GetErrorType()) + require.True(t, completion.GetFailureDetails().GetIsNonRetriable()) +} diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 9eb8cef4..5141ff44 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -129,13 +129,13 @@ configurations is replay-breaking for affected in-flight orchestrations. ### Advanced management `TaskHubGrpcClient` exposes bounded `QueryInstances` and `ListInstanceIDs` -operations with opaque continuation tokens, plus `RestartInstance` and +operations with opaque continuation tokens, plus `RestartInstance`, `RewindInstance`, and batch/filter `PurgeInstances`. Queries can filter locally by exact tag key/value pairs when the current wire contract does not carry tag filters. Provision and delete task hubs through the Azure control plane or Azure CLI, -not SDK RPCs. The SDK does not expose task-hub lifecycle, rewind, or +not SDK RPCs. The SDK does not expose task-hub lifecycle or skip-graceful-termination operations. Use `TerminateOrchestration` for normal orchestration termination. @@ -151,6 +151,39 @@ service limitations: filtered purge can complete without deleting matches, and `ListInstanceIds` can omit matching IDs. The emulator integration tests record these limitations explicitly. +### Rewind + +`RewindInstance(ctx, id, api.WithRewindReason("dependency repaired"))` enqueues +recovery of a **failed** orchestration. It rejects empty IDs and entity IDs. +Missing instances, invalid states, unsupported services, and cancellation retain +the existing `errors.Is` categories (`api.ErrInstanceNotFound`, +`api.ErrInvalidState`, `api.ErrFeatureNotSupported`, and context errors). +Cancellation stops enqueueing, not recovery that the service already accepted. + +Recovery is a two-step worker/service operation. Before executing user code, the +worker sends `RewindOrchestrationAction.NewHistory`: failed activity schedules and +results, failed child results, and terminal completion records are removed. +Successful history and failed child creation records are retained. The start +record receives a new execution ID; a rewound child's parent execution ID is +updated. DTS persists this history, recursively rewinds failed children, and +wakes the leaf for ordinary replay. Successful activities are replayed from their +results rather than executed again. + +This follows the standalone Python SDK's +[`_build_rewind_result`](https://github.com/microsoft/durabletask-python/blob/a6c18f24a230332d143d05e8798fbfdd951d868c/durabletask/worker.py#L2346-L2447), +not the incomplete standalone .NET worker implementation. An accepted RPC alone +does not prove recovery. Poll for a changed execution ID and successful final +status/output; `WaitForOrchestrationCompletion` can initially return the old +failed execution. The [runnable sample](../samples/rewind) demonstrates this and +cleans up only its own instance. + +**Limitations:** the pinned Python algorithm retains timer events between +activity retry attempts. Rewinding an activity that used a retry policy can +therefore produce a replay nondeterminism error; that case is not supported. +Rewind does not repair unchanged failing application code. Replacement history +uses the normal large-payload handling and orchestration response-size bound; +oversized responses fail explicitly rather than being silently truncated. + ### Worker routing and capabilities Use `client.WithTaskVersioning` for `None`, `Strict`, or `CurrentOrOlder` @@ -307,6 +340,7 @@ cancels them only if the shutdown context expires. | Schedule, bounded query/list, and wait for orchestrations | Supported | | Tags on schedule, metadata, query, sub-orchestration, continue-as-new, and restart | Supported; distinct from immutable context fields | | Restart and batch/filter purge | Supported; see emulator limitations above | +| Rewind API and failed activity/child recovery | Supported with a rewind-capable DTS service; enqueue-only API, Python-aligned worker history replacement; see [limitations](#rewind) | | Raise events, suspend/resume, terminate, and single-instance purge | Supported | | Orchestration and activity execution | Supported | | Bounded orchestration/activity/entity concurrency | Supported | diff --git a/samples/rewind/rewind.go b/samples/rewind/rewind.go new file mode 100644 index 00000000..59757cb8 --- /dev/null +++ b/samples/rewind/rewind.go @@ -0,0 +1,132 @@ +// Command rewind repairs a failed activity without repeating successful work. +// +// export DTS_CONNECTION_STRING="Endpoint=http://localhost:8080;TaskHub=default;Authentication=None" +// go run ./samples/rewind +package main + +import ( + "context" + "errors" + "fmt" + "log" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/microsoft/durabletask-go/task" +) + +func main() { + if err := run(); err != nil { + log.Fatal(err) + } +} + +func run() (err error) { + name := "RewindSample_" + uuid.NewString() + var repaired atomic.Bool + var successfulCalls, failedCalls atomic.Int32 + registry := task.NewTaskRegistry() + if err := registry.AddActivityN(name+"Good", func(task.ActivityContext) (any, error) { + successfulCalls.Add(1) + return "kept", nil + }); err != nil { + return err + } + if err := registry.AddActivityN(name+"Flaky", func(task.ActivityContext) (any, error) { + failedCalls.Add(1) + if !repaired.Load() { + return nil, errors.New("dependency unavailable") + } + return "recovered", nil + }); err != nil { + return err + } + if err := registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + var good, recovered string + if err := ctx.CallActivity(name + "Good").Await(&good); err != nil { + return nil, err + } + if err := ctx.CallActivity(name + "Flaky").Await(&recovered); err != nil { + return nil, err + } + return good + ":" + recovered, nil + }); err != nil { + return err + } + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + // Registry-derived filters and unique task names isolate this sample's work. + app, err := dtssample.Start(ctx, registry) + if err != nil { + return err + } + defer func() { err = errors.Join(err, app.Shutdown()) }() + id := api.InstanceID(name) + defer func() { err = errors.Join(err, cleanup(app.Client, id)) }() + if _, err := app.Client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id)); err != nil { + return err + } + failed, err := app.Client.WaitForOrchestrationCompletion(ctx, id) + if err != nil { + return err + } + if failed.RuntimeStatus != api.RUNTIME_STATUS_FAILED { + return fmt.Errorf("expected initial failure, got %s", failed.RuntimeStatus) + } + + repaired.Store(true) + if err := app.Client.RewindInstance(ctx, id, api.WithRewindReason("dependency repaired")); err != nil { + return err + } + fmt.Printf("Rewind enqueued for %s (previous execution %s)\n", id, failed.ExecutionID) + + // A normal completion wait can still return the previous Failed execution. + // Require the replacement execution to complete successfully instead. + ticker := time.NewTicker(250 * time.Millisecond) + defer ticker.Stop() + for { + current, err := app.Client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return err + } + if current.ExecutionID != failed.ExecutionID && current.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED { + if current.SerializedOutput != `"kept:recovered"` || successfulCalls.Load() != 1 || failedCalls.Load() != 2 { + return fmt.Errorf("unexpected recovery: output=%s successful calls=%d failed calls=%d", + current.SerializedOutput, successfulCalls.Load(), failedCalls.Load()) + } + fmt.Printf("Recovered execution %s: %s; successful activity ran once, failed activity ran twice\n", + current.ExecutionID, current.SerializedOutput) + return nil + } + select { + case <-ctx.Done(): + return fmt.Errorf("recovery not observed after rewind was enqueued: %w", ctx.Err()) + case <-ticker.C: + } + } +} + +func cleanup(client *durabletaskscheduler.Client, id api.InstanceID) error { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + if errors.Is(err, api.ErrInstanceNotFound) { + return nil + } + if err != nil { + return err + } + if !metadata.IsComplete() { + if err := client.TerminateOrchestration(ctx, id); err != nil { + return err + } + if _, err := client.WaitForOrchestrationCompletion(ctx, id); err != nil { + return err + } + } + return client.PurgeOrchestrationState(ctx, id) +} diff --git a/task/executor.go b/task/executor.go index f610b402..57fd2484 100644 --- a/task/executor.go +++ b/task/executor.go @@ -267,6 +267,9 @@ func (te *taskExecutor) ExecuteOrchestrator( if err != nil { return nil, err } + if isRewindRequest(oldEvents, newEvents) { + return buildRewindResult(id, oldEvents, newEvents) + } started := startedEvent(oldEvents, newEvents) name := started.GetName() version := started.GetVersion().GetValue() diff --git a/task/rewind.go b/task/rewind.go new file mode 100644 index 00000000..c3f4909f --- /dev/null +++ b/task/rewind.go @@ -0,0 +1,86 @@ +package task + +import ( + "encoding/hex" + "fmt" + "slices" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func isRewindRequest(oldEvents, newEvents []*protos.HistoryEvent) bool { + return slices.ContainsFunc(newEvents, func(event *protos.HistoryEvent) bool { + return event.GetExecutionRewound() != nil + }) && slices.ContainsFunc(oldEvents, func(event *protos.HistoryEvent) bool { + return event.GetExecutionCompleted() != nil + }) +} + +// A terminal rewind first returns replacement history; DTS persists it and +// rewinds failed children. The subsequent leaf wake-up has no old completion +// marker, so normal replay reissues only the removed activity schedules. +// Keep this transformation aligned with durabletask-python's +// _OrchestrationExecutor._build_rewind_result and the DTS backend contract. +func buildRewindResult(id api.InstanceID, oldEvents, newEvents []*protos.HistoryEvent) (*ExecutionResults, error) { + if len(newEvents) != 2 || newEvents[1].GetExecutionRewound() == nil { + return nil, fmt.Errorf("rewind requires exactly two new events: orchestrator started and execution rewound") + } + rewound := newEvents[1].GetExecutionRewound() + executionID, err := uuid.NewRandom() + if err != nil { + return nil, fmt.Errorf("failed to generate rewind execution ID: %w", err) + } + + failedTasks := make(map[int32]struct{}) + for _, events := range [][]*protos.HistoryEvent{oldEvents, newEvents} { + for _, event := range events { + if failed := event.GetTaskFailed(); failed != nil { + failedTasks[failed.TaskScheduledId] = struct{}{} + } + } + } + + history := make([]*protos.HistoryEvent, 0, len(oldEvents)+len(newEvents)) + for _, events := range [][]*protos.HistoryEvent{oldEvents, newEvents} { + for _, event := range events { + if event.GetTaskFailed() != nil || + event.GetSubOrchestrationInstanceFailed() != nil || + event.GetExecutionCompleted() != nil { + continue + } + if event.GetTaskScheduled() != nil { + if _, failed := failedTasks[event.EventId]; failed { + continue + } + } + if event.GetExecutionStarted() != nil { + event = proto.CloneOf(event) + started := event.GetExecutionStarted() + if started.OrchestrationInstance == nil { + started.OrchestrationInstance = &protos.OrchestrationInstance{} + } + started.OrchestrationInstance.ExecutionId = wrapperspb.String(hex.EncodeToString(executionID[:])) + if rewound.GetParentExecutionId().GetValue() != "" && started.ParentInstance != nil { + if started.ParentInstance.OrchestrationInstance == nil { + started.ParentInstance.OrchestrationInstance = &protos.OrchestrationInstance{} + } + started.ParentInstance.OrchestrationInstance.ExecutionId = proto.CloneOf(rewound.ParentExecutionId) + } + } + history = append(history, event) + } + } + return &ExecutionResults{Response: &protos.OrchestratorResponse{ + InstanceId: string(id), + Actions: []*protos.OrchestratorAction{{ + Id: -1, + OrchestratorActionType: &protos.OrchestratorAction_RewindOrchestration{ + RewindOrchestration: &protos.RewindOrchestrationAction{NewHistory: history}, + }, + }}, + }}, nil +} diff --git a/task/rewind_test.go b/task/rewind_test.go new file mode 100644 index 00000000..d5a5c401 --- /dev/null +++ b/task/rewind_test.go @@ -0,0 +1,168 @@ +package task + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func rewindEvent(parentExecutionID string) *protos.HistoryEvent { + rewound := &protos.ExecutionRewoundEvent{Reason: wrapperspb.String("dependency repaired")} + if parentExecutionID != "" { + rewound.ParentExecutionId = wrapperspb.String(parentExecutionID) + } + return &protos.HistoryEvent{EventId: -1, EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: rewound}} +} + +func failedCompletionEvent() *protos.HistoryEvent { + return &protos.HistoryEvent{EventId: 10, EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{ + OrchestrationStatus: api.RUNTIME_STATUS_FAILED, + FailureDetails: &protos.TaskFailureDetails{ErrorType: "Failure", ErrorMessage: "unavailable"}, + }, + }} +} + +func TestRewindHistoryPreservesSuccessfulWork(t *testing.T) { + start := helpers.NewExecutionStartedEvent("workflow", "instance", wrapperspb.String(`"input"`), nil, nil, nil) + start.GetExecutionStarted().Tags = map[string]string{"tag": "value"} + start.ProtoReflect().SetUnknown([]byte{0xa0, 0x06, 0x01}) + old := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + start, + helpers.NewTaskScheduledEvent(0, "good", nil, nil, nil), + helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"kept"`)), + helpers.NewTaskScheduledEvent(1, "bad", nil, nil, nil), + helpers.NewTaskFailedEvent(1, &protos.TaskFailureDetails{ErrorMessage: "failed"}), + {EventId: 2, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{InstanceId: "failed-child", Name: "child"}, + }}, + {EventType: &protos.HistoryEvent_SubOrchestrationInstanceFailed{ + SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{TaskScheduledId: 2}, + }}, + {EventId: 3, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{InstanceId: "successful-child", Name: "child"}, + }}, + {EventType: &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{TaskScheduledId: 3, Result: wrapperspb.String(`"child-kept"`)}, + }}, + helpers.NewTimerCreatedEvent(4, start.Timestamp), + {EventType: &protos.HistoryEvent_TimerFired{TimerFired: &protos.TimerFiredEvent{TimerId: 4}}}, + {EventType: &protos.HistoryEvent_GenericEvent{GenericEvent: &protos.GenericEvent{Data: wrapperspb.String("audit")}}}, + {EventType: &protos.HistoryEvent_OrchestratorCompleted{OrchestratorCompleted: &protos.OrchestratorCompletedEvent{}}}, + rewindEvent(""), + failedCompletionEvent(), + } + newEvents := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")} + original := proto.CloneOf(&protos.OrchestratorRequest{PastEvents: old, NewEvents: newEvents}) + // Rewriting is a protocol operation, not application replay or version dispatch. + executor := NewTaskExecutor(NewTaskRegistry(), + WithOrchestratorNotFoundStrategy(OrchestratorNotFoundReject), + WithVersioning(VersioningOptions{DefaultVersion: "2.0", MatchStrategy: VersionMatchStrict, FailureStrategy: VersionFailureReject}), + WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1, MaxHistoryEvents: 1}), + ) + result, err := executor.ExecuteOrchestrator(context.Background(), "instance", old, newEvents, nil) + require.NoError(t, err) + require.Equal(t, "instance", result.Response.InstanceId) + require.Nil(t, result.Response.CustomStatus) + require.Nil(t, result.Response.NumEventsProcessed) + require.Len(t, result.Response.Actions, 1) + action := result.Response.Actions[0] + require.EqualValues(t, -1, action.Id) + history := action.GetRewindOrchestration().GetNewHistory() + want := []*protos.HistoryEvent{old[0], proto.CloneOf(start), old[2], old[3], old[6], old[8], old[9], old[10], old[11], old[12], old[13], old[14], newEvents[0], newEvents[1]} + require.Len(t, history, len(want)) + newID := history[1].GetExecutionStarted().GetOrchestrationInstance().GetExecutionId().GetValue() + require.Len(t, newID, 32) + parsed, err := uuid.Parse(newID) + require.NoError(t, err) + require.Equal(t, uuid.Version(4), parsed.Version()) + require.NotEqual(t, start.GetExecutionStarted().GetOrchestrationInstance().GetExecutionId().GetValue(), newID) + want[1].GetExecutionStarted().OrchestrationInstance.ExecutionId = wrapperspb.String(newID) + for i := range want { + require.Truef(t, proto.Equal(want[i], history[i]), "history event %d changed", i) + } + require.True(t, proto.Equal(original, &protos.OrchestratorRequest{PastEvents: old, NewEvents: newEvents})) + + redelivered, err := executor.ExecuteOrchestrator(context.Background(), "instance", old, newEvents, nil) + require.NoError(t, err) + require.NotEqual(t, newID, redelivered.Response.Actions[0].GetRewindOrchestration().NewHistory[1].GetExecutionStarted().GetOrchestrationInstance().GetExecutionId().GetValue()) +} + +func TestRewindParentExecutionIdentity(t *testing.T) { + for _, test := range []struct { + name string + parent *protos.ParentInstanceInfo + requestID string + wantID string + }{ + {"update child", &protos.ParentInstanceInfo{OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: "parent", ExecutionId: wrapperspb.String("old")}}, "new-parent", "new-parent"}, + {"preserve parent", &protos.ParentInstanceInfo{OrchestrationInstance: &protos.OrchestrationInstance{ExecutionId: wrapperspb.String("old")}}, "", "old"}, + {"missing nested identity", &protos.ParentInstanceInfo{}, "new-parent", "new-parent"}, + {"no synthetic parent", nil, "new-parent", ""}, + } { + t.Run(test.name, func(t *testing.T) { + start := helpers.NewExecutionStartedEvent("child", "child-id", nil, test.parent, nil, nil) + original := proto.CloneOf(start) + result, err := buildRewindResult("child-id", + []*protos.HistoryEvent{start, failedCompletionEvent()}, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent(test.requestID)}) + require.NoError(t, err) + rewritten := result.Response.Actions[0].GetRewindOrchestration().NewHistory[0].GetExecutionStarted() + require.Equal(t, test.wantID, rewritten.GetParentInstance().GetOrchestrationInstance().GetExecutionId().GetValue()) + if test.parent == nil { + require.Nil(t, rewritten.ParentInstance) + } + require.True(t, proto.Equal(original, start)) + }) + } +} + +func TestRewindJumpStartReplaysWithoutRewriting(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("workflow", func(ctx *OrchestrationContext) (any, error) { + var good string + if err := ctx.CallActivity("good").Await(&good); err != nil { + return nil, err + } + return nil, ctx.CallActivity("bad", WithActivityInput(good)).Await(nil) + })) + start := helpers.NewExecutionStartedEvent("workflow", "instance", nil, nil, nil, nil) + old := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), start, + helpers.NewTaskScheduledEvent(0, "good", nil, nil, nil), + helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"kept"`)), + rewindEvent(""), + } + result, err := NewTaskExecutor(registry, WithOrchestrationOptions(OrchestrationOptions{MaxEventsPerTurn: 1})). + ExecuteOrchestrator(context.Background(), "instance", old, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")}, nil) + require.NoError(t, err) + require.Len(t, result.Response.Actions, 1) + require.EqualValues(t, 1, result.Response.Actions[0].Id) + scheduled := result.Response.Actions[0].GetScheduleTask() + require.NotNil(t, scheduled) + require.Equal(t, "bad", scheduled.Name) + require.Equal(t, `"kept"`, scheduled.GetInput().GetValue()) +} + +func TestRewindMalformedRequests(t *testing.T) { + for _, events := range [][]*protos.HistoryEvent{ + {rewindEvent("")}, + {rewindEvent(""), helpers.NewOrchestratorStartedEvent()}, + {helpers.NewOrchestratorStartedEvent(), rewindEvent(""), helpers.NewEventRaisedEvent("extra", nil)}, + } { + _, err := NewTaskExecutor(NewTaskRegistry()).ExecuteOrchestrator(context.Background(), "instance", + []*protos.HistoryEvent{failedCompletionEvent()}, events, nil) + require.ErrorContains(t, err, "rewind requires exactly two new events") + } + require.False(t, isRewindRequest([]*protos.HistoryEvent{rewindEvent(""), failedCompletionEvent()}, nil)) + require.False(t, isRewindRequest(nil, []*protos.HistoryEvent{rewindEvent("")})) +} diff --git a/tests/durabletaskscheduler/dts_rewind_test.go b/tests/durabletaskscheduler/dts_rewind_test.go new file mode 100644 index 00000000..0c83eac2 --- /dev/null +++ b/tests/durabletaskscheduler/dts_rewind_test.go @@ -0,0 +1,256 @@ +package durabletaskscheduler_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" +) + +func TestDTSRewindRecovery(t *testing.T) { + for _, nested := range []bool{false, true} { + name := "activity" + if nested { + name = "child" + } + t.Run(name, func(t *testing.T) { + prefix := "GoRewind_" + uuid.NewString() + id := api.InstanceID(prefix) + childID := id + "-child" + var repaired atomic.Bool + var goodCalls, parentGoodCalls, failedCalls atomic.Int32 + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddActivityN(prefix+"Good", func(task.ActivityContext) (any, error) { + goodCalls.Add(1) + return "kept", nil + })) + require.NoError(t, registry.AddActivityN(prefix+"ParentGood", func(task.ActivityContext) (any, error) { + parentGoodCalls.Add(1) + return "parent", nil + })) + require.NoError(t, registry.AddActivityN(prefix+"Flaky", func(ctx task.ActivityContext) (any, error) { + failedCalls.Add(1) + var input string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if !repaired.Load() { + return nil, errors.New("rewind dependency unavailable") + } + return "recovered:" + input, nil + })) + require.NoError(t, registry.AddOrchestratorN(prefix+"Leaf", func(ctx *task.OrchestrationContext) (any, error) { + var input, good, result string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.CallActivity(prefix + "Good").Await(&good); err != nil { + return nil, err + } + if err := ctx.CallActivity(prefix+"Flaky", task.WithActivityInput(input)).Await(&result); err != nil { + return nil, err + } + return good + ":" + result, nil + })) + require.NoError(t, registry.AddOrchestratorN(prefix+"Parent", func(ctx *task.OrchestrationContext) (any, error) { + var input, good, result string + if err := ctx.GetInput(&input); err != nil { + return nil, err + } + if err := ctx.CallActivity(prefix + "ParentGood").Await(&good); err != nil { + return nil, err + } + if err := ctx.CallSubOrchestrator(prefix+"Leaf", + task.WithSubOrchestratorInput(input), + task.WithSubOrchestrationInstanceID(string(childID)), + ).Await(&result); err != nil { + return nil, err + } + return good + ":" + result, nil + })) + client, _, _ := startEmulatorClientAndWorker(t, registry, durabletaskclient.WithAutoWorkItemFilters()) + ids := []api.InstanceID{id} + if nested { + ids = append(ids, childID) + } + t.Cleanup(func() { cleanupRewindInstances(t, client, ids) }) + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + defer cancel() + orchestrator := prefix + "Leaf" + if nested { + orchestrator = prefix + "Parent" + } + _, err := client.ScheduleNewOrchestration(ctx, orchestrator, api.WithInstanceID(id), api.WithInput("payload")) + require.NoError(t, err) + failed, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, failed.RuntimeStatus) + require.Contains(t, failed.FailureDetails.ErrorMessage, "rewind dependency unavailable") + require.EqualValues(t, 1, goodCalls.Load()) + require.EqualValues(t, 1, failedCalls.Load()) + before := make(map[api.InstanceID]*api.OrchestrationHistory, len(ids)) + for _, instanceID := range ids { + before[instanceID] = readRewindHistory(t, client, instanceID) + } + + repaired.Store(true) + require.NoError(t, client.RewindInstance(ctx, id, api.WithRewindReason("dependency repaired"))) + t.Logf("rewind enqueued: instance=%s previousExecution=%s", id, failed.ExecutionID) + completed, recoveryErr := waitForRewindRecovery(ctx, client, id, failed.ExecutionID) + after := make(map[api.InstanceID]*api.OrchestrationHistory, len(ids)) + for _, instanceID := range ids { + after[instanceID] = readRewindHistory(t, client, instanceID) + } + require.NoError(t, recoveryErr) + want := `"kept:recovered:payload"` + if nested { + want = `"parent:kept:recovered:payload"` + require.EqualValues(t, 1, parentGoodCalls.Load()) + } + require.Equal(t, want, completed.SerializedOutput) + require.EqualValues(t, 1, goodCalls.Load(), "successful activity must be replayed, not executed again") + require.EqualValues(t, 2, failedCalls.Load()) + for _, instanceID := range ids { + require.NotEmpty(t, before[instanceID].ExecutionID) + require.NotEqual(t, before[instanceID].ExecutionID, after[instanceID].ExecutionID) + var successfulResults int + for _, event := range after[instanceID].Events { + require.NotEqual(t, api.HistoryEventTaskFailed, event.Type) + require.NotEqual(t, api.HistoryEventSubOrchestrationInstanceFailed, event.Type) + if event.TaskCompleted != nil && event.TaskCompleted.SerializedResult == `"kept"` { + successfulResults++ + } + if event.ExecutionStarted != nil && instanceID == childID { + require.Equal(t, after[id].ExecutionID, event.ExecutionStarted.Parent.ExecutionID) + } + } + if !nested || instanceID == childID { + require.Equal(t, 1, successfulResults) + } + } + }) + } +} + +func waitForRewindRecovery(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, oldExecution string) (*api.OrchestrationMetadata, error) { + ctx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + var current *api.OrchestrationMetadata + for { + var err error + current, err = client.FetchOrchestrationMetadata(ctx, id, api.WithFetchPayloads(true)) + if err != nil { + return current, err + } + if current.ExecutionID != oldExecution && current.RuntimeStatus == api.RUNTIME_STATUS_COMPLETED { + return current, nil + } + timer := time.NewTimer(250 * time.Millisecond) + select { + case <-ctx.Done(): + timer.Stop() + return current, fmt.Errorf("rewind recovery not observed: execution=%s status=%s: %w", current.ExecutionID, current.RuntimeStatus, ctx.Err()) + case <-timer.C: + } + } +} + +func readRewindHistory(t *testing.T, client *durabletaskscheduler.Client, id api.InstanceID) *api.OrchestrationHistory { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + history, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{}) + require.NoError(t, err) + data, err := json.Marshal(history) + require.NoError(t, err) + t.Logf("history: %s", data) + return history +} + +func cleanupRewindInstances(t *testing.T, client *durabletaskscheduler.Client, ids []api.InstanceID) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, id := range ids { + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + if errors.Is(err, api.ErrInstanceNotFound) { + continue + } + require.NoError(t, err) + if !metadata.IsComplete() { + require.NoError(t, client.TerminateOrchestration(ctx, id)) + _, err = client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + } + require.NoError(t, client.PurgeOrchestrationState(ctx, id)) + } +} + +func TestDTSRewindRejectedRequests(t *testing.T) { + name := "GoRewindStates_" + uuid.NewString() + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + var wait bool + if err := ctx.GetInput(&wait); err != nil { + return nil, err + } + if wait { + return nil, ctx.WaitForSingleEvent("release", -1).Await(nil) + } + return "done", nil + })) + client, _, _ := startEmulatorClientAndWorker(t, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + require.ErrorIs(t, client.RewindInstance(ctx, api.InstanceID(name+"-missing")), api.ErrInstanceNotFound) + + for _, state := range []api.OrchestrationStatus{ + api.RUNTIME_STATUS_COMPLETED, + api.RUNTIME_STATUS_RUNNING, + api.RUNTIME_STATUS_SUSPENDED, + api.RUNTIME_STATUS_TERMINATED, + api.RUNTIME_STATUS_PENDING, + } { + t.Run(state.String(), func(t *testing.T) { + id := api.InstanceID(name + "-" + state.String()) + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + options := []api.NewOrchestrationOptions{ + api.WithInstanceID(id), + api.WithInput(state != api.RUNTIME_STATUS_COMPLETED), + } + if state == api.RUNTIME_STATUS_PENDING { + options = append(options, api.WithStartTime(time.Now().Add(time.Hour))) + } + _, err := client.ScheduleNewOrchestration(ctx, name, options...) + require.NoError(t, err) + if state != api.RUNTIME_STATUS_PENDING { + _, err = client.WaitForOrchestrationStart(ctx, id) + require.NoError(t, err) + } + switch state { + case api.RUNTIME_STATUS_SUSPENDED: + require.NoError(t, client.SuspendOrchestration(ctx, id, "rewind state test")) + case api.RUNTIME_STATUS_TERMINATED: + require.NoError(t, client.TerminateOrchestration(ctx, id)) + } + require.Eventually(t, func() bool { + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + return err == nil && metadata.RuntimeStatus == state + }, 10*time.Second, 100*time.Millisecond) + require.ErrorIs(t, client.RewindInstance(ctx, id), api.ErrInvalidState) + metadata, err := client.FetchOrchestrationMetadata(ctx, id) + require.NoError(t, err) + require.Equal(t, state, metadata.RuntimeStatus) + }) + } +} From 681ba0744f5a645ae100bb8def863b02dbe29b11 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 11:34:18 -0700 Subject: [PATCH 13/26] docs: remove task-hub lifecycle caveats Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 4 +--- durabletaskscheduler/README.md | 5 ++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0375804f..38f47100 100644 --- a/README.md +++ b/README.md @@ -335,9 +335,7 @@ The client also does these operations, if the connected service supplies them: - Terminate an orchestration. - Read the tags and the worker capabilities. -Provision and delete task hubs through the Azure control plane or Azure CLI. -The SDK does not expose task-hub lifecycle or skip-graceful-termination -operations. +The SDK does not expose skip-graceful-termination operations. `RewindInstance(ctx, id, api.WithRewindReason("dependency repaired"))` only enqueues recovery. Observe a **new execution ID** and its final status separately; diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 5141ff44..137db481 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -134,9 +134,8 @@ batch/filter `PurgeInstances`. Queries can filter locally by exact tag key/value pairs when the current wire contract does not carry tag filters. -Provision and delete task hubs through the Azure control plane or Azure CLI, -not SDK RPCs. The SDK does not expose task-hub lifecycle or -skip-graceful-termination operations. Use `TerminateOrchestration` for normal +The SDK does not expose skip-graceful-termination operations. +Use `TerminateOrchestration` for normal orchestration termination. `GetOrchestrationHistory` returns API-owned history records with validated event From afa500c0295d8c0316eface23959d49d4b554673 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:34:36 -0700 Subject: [PATCH 14/26] Use only V2 entity work items Remove legacy entity execution, state-elision handshakes, and trace forwarding. Require V2 routing metadata and remove the emulator-tests documentation section. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- CHANGELOG.md | 3 +- README.md | 4 +- client/grpc_worker_processor.go | 44 +++------ client/grpc_worker_test.go | 159 ++++++++++++++++++++------------ durabletaskscheduler/README.md | 38 +------- task/entity.go | 25 +++-- task/entity_object_test.go | 14 +-- task/entity_test.go | 25 ++--- task/executor.go | 21 ----- task/executor_contract.go | 5 +- tests/entity_executor_test.go | 99 +++++--------------- 11 files changed, 167 insertions(+), 270 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d463d709..31bf4581 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Added a flat catalogue of directly runnable DTS samples with outcome assertions, owned-resource cleanup, and process-level E2E validation selected by sample name using Go's `-run` flag. -- Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, distinct activity and sub-orchestration action trace contexts, and legacy entity-operation trace forwarding. +- Added the exported `task.CallActivityOption` type, per-activity tag options, completion-action tag propagation, and distinct activity and sub-orchestration action trace contexts. - Added the top-level `durabletaskscheduler` transport package, a dedicated resilient gRPC worker, DTS emulator tests, and an environment-driven sample. - Added advanced management APIs for bounded instance queries/listing, restart, and batch/filter purge polling. - Restored `RewindInstance`, `api.RewindOptions`, and `api.WithRewindReason` with request validation and typed errors. Added Python-aligned worker history replacement for real failed-activity and recursive child recovery, preserving successful work and renewing execution identity. Rewind returns after enqueueing, not recovery; activity retry-policy histories with retry timers remain unsupported, matching the pinned Python algorithm. Added a focused rewind sample. @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Removed +- Removed legacy entity work-item execution, state-elision handshakes, and legacy entity-operation trace forwarding. DTS workers accept only `EntityRequestV2`; the normalized entity batch representation remains internal. - Removed the unsupported SDK methods `SkipGracefulOrchestrationTerminations`, `CreateTaskHub`, and `DeleteTaskHub`, along with `api.CreateTaskHubOptions`, `api.WithRecreateTaskHub`, and the lifecycle-only `client.ErrTaskHubExists` sentinel. Provision and delete task hubs through the Azure control plane or Azure CLI. `RestartInstance`, normal termination, instance purge, and entity maintenance remain supported; `client.ErrTaskHubNotFound` still identifies a configured task hub that does not exist. - **Removed the sqlite and PostgreSQL storage backends** (`backend/sqlite` and `backend/postgres`) and the `modernc.org/sqlite` and `github.com/jackc/pgx/v5` dependencies. Durable Task Scheduler is now the only supported runtime, and this repository is the DTS Go SDK rather than an embeddable engine plus storage providers. Applications that hosted an embedded task hub must move to `durabletaskscheduler.NewClient` and `durabletaskscheduler.NewWorker`. - **Removed the standalone local gRPC server** (root `main.go` and its `Dockerfile`), along with the `--port`/`--host`/`--db` commands and the `.NET` client-SDK integration-test instructions that targeted it. diff --git a/README.md b/README.md index 4c66d6e5..80e93cce 100644 --- a/README.md +++ b/README.md @@ -397,7 +397,7 @@ The SDK also supports scheduled signals, orchestration calls, entity-to-entity s Entity names and operation names are matched case-insensitively using the same invariant rule as the .NET SDK, so a name resolves to the same entity in both SDKs. -The DTS worker accepts the legacy `EntityBatchRequest` work item and the current `EntityRequestV2` work item. +The DTS worker accepts only V2 entity work items (`EntityRequestV2`). Full sample: [samples/entity](./samples/entity). @@ -442,7 +442,7 @@ The SDK does not save the identity of the converter. A new converter must contin Use `api.WithTags`, `task.WithActivityTags`, and `task.WithSubOrchestrationTags` to attach user tags. An activity and a sub-orchestration inherit the tags of the parent orchestration. A tag on the action has priority over an inherited tag. The completion actions carry the current tags, so ContinueAsNew keeps them. -The client sends the sampled caller trace context when it schedules an orchestration or signals an entity. The worker adds separate action trace contexts for the service-owned activity and sub-orchestration spans. The worker does not emit duplicate local Durable Task spans. A legacy entity operation request also sends its operation trace context to the entity actions. The current DTS V2 entity request does not carry that source field. +The client sends the sampled caller trace context when it schedules an orchestration or signals an entity. The worker adds separate action trace contexts for the service-owned activity and sub-orchestration spans. The worker does not emit duplicate local Durable Task spans. V2 entity requests do not carry per-operation trace context, so entity-emitted actions cannot inherit it. Use `task.OrchestrationOptions.MaxEventsPerTurn` to limit the new events in one turn. If the worker uses only part of a batch, it sets `numEventsProcessed`. DTS then keeps the remaining events for the next replay. This count obeys the DTS work-item rules. The orchestration control markers do not count against the limit. diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index 0fd2e53d..f1882076 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -45,7 +45,8 @@ func (w *TaskHubGrpcWorker) consumeConnection(run *grpcWorkerRun, connection *gr } case *protos.WorkItem_EntityRequest: if err := w.dispatchEntity(run, connection, workItem.GetCompletionToken(), func(ctx context.Context) { - w.processEntityBatch(ctx, connection.client, workItem.GetCompletionToken(), request.EntityRequest, nil) + w.logger.Error("legacy entity work items are not supported; the scheduler must send EntityRequestV2") + w.abandonEntity(ctx, connection.client, workItem.GetCompletionToken()) }); err != nil { return observedMessage, err } @@ -569,19 +570,9 @@ func (w *TaskHubGrpcWorker) processEntityV2( w.abandonEntity(ctx, client, completionToken) return } - w.processEntityBatch(ctx, client, completionToken, batch, operationInfos) -} - -func (w *TaskHubGrpcWorker) processEntityBatch( - ctx context.Context, - client protos.TaskHubSidecarServiceClient, - completionToken string, - request *protos.EntityBatchRequest, - operationInfos []*protos.OperationInfo, -) { - entityID, parseErr := api.EntityIDFromString(request.GetInstanceId()) + entityID, parseErr := api.EntityIDFromString(batch.GetInstanceId()) if parseErr != nil { - w.logger.Errorf("%s: invalid entity instance ID: %v", request.GetInstanceId(), parseErr) + w.logger.Errorf("%s: invalid entity instance ID: %v", batch.GetInstanceId(), parseErr) w.abandonEntity(ctx, client, completionToken) return } @@ -590,7 +581,7 @@ func (w *TaskHubGrpcWorker) processEntityBatch( w.abandonEntity(ctx, client, completionToken) return } - if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, request); err != nil { + if err := largepayload.TransformEntityBatchRequest(ctx, w.options.largePayloads, batch); err != nil { w.logger.Errorf("%s: failed to hydrate entity batch payloads: %v", request.GetInstanceId(), err) if entityProcessingCanceled(ctx, err) { w.abandonEntity(ctx, client, completionToken) @@ -605,7 +596,7 @@ func (w *TaskHubGrpcWorker) processEntityBatch( w.abandonEntity(ctx, client, completionToken) return } - result, err := executeEntitySafely(ctx, executor, request) + result, err := executeEntitySafely(ctx, executor, batch) if err != nil { if entityProcessingCanceled(ctx, err) { w.logger.Warnf("%s: entity execution canceled; abandoning work item", request.GetInstanceId()) @@ -620,7 +611,7 @@ func (w *TaskHubGrpcWorker) processEntityBatch( result = newEntityBatchFailure(completionToken, missingResultErr) } result.CompletionToken = completionToken - if err := validateEntityBatchResult(result, len(request.Operations), operationInfos); err != nil { + if err := validateEntityBatchResult(result, operationInfos); err != nil { w.logger.Errorf("%s: invalid entity executor result: %v", request.GetInstanceId(), err) result = newEntityBatchFailure(completionToken, err) } @@ -685,33 +676,22 @@ func (w *TaskHubGrpcWorker) completeEntityBatchFailure( func validateEntityBatchResult( result *protos.EntityBatchResult, - operationCount int, operationInfos []*protos.OperationInfo, ) error { - if result.FailureDetails != nil || result.RequiresState { - if result.FailureDetails != nil && result.RequiresState { - return fmt.Errorf("entity result cannot contain both failure details and a state request") - } + if result.RequiresState { + return fmt.Errorf("V2 entity results cannot request state; every work item includes entity state") + } + if result.FailureDetails != nil { if len(result.Results) != 0 || len(result.Actions) != 0 || result.EntityState != nil || len(result.OperationInfos) != 0 { - return fmt.Errorf("entity batch failures and state requests must not contain partial effects") + return fmt.Errorf("entity batch failures must not contain partial effects") } result.OperationInfos = nil return nil } - if len(result.Results) != operationCount { - return fmt.Errorf( - "entity result count %d does not match operation count %d", - len(result.Results), - operationCount, - ) - } if len(result.OperationInfos) != 0 { return fmt.Errorf("entity executor result must not set operation routing metadata") } - if operationInfos == nil { - return nil - } if len(operationInfos) != len(result.Results) { return fmt.Errorf( "V2 entity result count %d does not match operation count %d", diff --git a/client/grpc_worker_test.go b/client/grpc_worker_test.go index 79ed0c85..02ae4ed7 100644 --- a/client/grpc_worker_test.go +++ b/client/grpc_worker_test.go @@ -1113,7 +1113,22 @@ func TestTaskHubGrpcWorkerAppliesActivityBackpressure(t *testing.T) { require.NoError(t, worker.Shutdown(context.Background())) } -func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { +func newFakeEntitySignalRequest(operation string, input *wrapperspb.StringValue) *protos.EntityRequest { + return &protos.EntityRequest{ + InstanceId: "@counter@key", + OperationRequests: []*protos.HistoryEvent{{ + EventType: &protos.HistoryEvent_EntityOperationSignaled{ + EntityOperationSignaled: &protos.EntityOperationSignaledEvent{ + RequestId: uuid.NewString(), + Operation: operation, + Input: input, + }, + }, + }}, + } +} + +func TestTaskHubGrpcWorkerCompletesV2EntityBatches(t *testing.T) { stream := newFakeWorkItemStream(2) client := &fakeSchedulerClient{stream: stream} worker := newFakeWorker(t, client, WithMaxConcurrentEntityWorkItems(1)) @@ -1121,6 +1136,7 @@ func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { executeEntity: func(_ context.Context, request *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { require.Equal(t, "@counter@key", request.InstanceId) require.Len(t, request.Operations, 1) + require.Equal(t, "1", request.Operations[0].Input.GetValue()) return &protos.EntityBatchResult{ Results: []*protos.OperationResult{{ ResultType: &protos.OperationResult_Success{ @@ -1131,14 +1147,11 @@ func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { }, nil }, } - legacyRequestID := uuid.NewString() - v2RequestID := uuid.NewString() + signal := newFakeEntitySignalRequest("add", wrapperspb.String("1")) + callRequestID := uuid.NewString() stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ - Request: &protos.WorkItem_EntityRequest{EntityRequest: &protos.EntityBatchRequest{ - InstanceId: "@counter@key", - Operations: []*protos.OperationRequest{{Operation: "add", RequestId: legacyRequestID}}, - }}, - CompletionToken: "legacy-token", + Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: signal}, + CompletionToken: "signal-token", }} stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ Request: &protos.WorkItem_EntityRequestV2{EntityRequestV2: &protos.EntityRequest{ @@ -1146,14 +1159,16 @@ func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { OperationRequests: []*protos.HistoryEvent{{ EventType: &protos.HistoryEvent_EntityOperationCalled{ EntityOperationCalled: &protos.EntityOperationCalledEvent{ - RequestId: v2RequestID, - Operation: "add", - ParentInstanceId: wrapperspb.String("caller"), + RequestId: callRequestID, + Operation: "add", + Input: wrapperspb.String("1"), + ParentInstanceId: wrapperspb.String("caller"), + ParentExecutionId: wrapperspb.String("caller-execution"), }, }, }}, }}, - CompletionToken: "v2-token", + CompletionToken: "call-token", }} ctx, cancel := context.WithCancel(context.Background()) @@ -1164,16 +1179,53 @@ func TestTaskHubGrpcWorkerCompletesLegacyAndV2EntityBatches(t *testing.T) { return len(client.entityCompletions) == 2 }, time.Second, time.Millisecond) client.mu.Lock() - require.Equal(t, "legacy-token", client.entityCompletions[0].CompletionToken) - require.Equal(t, "v2-token", client.entityCompletions[1].CompletionToken) + require.Equal(t, "signal-token", client.entityCompletions[0].CompletionToken) + require.Len(t, client.entityCompletions[0].OperationInfos, 1) + require.Equal(t, signal.OperationRequests[0].GetEntityOperationSignaled().RequestId, client.entityCompletions[0].OperationInfos[0].RequestId) + require.Nil(t, client.entityCompletions[0].OperationInfos[0].ResponseDestination) + require.Equal(t, "call-token", client.entityCompletions[1].CompletionToken) require.Len(t, client.entityCompletions[1].OperationInfos, 1) - require.Equal(t, v2RequestID, client.entityCompletions[1].OperationInfos[0].RequestId) + require.Equal(t, callRequestID, client.entityCompletions[1].OperationInfos[0].RequestId) require.Equal(t, "caller", client.entityCompletions[1].OperationInfos[0].ResponseDestination.InstanceId) + require.Equal(t, "caller-execution", client.entityCompletions[1].OperationInfos[0].ResponseDestination.ExecutionId.GetValue()) client.mu.Unlock() cancel() require.NoError(t, worker.Shutdown(context.Background())) } +func TestTaskHubGrpcWorkerRejectsLegacyEntityWorkItems(t *testing.T) { + stream := newFakeWorkItemStream(1) + client := &fakeSchedulerClient{stream: stream} + worker := newFakeWorker(t, client) + var executions atomic.Int32 + worker.executor = &recordingExecutor{ + executeEntity: func(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) { + executions.Add(1) + return &protos.EntityBatchResult{}, nil + }, + } + stream.results <- fakeWorkItemResult{item: &protos.WorkItem{ + Request: &protos.WorkItem_EntityRequest{EntityRequest: &protos.EntityBatchRequest{ + InstanceId: "@counter@key", + Operations: []*protos.OperationRequest{{Operation: "add", RequestId: uuid.NewString()}}, + }}, + CompletionToken: "legacy-token", + }} + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + require.NoError(t, worker.Start(ctx)) + require.Eventually(t, func() bool { + client.mu.Lock() + defer client.mu.Unlock() + return client.entityAbandonAttempts == 1 + }, time.Second, time.Millisecond) + cancel() + require.NoError(t, worker.Shutdown(context.Background())) + require.Zero(t, executions.Load()) + require.Empty(t, client.entityCompletions) +} + func TestTaskHubGrpcWorkerForwardsEntityParameters(t *testing.T) { client := new(fakeSchedulerClient) worker := newFakeWorker(t, client) @@ -1244,15 +1296,11 @@ func TestTaskHubGrpcWorkerCompletesEntityFrameworkFailure(t *testing.T) { }, } - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "entity-token", - &protos.EntityBatchRequest{ - InstanceId: "@counter@key", - Operations: []*protos.OperationRequest{{Operation: "get"}}, - }, - nil, + newFakeEntitySignalRequest("get", nil), ) require.Len(t, client.entityCompletions, 1) @@ -1291,12 +1339,11 @@ func TestTaskHubGrpcWorkerCompletesEntityExecutorPanicAndNilResult(t *testing.T) client := new(fakeSchedulerClient) worker := newFakeWorker(t, client) worker.executor = &recordingExecutor{executeEntity: test.executeEntity} - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "entity-token", - &protos.EntityBatchRequest{InstanceId: "@counter@key"}, - nil, + &protos.EntityRequest{InstanceId: "@counter@key"}, ) require.Len(t, client.entityCompletions, 1) require.Contains(t, client.entityCompletions[0].FailureDetails.ErrorMessage, test.errorContains) @@ -1344,15 +1391,11 @@ func TestTaskHubGrpcWorkerCompletesEntityPayloadFailures(t *testing.T) { ThresholdBytes: 1, MaxPayloadBytes: 1024, })) - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "token", - &protos.EntityBatchRequest{ - InstanceId: "@counter@key", - Operations: []*protos.OperationRequest{{Operation: "run", Input: input}}, - }, - nil, + newFakeEntitySignalRequest("run", input), ) require.Len(t, client.entityCompletions, 1) require.Contains(t, client.entityCompletions[0].FailureDetails.ErrorMessage, "hydrate failed") @@ -1367,15 +1410,11 @@ func TestTaskHubGrpcWorkerCompletesEntityPayloadFailures(t *testing.T) { ThresholdBytes: 1, MaxPayloadBytes: 1024, })) - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "token", - &protos.EntityBatchRequest{ - InstanceId: "@counter@key", - Operations: []*protos.OperationRequest{{Operation: "run", Input: input}}, - }, - nil, + newFakeEntitySignalRequest("run", input), ) require.Empty(t, client.entityCompletions) require.Equal(t, 1, client.entityAbandonAttempts) @@ -1399,12 +1438,11 @@ func TestTaskHubGrpcWorkerCompletesEntityPayloadFailures(t *testing.T) { return &protos.EntityBatchResult{EntityState: wrapperspb.String("large")}, nil }, } - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "token", - &protos.EntityBatchRequest{InstanceId: "@counter@key"}, - nil, + &protos.EntityRequest{InstanceId: "@counter@key"}, ) require.Len(t, client.entityCompletions, 1) completion := client.entityCompletions[0] @@ -1431,33 +1469,38 @@ func TestTaskHubGrpcWorkerAbandonsWhenEntityFailureFallbackCannotComplete(t *tes }, } - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "token", - &protos.EntityBatchRequest{InstanceId: "@counter@key"}, - nil, + &protos.EntityRequest{InstanceId: "@counter@key"}, ) require.Equal(t, 1, client.entityAbandonAttempts) } func TestTaskHubGrpcWorkerConvertsInvalidEntityResultsToBatchFailure(t *testing.T) { tests := []struct { - name string - result *protos.EntityBatchResult - operationInfos []*protos.OperationInfo - errorContains string + name string + result *protos.EntityBatchResult + errorContains string }{ { - name: "V1 result count mismatch", + name: "V2 result count mismatch", result: &protos.EntityBatchResult{}, errorContains: "does not match operation count", }, { - name: "V2 result count mismatch", - result: &protos.EntityBatchResult{}, - operationInfos: []*protos.OperationInfo{{RequestId: uuid.NewString()}}, - errorContains: "does not match operation count", + name: "state elision is unsupported", + result: &protos.EntityBatchResult{RequiresState: true}, + errorContains: "V2 entity results cannot request state", + }, + { + name: "executor supplies routing metadata", + result: &protos.EntityBatchResult{ + Results: []*protos.OperationResult{{}}, + OperationInfos: []*protos.OperationInfo{{RequestId: uuid.NewString()}}, + }, + errorContains: "must not set operation routing metadata", }, { name: "batch failure with partial state", @@ -1478,15 +1521,11 @@ func TestTaskHubGrpcWorkerConvertsInvalidEntityResultsToBatchFailure(t *testing. }, } - worker.processEntityBatch( + worker.processEntityV2( context.Background(), client, "entity-token", - &protos.EntityBatchRequest{ - InstanceId: "@counter@key", - Operations: []*protos.OperationRequest{{Operation: "get"}}, - }, - test.operationInfos, + newFakeEntitySignalRequest("get", nil), ) require.Len(t, client.entityCompletions, 1) @@ -1494,10 +1533,16 @@ func TestTaskHubGrpcWorkerConvertsInvalidEntityResultsToBatchFailure(t *testing. require.Contains(t, completion.FailureDetails.ErrorMessage, test.errorContains) require.Empty(t, completion.OperationInfos) require.Nil(t, completion.EntityState) + require.False(t, completion.RequiresState) }) } } +func TestValidateEntityBatchResultRequiresV2Routing(t *testing.T) { + result := &protos.EntityBatchResult{Results: []*protos.OperationResult{{}}} + require.ErrorContains(t, validateEntityBatchResult(result, nil), "does not match operation count") +} + func TestTaskHubGrpcWorkerAbandonsInvalidV2EntityWithBoundedRetry(t *testing.T) { stream := newFakeWorkItemStream(1) client := &fakeSchedulerClient{stream: stream, entityAbandonFailures: 2} diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 137db481..9f65e679 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -356,40 +356,12 @@ cancels them only if the shutdown context expires. | Scheduled-task capability | Supported; register system tasks and opt in with `durabletaskscheduler.WithScheduledTasks()` | | Azure Blob `blob:v2` payloads | Supported with connection-string or identity authentication and .NET-compatible gzip/token semantics | | Large-payload capability | Supported and advertised only when a store/resolver is configured | -| Durable entities | Supported: legacy and V2 work items, scheduled signals, calls, queries, and critical sections | +| Durable entities | Supported: V2 work items, scheduled signals, calls, queries, and critical sections | | Status-based instance-ID deduplication and replacement | Supported through `api.OrchestrationIDReusePolicy.DedupeStatuses` | | History export jobs (preview) | Buffers each complete history in worker memory and writes `api.HistoryEvent` JSON/JSONL; the schema version defaults to preview value `1.0` and is caller-configurable, so assess memory and schema compatibility before enabling | | Sandbox worker profiles | Not implemented | -The current V2 protobuf cannot carry per-operation trace context or request time -to an entity worker, and it has no properties map for legacy extended-session -state elision. DTS therefore sends entity state on every V2 work item; causal -trace metadata on entity-emitted actions is best-effort. - -## Emulator tests - -On Apple silicon with Apple Container, the current MCR emulator image runs -under Rosetta: - -```bash -container image pull mcr.microsoft.com/dts/dts-emulator:latest -container run --detach --name dts-emulator \ - --arch amd64 --rosetta \ - --publish 8080:8080 --publish 8082:8082 \ - --env DTS_TASK_HUB_NAMES=default \ - mcr.microsoft.com/dts/dts-emulator:latest -``` - -The integration suite is environment-gated: - -```bash -DTS_EMULATOR_ENDPOINT=http://127.0.0.1:8080 \ -DTS_TASK_HUB=default \ -go test ./tests/durabletaskscheduler -count=1 -``` - -Azurite-backed blob tests additionally use: - -```bash -export AZURITE_CONNECTION_STRING='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;' -``` +The V2 protobuf does not carry per-operation trace context or request time to an +entity worker. DTS sends entity state on every work item; a null state means the +entity does not exist. Entity-emitted actions cannot inherit per-operation trace +context. diff --git a/task/entity.go b/task/entity.go index ca3aaae4..0d0cd383 100644 --- a/task/entity.go +++ b/task/entity.go @@ -57,7 +57,6 @@ type EntityContext struct { ctx context.Context logger *slog.Logger converter api.DataConverter - parentTrace *protos.TraceContext } type entityPayload struct { @@ -205,12 +204,11 @@ func (ctx *EntityContext) signalEntity(entityID api.EntityID, operationName stri Id: ctx.nextActionID(), OperationActionType: &protos.OperationAction_SendSignal{ SendSignal: &protos.SendSignalAction{ - InstanceId: entityID.String(), - Name: operationName, - Input: rawInput, - RequestTime: timestampOrNil(ctx.currentTime), - ScheduledTime: timestampOrNil(scheduledTime), - ParentTraceContext: helpers.CloneTraceContext(ctx.parentTrace), + InstanceId: entityID.String(), + Name: operationName, + Input: rawInput, + RequestTime: timestampOrNil(ctx.currentTime), + ScheduledTime: timestampOrNil(scheduledTime), }, }, } @@ -241,13 +239,12 @@ func (ctx *EntityContext) StartNewOrchestration(name string, opts ...EntityStart Id: ctx.nextActionID(), OperationActionType: &protos.OperationAction_StartNewOrchestration{ StartNewOrchestration: &protos.StartNewOrchestrationAction{ - InstanceId: options.instanceID, - Name: name, - Version: options.version, - Input: options.rawInput, - ScheduledTime: timestampOrNil(options.scheduledTime), - RequestTime: timestampOrNil(ctx.currentTime), - ParentTraceContext: helpers.CloneTraceContext(ctx.parentTrace), + InstanceId: options.instanceID, + Name: name, + Version: options.version, + Input: options.rawInput, + ScheduledTime: timestampOrNil(options.scheduledTime), + RequestTime: timestampOrNil(ctx.currentTime), }, }, } diff --git a/task/entity_object_test.go b/task/entity_object_test.go index 564afe5b..c991a002 100644 --- a/task/entity_object_test.go +++ b/task/entity_object_test.go @@ -10,7 +10,6 @@ import ( "github.com/microsoft/durabletask-go/internal/protos" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -93,7 +92,7 @@ func Test_EntityObjectFactory_StateLifecycleAndDispatch(t *testing.T) { require.JSONEq(t, `{"value":10}`, result.EntityState.GetValue()) } -func Test_EntityFactory_RunsAfterStateHandshake(t *testing.T) { +func Test_EntityFactory_RunsWithoutExistingState(t *testing.T) { registry := NewTaskRegistry() factoryCalls := 0 require.NoError(t, registry.AddEntityFactoryN("counter", func(EntityFactoryContext) (EntityBatch, error) { @@ -106,18 +105,9 @@ func Test_EntityFactory_RunsAfterStateHandshake(t *testing.T) { result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ InstanceId: "@counter@key", - Properties: map[string]*structpb.Value{ - "IncludeState": structpb.NewBoolValue(false), - }, - }) - require.NoError(t, err) - require.True(t, result.RequiresState) - require.Zero(t, factoryCalls) - - _, err = executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ - InstanceId: "@counter@key", }) require.NoError(t, err) + require.False(t, result.RequiresState) require.Equal(t, 1, factoryCalls) } diff --git a/task/entity_test.go b/task/entity_test.go index 1235f286..b122d46f 100644 --- a/task/entity_test.go +++ b/task/entity_test.go @@ -5,10 +5,8 @@ import ( "testing" "github.com/microsoft/durabletask-go/api" - "github.com/microsoft/durabletask-go/internal/protos" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/wrapperspb" ) func presentEntityPayload(value string) entityPayload { @@ -107,14 +105,9 @@ func Test_EntityContext_RawState(t *testing.T) { } func Test_EntityContext_SignalEntity(t *testing.T) { - parentTrace := &protos.TraceContext{ - TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", - TraceState: wrapperspb.String("vendor=value"), - } ctx := &EntityContext{ - ID: api.NewEntityID("test", "key1"), - Operation: "op", - parentTrace: parentTrace, + ID: api.NewEntityID("test", "key1"), + Operation: "op", } err := ctx.SignalEntity(api.NewEntityID("other", "key2"), "increment", 5) @@ -127,8 +120,7 @@ func Test_EntityContext_SignalEntity(t *testing.T) { assert.Equal(t, "@other@key2", signal.InstanceId) assert.Equal(t, "increment", signal.Name) assert.Equal(t, "5", signal.Input.GetValue()) - assert.Equal(t, parentTrace, signal.ParentTraceContext) - assert.NotSame(t, parentTrace, signal.ParentTraceContext) + assert.Nil(t, signal.ParentTraceContext) } func Test_EntityContext_TypedNilSignalInputIsAbsent(t *testing.T) { @@ -167,13 +159,9 @@ func Test_EntityContext_SignalEntity_RejectsInvalidEntityID(t *testing.T) { } func Test_EntityContext_StartNewOrchestration(t *testing.T) { - parentTrace := &protos.TraceContext{ - TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", - } ctx := &EntityContext{ - ID: api.NewEntityID("test", "key1"), - Operation: "op", - parentTrace: parentTrace, + ID: api.NewEntityID("test", "key1"), + Operation: "op", } err := ctx.StartNewOrchestration("MyOrchestrator", @@ -189,8 +177,7 @@ func Test_EntityContext_StartNewOrchestration(t *testing.T) { assert.Equal(t, "MyOrchestrator", startOrch.Name) assert.Equal(t, "my-instance", startOrch.InstanceId) assert.Equal(t, `"hello"`, startOrch.Input.GetValue()) - assert.Equal(t, parentTrace, startOrch.ParentTraceContext) - assert.NotSame(t, parentTrace, startOrch.ParentTraceContext) + assert.Nil(t, startOrch.ParentTraceContext) } func Test_EntityContext_StartNewOrchestration_RawInput(t *testing.T) { diff --git a/task/executor.go b/task/executor.go index 57fd2484..cc26ec2d 100644 --- a/task/executor.go +++ b/task/executor.go @@ -14,7 +14,6 @@ import ( "github.com/microsoft/durabletask-go/internal/failure" "github.com/microsoft/durabletask-go/internal/helpers" "github.com/microsoft/durabletask-go/internal/protos" - "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -373,11 +372,6 @@ func (te *taskExecutor) ExecuteEntity( if err != nil { return nil, fmt.Errorf("invalid entity instance ID: %w", err) } - if !entityStateIncluded(req.Properties) { - // The scheduler omitted the entity state, so the batch cannot run until - // it is retransmitted. - return &protos.EntityBatchResult{RequiresState: true}, nil - } factory, ok := te.Registry.getEntityFactory(entityID.Name) if !ok { result := &protos.EntityBatchResult{ @@ -481,20 +475,6 @@ func (te *taskExecutor) ExecuteEntity( return result, nil } -// entityStateIncluded reports whether the scheduler attached entity state to the -// work item. It mirrors .NET's GrpcInstanceRunnerUtils: a missing property, or -// one whose value is not a boolean, means the state was included. -func entityStateIncluded(properties map[string]*structpb.Value) bool { - property, exists := properties["IncludeState"] - if !exists { - return true - } - if _, isBool := property.GetKind().(*structpb.Value_BoolValue); !isBool { - return true - } - return property.GetBoolValue() -} - func (te *taskExecutor) newEntityContext( ctx context.Context, entityID api.EntityID, @@ -536,7 +516,6 @@ func (te *taskExecutor) newEntityContext( ctx: ctx, logger: logger, converter: te.converter, - parentTrace: helpers.CloneTraceContext(operation.GetTraceContext()), } } diff --git a/task/executor_contract.go b/task/executor_contract.go index afb46fd6..1d5c3d69 100644 --- a/task/executor_contract.go +++ b/task/executor_contract.go @@ -30,8 +30,9 @@ type Executor interface { Shutdown(ctx context.Context) error } -// EntityExecutor is implemented by executors that process durable entity -// operation batches. +// EntityExecutor processes normalized operation batches from V2 entity work +// items. EntityBatchRequest is an internal execution model, not a supported +// legacy work-item envelope. type EntityExecutor interface { ExecuteEntity(context.Context, *protos.EntityBatchRequest) (*protos.EntityBatchResult, error) } diff --git a/tests/entity_executor_test.go b/tests/entity_executor_test.go index 41365191..632f569d 100644 --- a/tests/entity_executor_test.go +++ b/tests/entity_executor_test.go @@ -12,7 +12,6 @@ import ( "github.com/microsoft/durabletask-go/task" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/structpb" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -20,7 +19,7 @@ func newEntityExecutor(r *task.TaskRegistry) task.EntityExecutor { return task.NewTaskExecutor(r).(task.EntityExecutor) } -func Test_Executor_EntityActionsInheritOperationTraceContext(t *testing.T) { +func Test_Executor_EntityActionsWithoutOperationTraceContext(t *testing.T) { registry := task.NewTaskRegistry() require.NoError(t, registry.AddEntityN("router", func(ctx *task.EntityContext) (any, error) { if err := ctx.SignalEntity(api.NewEntityID("target", "one"), "signal", nil); err != nil { @@ -31,27 +30,20 @@ func Test_Executor_EntityActionsInheritOperationTraceContext(t *testing.T) { task.WithEntityStartOrchestrationInstanceID("child-instance"), ) })) - parent := &protos.TraceContext{ - TraceParent: "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01", - TraceState: wrapperspb.String("vendor=value"), - } result, err := newEntityExecutor(registry).ExecuteEntity( context.Background(), &protos.EntityBatchRequest{ InstanceId: "@router@key", Operations: []*protos.OperationRequest{{ - Operation: "route", - RequestId: "request", - TraceContext: parent, + Operation: "route", + RequestId: "request", }}, }, ) require.NoError(t, err) require.Len(t, result.Actions, 2) - require.Equal(t, parent, result.Actions[0].GetSendSignal().GetParentTraceContext()) - require.Equal(t, parent, result.Actions[1].GetStartNewOrchestration().GetParentTraceContext()) - require.NotSame(t, parent, result.Actions[0].GetSendSignal().GetParentTraceContext()) - require.NotSame(t, parent, result.Actions[1].GetStartNewOrchestration().GetParentTraceContext()) + require.Nil(t, result.Actions[0].GetSendSignal().GetParentTraceContext()) + require.Nil(t, result.Actions[1].GetStartNewOrchestration().GetParentTraceContext()) } func Test_Executor_EntityBasicOperation(t *testing.T) { @@ -777,89 +769,42 @@ func Test_Executor_EntityStartOrchestrationIDIsStableAcrossRetry(t *testing.T) { ) } -// The scheduler signals an elided entity state with the "IncludeState" property, -// matching Microsoft.DurableTask's GrpcInstanceRunnerUtils. -func Test_Executor_EntityStateElisionRequestsState(t *testing.T) { - var invoked bool - r := task.NewTaskRegistry() - require.NoError(t, r.AddEntityN("cached", func(*task.EntityContext) (any, error) { - invoked = true - return nil, nil - })) - executor := newEntityExecutor(r) - result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ - InstanceId: "@cached@key", - Operations: []*protos.OperationRequest{{ - Operation: "get", - RequestId: uuid.NewString(), - }}, - Properties: map[string]*structpb.Value{ - "IncludeState": structpb.NewBoolValue(false), - }, - }) - require.NoError(t, err) - assert.True(t, result.RequiresState) - assert.False(t, invoked) - assert.Empty(t, result.Results) - assert.Nil(t, result.EntityState) -} - -// A state request is driven by the property alone, so an attached state does not -// suppress it and an unregistered entity does not mask it. -func Test_Executor_EntityStateElisionIgnoresAttachedStateAndRegistration(t *testing.T) { - executor := newEntityExecutor(task.NewTaskRegistry()) - result, err := executor.ExecuteEntity(context.Background(), &protos.EntityBatchRequest{ - InstanceId: "@unregistered@key", - EntityState: wrapperspb.String("42"), - Operations: []*protos.OperationRequest{{ - Operation: "get", - RequestId: uuid.NewString(), - }}, - Properties: map[string]*structpb.Value{ - "IncludeState": structpb.NewBoolValue(false), - }, - }) - require.NoError(t, err) - assert.True(t, result.RequiresState) - assert.Empty(t, result.Results) -} - -// A missing or non-boolean property means the state was included. -func Test_Executor_EntityStateElisionDefaultsToIncluded(t *testing.T) { - properties := map[string]map[string]*structpb.Value{ - "absent": nil, - "wrong-type": { - "IncludeState": structpb.NewStringValue("false"), - }, - "misspelled": { - "includestate": structpb.NewBoolValue(false), - }, - "true": { - "IncludeState": structpb.NewBoolValue(true), - }, +func Test_Executor_EntityStateAlwaysIncluded(t *testing.T) { + states := map[string]*wrapperspb.StringValue{ + "new entity": nil, + "existing entity": wrapperspb.String("42"), } - for name, property := range properties { + for name, state := range states { t.Run(name, func(t *testing.T) { var invoked bool r := task.NewTaskRegistry() - require.NoError(t, r.AddEntityN("cached", func(*task.EntityContext) (any, error) { + require.NoError(t, r.AddEntityN("counter", func(ctx *task.EntityContext) (any, error) { invoked = true + assert.Equal(t, state != nil, ctx.HasState()) + if state != nil { + value, hasValue := ctx.GetRawState() + assert.True(t, hasValue) + assert.Equal(t, state.Value, value) + } return nil, nil })) result, err := newEntityExecutor(r).ExecuteEntity( context.Background(), &protos.EntityBatchRequest{ - InstanceId: "@cached@key", + InstanceId: "@counter@key", + EntityState: state, Operations: []*protos.OperationRequest{{ Operation: "get", RequestId: uuid.NewString(), }}, - Properties: property, }, ) require.NoError(t, err) assert.False(t, result.RequiresState) assert.True(t, invoked) + require.Len(t, result.Results, 1) + require.NotNil(t, result.Results[0].GetSuccess()) + assert.Equal(t, state, result.EntityState) }) } } From c18ff4aad299be86c7c473bbd7537f1bf744e61e Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 12:35:47 -0700 Subject: [PATCH 15/26] Remove legacy chunking note from DTS feature matrix Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- durabletaskscheduler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 9f65e679..fd0742ad 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -345,7 +345,7 @@ cancels them only if the shutdown context expires. | Bounded orchestration/activity/entity concurrency | Supported | | Work-item filters for orchestrations, activities, and entities | Supported | | Completion tokens and abandon RPCs | Supported | -| Oversized orchestration responses | Blob-externalized before send; residual responses above the effective limit fail once with non-retriable guidance; the Go worker does not use deprecated legacy chunking | +| Oversized orchestration responses | Blob-externalized before send; residual responses above the effective limit fail once with non-retriable guidance | | Health pings, silent-disconnect detection, auth/RBAC recovery, and channel recreation | Supported | | Public orchestration history | Supported through buffered and callback-streaming API-owned records | | Version-aware registry dispatch and controlled unversioned fallback | Supported | From 708181ca3775862a76ff6328f0ae010aabeb1d57 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:10:45 -0700 Subject: [PATCH 16/26] fix: preserve carryover payloads and mixed version routes Verify filesystem payload integrity, cover carryover with memory/file/Blob stores and strict mixed-version dispatch end to end, and document the chosen Python-compatible entity lock ordering. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 8 ++ client/grpc_worker.go | 6 +- client/grpc_worker_test.go | 33 +++++ internal/largepayload/codec.go | 6 +- internal/largepayload/codec_test.go | 47 +++++++ payload/file.go | 13 ++ payload/store_test.go | 45 ++++++ .../dts_payload_routing_review_test.go | 132 ++++++++++++++++++ 8 files changed, 288 insertions(+), 2 deletions(-) create mode 100644 tests/durabletaskscheduler/dts_payload_routing_review_test.go diff --git a/README.md b/README.md index 80e93cce..0ab22f66 100644 --- a/README.md +++ b/README.md @@ -395,6 +395,14 @@ The SDK supports raw entity functions, state-struct dispatch, and persistent ent The SDK also supports scheduled signals, orchestration calls, entity-to-entity signals, queries, cleanup, and ordered critical sections across more than one entity. +Multi-entity locks use lexicographic ordering of serialized `@name@key` IDs, +matching the Python SDK's ordering. The .NET SDK currently orders the name and +key separately using culture-sensitive comparison. These orders can differ, +even for names such as `counter` and `counter2`. Do not use Go/Python and .NET +orchestrations to acquire overlapping multi-entity lock sets concurrently; +different acquisition orders can deadlock. Keep those lock sets isolated until +the SDKs share an ordering contract. + Entity names and operation names are matched case-insensitively using the same invariant rule as the .NET SDK, so a name resolves to the same entity in both SDKs. The DTS worker accepts only V2 entity work items (`EntityRequestV2`). diff --git a/client/grpc_worker.go b/client/grpc_worker.go index 2ca3cad7..d1fa6110 100644 --- a/client/grpc_worker.go +++ b/client/grpc_worker.go @@ -795,9 +795,13 @@ func taskRegistrationsToFilters( if versioning != nil && versioning.MatchStrategy == task.VersionMatchStrict { if _, allowed := allowedUnversioned[strings.ToLower(group.name)]; allowed { if unversioned, ok := group.versions[""]; ok { + versions := []string{unversioned} + if _, registered := group.versions[strings.ToLower(versioning.Version)]; registered && versioning.Version != "" { + versions = append(versions, versioning.Version) + } filters = append(filters, WorkItemFilter{ Name: group.name, - Versions: []string{unversioned}, + Versions: versions, }) continue } diff --git a/client/grpc_worker_test.go b/client/grpc_worker_test.go index 02ae4ed7..1582daf6 100644 --- a/client/grpc_worker_test.go +++ b/client/grpc_worker_test.go @@ -629,6 +629,39 @@ func TestStrictAutoFiltersPreserveAllowedUnversionedOrchestrator(t *testing.T) { require.Contains(t, filters.Orchestrations, WorkItemFilter{Name: "application", Versions: []string{"1.0"}}) } +func TestStrictAutoFiltersPreserveMixedRegistrations(t *testing.T) { + for _, workerVersion := range []string{"V1", ""} { + t.Run(workerVersion, func(t *testing.T) { + registrations := []task.TaskRegistration{ + {Name: "Mixed"}, + {Name: "mixed", Version: "v1"}, + {Name: "Mixed", Version: "v2"}, + } + snapshot := task.TaskRegistrySnapshot{Orchestrators: registrations, Activities: registrations} + allowed := map[string]struct{}{"mixed": {}} + filters := workItemFiltersFromRegistry(snapshot, + &task.VersioningOptions{Version: workerVersion, MatchStrategy: task.VersionMatchStrict}, + allowed, allowed) + expected := []string{""} + if workerVersion != "" { + expected = append(expected, workerVersion) + } + require.Equal(t, []WorkItemFilter{{Name: "Mixed", Versions: expected}}, filters.Orchestrations) + require.Equal(t, filters.Orchestrations, filters.Activities) + normalized, err := cloneWorkItemFilters(filters) + require.NoError(t, err) + wire := workItemFiltersToProto(normalized) + require.Equal(t, wire.Orchestrations[0].Versions, wire.Activities[0].Versions) + require.Len(t, wire.Orchestrations[0].Versions, len(expected)) + for _, orchestrator := range []bool{false, true} { + require.True(t, matchesWorkItemFilters(normalized, orchestrator, "MIXED", "")) + require.Equal(t, workerVersion != "", matchesWorkItemFilters(normalized, orchestrator, "mixed", "v1")) + require.False(t, matchesWorkItemFilters(normalized, orchestrator, "mixed", "v2")) + } + }) + } +} + // TestStrictAutoFiltersPreserveAllowedUnversionedActivity keeps a system // component's unversioned activities routable under strict worker versioning. // An activity inherits its caller's version, so an unversioned system diff --git a/internal/largepayload/codec.go b/internal/largepayload/codec.go index e401193c..97915c17 100644 --- a/internal/largepayload/codec.go +++ b/internal/largepayload/codec.go @@ -246,7 +246,11 @@ func TransformOrchestratorResponse( case action.GetSendEvent() != nil: plan.add(&action.GetSendEvent().Data) case action.GetCompleteOrchestration() != nil: - plan.add(&action.GetCompleteOrchestration().Result) + completion := action.GetCompleteOrchestration() + plan.add(&completion.Result) + for _, event := range completion.CarryoverEvents { + planHistoryEvent(plan, event) + } case action.GetTerminateOrchestration() != nil: plan.add(&action.GetTerminateOrchestration().Reason) case action.GetSendEntityMessage() != nil: diff --git a/internal/largepayload/codec_test.go b/internal/largepayload/codec_test.go index ae0c143b..8bb840f0 100644 --- a/internal/largepayload/codec_test.go +++ b/internal/largepayload/codec_test.go @@ -3,12 +3,14 @@ package largepayload import ( "context" "errors" + "strings" "testing" "github.com/microsoft/durabletask-go/api" "github.com/microsoft/durabletask-go/internal/protos" "github.com/microsoft/durabletask-go/payload" "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -109,6 +111,51 @@ func TestNativeLargePayloadThresholdIsInclusive(t *testing.T) { require.Equal(t, "blob:v2:https://account.example/payload", externalized.GetValue()) } +func TestTransformContinueAsNewCarryoverEvents(t *testing.T) { + store := payload.NewMemoryStore() + options := &api.LargePayloadOptions{Store: store, Resolver: store} + large := strings.Repeat("x", 5*1024*1024) + events := []*protos.HistoryEvent{ + { + EventId: 42, + EventType: &protos.HistoryEvent_EventRaised{ + EventRaised: &protos.EventRaisedEvent{Name: "large", Input: wrapperspb.String(large)}, + }, + }, + nil, + { + EventId: 43, + EventType: &protos.HistoryEvent_EventRaised{ + EventRaised: &protos.EventRaisedEvent{Name: "small", Input: wrapperspb.String("small")}, + }, + }, + } + completion := &protos.CompleteOrchestrationAction{ + OrchestrationStatus: protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, + Result: wrapperspb.String("next"), + CarryoverEvents: events, + } + response := &protos.OrchestratorResponse{Actions: []*protos.OrchestratorAction{{ + OrchestratorActionType: &protos.OrchestratorAction_CompleteOrchestration{ + CompleteOrchestration: completion, + }, + }}} + require.NoError(t, TransformOrchestratorResponse(context.Background(), options, response)) + require.Less(t, proto.Size(response), 4096) + require.Equal(t, protos.OrchestrationStatus_ORCHESTRATION_STATUS_CONTINUED_AS_NEW, completion.OrchestrationStatus) + require.Equal(t, int32(42), events[0].EventId) + require.Equal(t, int32(43), events[2].EventId) + require.Equal(t, "small", events[2].GetEventRaised().GetInput().GetValue()) + reference := events[0].GetEventRaised().GetInput().GetValue() + require.NotEqual(t, large, reference) + require.NoError(t, TransformOrchestratorResponse(context.Background(), options, response)) + require.Equal(t, reference, events[0].GetEventRaised().GetInput().GetValue()) + + require.NoError(t, TransformOrchestratorRequest(context.Background(), options, + &protos.OrchestratorRequest{NewEvents: events})) + require.Equal(t, large, events[0].GetEventRaised().GetInput().GetValue()) +} + func TestTransformOrchestratorResponsePayloadFields(t *testing.T) { store := payload.NewMemoryStore() options := &api.LargePayloadOptions{ diff --git a/payload/file.go b/payload/file.go index 75fed472..59af6feb 100644 --- a/payload/file.go +++ b/payload/file.go @@ -56,6 +56,9 @@ func (s *FileStore) Store(ctx context.Context, payload []byte) (string, error) { location := "file://sha256/" + hash path := filepath.Join(s.root, hash+".payload") if _, err := os.Stat(path); err == nil { + if _, err := s.Resolve(ctx, location); err != nil { + return "", fmt.Errorf("failed to verify existing payload file: %w", err) + } return location, nil } else if !errors.Is(err, os.ErrNotExist) { return "", fmt.Errorf("failed to inspect payload file: %w", err) @@ -83,6 +86,9 @@ func (s *FileStore) Store(ctx context.Context, payload []byte) (string, error) { } if err := os.Rename(tempPath, path); err != nil { if _, statErr := os.Stat(path); statErr == nil { + if _, verifyErr := s.Resolve(ctx, location); verifyErr != nil { + return "", fmt.Errorf("failed to verify existing payload file: %w", verifyErr) + } return location, nil } return "", fmt.Errorf("failed to publish payload file: %w", err) @@ -92,6 +98,9 @@ func (s *FileStore) Store(ctx context.Context, payload []byte) (string, error) { // Resolve reads a payload location created by this store. func (s *FileStore) Resolve(ctx context.Context, location string) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } hash, err := parseFileLocation(location) if err != nil { return nil, err @@ -112,6 +121,10 @@ func (s *FileStore) Resolve(ctx context.Context, location string) ([]byte, error if _, err := io.ReadFull(&contextReader{ctx: ctx, reader: file}, payload); err != nil { return nil, fmt.Errorf("failed to read payload file: %w", err) } + digest := sha256.Sum256(payload) + if !strings.EqualFold(hex.EncodeToString(digest[:]), hash) { + return nil, fmt.Errorf("%w: file SHA-256 mismatch", api.ErrLargePayloadIntegrity) + } return payload, nil } diff --git a/payload/store_test.go b/payload/store_test.go index 1de691c6..63ac1ef3 100644 --- a/payload/store_test.go +++ b/payload/store_test.go @@ -2,6 +2,8 @@ package payload import ( "context" + "os" + "path/filepath" "testing" "github.com/microsoft/durabletask-go/api" @@ -40,3 +42,46 @@ func TestFileStoreRoundTripAndLimit(t *testing.T) { _, err = store.Resolve(context.Background(), "file://sha256/not-a-hash") require.Error(t, err) } + +func TestFileStoreRejectsCorruptedContent(t *testing.T) { + for _, corrupted := range []string{"changed", "short", "", "payload with appended bytes"} { + t.Run(corrupted, func(t *testing.T) { + store, err := NewFileStore(t.TempDir(), 64) + require.NoError(t, err) + original := []byte("payload") + location, err := store.Store(context.Background(), original) + require.NoError(t, err) + hash, err := parseFileLocation(location) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(store.root, hash+".payload"), []byte(corrupted), 0o600)) + + resolved, err := store.Resolve(context.Background(), location) + require.ErrorIs(t, err, api.ErrLargePayloadIntegrity) + require.Nil(t, resolved) + reused, err := store.Store(context.Background(), original) + require.ErrorIs(t, err, api.ErrLargePayloadIntegrity) + require.Empty(t, reused) + }) + } +} + +func TestFileStoreVerifiedReuseAndCancellation(t *testing.T) { + store, err := NewFileStore(t.TempDir()) + require.NoError(t, err) + for _, content := range []string{"", "payload"} { + location, err := store.Store(context.Background(), []byte(content)) + require.NoError(t, err) + reused, err := store.Store(context.Background(), []byte(content)) + require.NoError(t, err) + require.Equal(t, location, reused) + resolved, err := store.Resolve(context.Background(), location) + require.NoError(t, err) + require.Equal(t, content, string(resolved)) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + resolved, err = store.Resolve(ctx, location) + require.ErrorIs(t, err, context.Canceled) + require.Nil(t, resolved) + } +} diff --git a/tests/durabletaskscheduler/dts_payload_routing_review_test.go b/tests/durabletaskscheduler/dts_payload_routing_review_test.go new file mode 100644 index 00000000..97990a53 --- /dev/null +++ b/tests/durabletaskscheduler/dts_payload_routing_review_test.go @@ -0,0 +1,132 @@ +package durabletaskscheduler_test + +import ( + "context" + "crypto/sha256" + "fmt" + "os" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/payload" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" +) + +func TestDTSContinueAsNewLargeCarryover(t *testing.T) { + for _, storeKind := range []string{"memory", "file", "blob"} { + t.Run(storeKind, func(t *testing.T) { + options := emulatorOptions(t) + switch storeKind { + case "memory": + store := payload.NewMemoryStore() + options.LargePayloads = &api.LargePayloadOptions{Store: store, Resolver: store} + case "file": + store, err := payload.NewFileStore(t.TempDir()) + require.NoError(t, err) + options.LargePayloads = &api.LargePayloadOptions{Store: store, Resolver: store} + case "blob": + connectionString := os.Getenv("AZURITE_CONNECTION_STRING") + if connectionString == "" { + t.Skip("set AZURITE_CONNECTION_STRING to run Blob carryover") + } + store, err := payload.NewAzureBlobStore(payload.AzureBlobStoreOptions{ + ConnectionString: connectionString, + Container: "carryover" + strings.ReplaceAll(uuid.NewString(), "-", ""), + AllowInsecureHTTP: true, + }) + require.NoError(t, err) + options.LargePayloads = &api.LargePayloadOptions{Store: store, Resolver: store} + } + + name := "DTSReviewCarryover-" + uuid.NewString() + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + var continued bool + if err := ctx.GetInput(&continued); err != nil { + return nil, err + } + if !continued { + if err := ctx.WaitForSingleEvent("continue", time.Minute).Await(nil); err != nil { + return nil, err + } + ctx.ContinueAsNew(true, task.WithKeepUnprocessedEvents()) + return nil, nil + } + var content string + if err := ctx.WaitForSingleEvent("large", time.Minute).Await(&content); err != nil { + return nil, err + } + return fmt.Sprintf("%x", sha256.Sum256([]byte(content))), nil + })) + client, _ := startEmulatorWithOptions(t, options, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + id := uniqueInstanceID("go-review-carryover") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithInput(false)) + require.NoError(t, err) + started, err := client.WaitForOrchestrationStart(ctx, id) + require.NoError(t, err) + require.NotEmpty(t, started.ExecutionID) + + content := strings.Repeat("x", 5*1024*1024) + require.NoError(t, client.RaiseEvent(ctx, id, "large", api.WithEventPayload(content))) + require.NoError(t, client.RaiseEvent(ctx, id, "continue")) + completed, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completed.RuntimeStatus, "%+v", completed.FailureDetails) + require.NotEqual(t, started.ExecutionID, completed.ExecutionID) + var digest string + require.NoError(t, completed.ReadOutput(&digest)) + require.Equal(t, fmt.Sprintf("%x", sha256.Sum256([]byte(content))), digest) + }) + } +} + +func TestDTSStrictMixedVersionRoutes(t *testing.T) { + options := emulatorOptions(t) + options.Versioning = &task.VersioningOptions{ + Version: "1.0", + DefaultVersion: "1.0", + MatchStrategy: task.VersionMatchStrict, + } + name := "DTSReviewMixed-" + uuid.NewString() + activityName := name + "-activity" + registry := task.NewTaskRegistry() + for _, version := range []string{"", "1.0"} { + require.NoError(t, registry.AddActivityNVersion(activityName, version, func(task.ActivityContext) (any, error) { + return "activity:" + version, nil + })) + require.NoError(t, registry.AddOrchestratorNVersion(name, version, func(ctx *task.OrchestrationContext) (any, error) { + var result string + if err := ctx.CallActivity(activityName).Await(&result); err != nil { + return nil, err + } + return "orchestration:" + version + "/" + result, nil + })) + } + client, _ := startEmulatorWithOptions(t, options, registry, + durabletaskclient.WithAutoWorkItemFilters(), + durabletaskclient.WithUnversionedOrchestratorNames(name), + durabletaskclient.WithUnversionedActivityNames(activityName)) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + for _, version := range []string{"", "1.0"} { + id := uniqueInstanceID("go-review-routing") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithVersion(version)) + require.NoError(t, err) + completed, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completed.RuntimeStatus, "%+v", completed.FailureDetails) + require.Equal(t, version, completed.Version) + var result string + require.NoError(t, completed.ReadOutput(&result)) + require.Equal(t, "orchestration:"+version+"/activity:"+version, result) + } +} From 36f34a478790d61d12b93083f608b236465f168d Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:20:23 -0700 Subject: [PATCH 17/26] fix: clear sample validation lint blockers Preserve unexpected error causes, distinguish missing expected failures, normalize shutdown diagnostics, and copy the sample build environment before appending overrides. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/history/history.go | 10 ++++++++-- samples/management/management.go | 15 ++++++++++++--- samples/worker/worker.go | 8 ++++---- tests/samples/samples_test.go | 3 ++- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/samples/history/history.go b/samples/history/history.go index b7b28319..c93345f5 100644 --- a/samples/history/history.go +++ b/samples/history/history.go @@ -206,12 +206,18 @@ func verifyStreamedHistory( func verifyHistoryCaps(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, executionID string) error { _, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID, MaxEvents: 1}) + if err == nil { + return errors.New("history event limit was not enforced") + } if !errors.Is(err, api.ErrHistoryLimitExceeded) { - return fmt.Errorf("MaxEvents cap error = %v, want %v", err, api.ErrHistoryLimitExceeded) + return fmt.Errorf("read event-limited history: %w", err) } _, err = client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{ExecutionID: executionID, MaxBytes: 64}) + if err == nil { + return errors.New("history byte limit was not enforced") + } if !errors.Is(err, api.ErrHistoryLimitExceeded) { - return fmt.Errorf("MaxBytes cap error = %v, want %v", err, api.ErrHistoryLimitExceeded) + return fmt.Errorf("read byte-limited history: %w", err) } fmt.Println("verified buffered history event and byte caps") return nil diff --git a/samples/management/management.go b/samples/management/management.go index fc0779eb..d9f5e330 100644 --- a/samples/management/management.go +++ b/samples/management/management.go @@ -210,7 +210,10 @@ func verifySuspendResume(ctx context.Context, client *durabletaskscheduler.Clien _, waitErr := client.WaitForOrchestrationCompletion(waitCtx, id) cancel() if !errors.Is(waitErr, context.DeadlineExceeded) { - return fmt.Errorf("suspended instance completed unexpectedly: %v", waitErr) + if waitErr != nil { + return fmt.Errorf("wait for suspended instance: %w", waitErr) + } + return errors.New("suspended instance completed unexpectedly") } suspended, err := client.FetchOrchestrationMetadata(ctx, id) if err != nil { @@ -345,7 +348,10 @@ func verifyStatusBasedIDReuse(ctx context.Context, client *durabletaskscheduler. DedupeStatuses: []api.OrchestrationStatus{api.RUNTIME_STATUS_COMPLETED}, }), ); !errors.Is(err, api.ErrDuplicateInstance) { - return fmt.Errorf("completed-status duplicate scheduling error=%v, want ErrDuplicateInstance", err) + if err != nil { + return fmt.Errorf("schedule duplicate completed instance: %w", err) + } + return errors.New("duplicate completed instance was not rejected") } if _, err := client.ScheduleNewOrchestration( ctx, @@ -417,7 +423,10 @@ func verifySingleAndBatchPurge(ctx context.Context, client *durabletaskscheduler } for _, id := range ids { if _, err := client.FetchOrchestrationMetadata(ctx, id); !errors.Is(err, api.ErrInstanceNotFound) { - return fmt.Errorf("purged instance %s remains readable or lookup failed: %v", id, err) + if err != nil { + return fmt.Errorf("read purged instance %s: %w", id, err) + } + return fmt.Errorf("purged instance %s remains readable", id) } } return nil diff --git a/samples/worker/worker.go b/samples/worker/worker.go index 7ad926e6..bce2beb7 100644 --- a/samples/worker/worker.go +++ b/samples/worker/worker.go @@ -139,9 +139,9 @@ func verifyStartConcurrencyAndDrain( select { case err := <-shutdownDone: if err != nil { - return fmt.Errorf("Shutdown completed with an error while activity was blocked: %w", err) + return fmt.Errorf("shutdown completed with an error while activity was blocked: %w", err) } - return errors.New("Shutdown completed before the accepted blocked activity was released") + return errors.New("shutdown completed before the accepted blocked activity was released") case <-time.After(200 * time.Millisecond): } barrier.Release() @@ -151,7 +151,7 @@ func verifyStartConcurrencyAndDrain( return err } case <-shutdownCtx.Done(): - return fmt.Errorf("Shutdown did not drain the released activity before deadline: %w", shutdownCtx.Err()) + return fmt.Errorf("shutdown did not drain the released activity before deadline: %w", shutdownCtx.Err()) } workerStopped = true if worker.Running() { @@ -238,7 +238,7 @@ func verifyRun( return err } case <-time.After(5 * time.Second): - return errors.New("Run worker did not stop after its context was canceled") + return errors.New("run worker did not stop after its context was canceled") } fmt.Println("verified Run blocks and exits cleanly when its context is canceled") return nil diff --git a/tests/samples/samples_test.go b/tests/samples/samples_test.go index 5b14b0be..84574336 100644 --- a/tests/samples/samples_test.go +++ b/tests/samples/samples_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "runtime" + "slices" "strings" "testing" "time" @@ -123,7 +124,7 @@ func TestSamplesE2E(t *testing.T) { buildCtx, stopBuild := context.WithTimeout(t.Context(), 3*time.Minute) build := exec.CommandContext(buildCtx, "go", buildArgs...) build.Dir = filepath.Join(root, "samples", entry.name) - build.Env = append(environment, "GOWORK=off") + build.Env = append(slices.Clone(environment), "GOWORK=off") output, err := build.CombinedOutput() stopBuild() if err != nil { From 298ff173aa0d0fce2029ee36b2c5fe0a625790bf Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 14:23:54 -0700 Subject: [PATCH 18/26] fix: retain export and maintenance sample error causes Clear remaining sample errorlint diagnostics, distinguishing unexpectedly readable resources from failed lookups and retaining both deadline and download failure causes. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/exporthistory/exporthistory.go | 7 +++++-- samples/serviceoperations/serviceoperations.go | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/samples/exporthistory/exporthistory.go b/samples/exporthistory/exporthistory.go index 2c899414..e52df632 100644 --- a/samples/exporthistory/exporthistory.go +++ b/samples/exporthistory/exporthistory.go @@ -314,7 +314,10 @@ func runScenario( return scenarioResult{}, fmt.Errorf("delete job %s: %w", jobID, err) } if _, err := exportClient.GetJob(ctx, jobID); !errors.Is(err, exporthistory.ErrJobNotFound) { - return scenarioResult{}, fmt.Errorf("expected deleted job %s to be missing, got %v", jobID, err) + if err != nil { + return scenarioResult{}, fmt.Errorf("read deleted job %s: %w", jobID, err) + } + return scenarioResult{}, fmt.Errorf("deleted job %s remains readable", jobID) } deleteJob = false return result, nil @@ -504,7 +507,7 @@ func waitForDownloadedHistories( lastErr = err select { case <-ctx.Done(): - return nil, fmt.Errorf("downloaded histories under %s never matched sources: %w; last error: %v", prefix, ctx.Err(), lastErr) + return nil, fmt.Errorf("downloaded histories under %s never matched sources: %w; last error: %w", prefix, ctx.Err(), lastErr) case <-time.After(time.Second): } } diff --git a/samples/serviceoperations/serviceoperations.go b/samples/serviceoperations/serviceoperations.go index 3c3617b6..b40bfc43 100644 --- a/samples/serviceoperations/serviceoperations.go +++ b/samples/serviceoperations/serviceoperations.go @@ -131,7 +131,10 @@ func verifyFilteredPurge(ctx context.Context, client *durabletaskscheduler.Clien } for _, id := range ids { if _, err := client.FetchOrchestrationMetadata(ctx, id); !errors.Is(err, api.ErrInstanceNotFound) { - return fmt.Errorf("purged instance %s remains readable or lookup failed: %v", id, err) + if err != nil { + return fmt.Errorf("read purged instance %s: %w", id, err) + } + return fmt.Errorf("purged instance %s remains readable", id) } } return nil From 332f513f8dc0dc3b001a0c30024bfc2677767119 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:14:25 -0700 Subject: [PATCH 19/26] fix: unwind canceled entity acquisitions across replay Determine commitment from matching request history, retain engine-owned cleanup until a late grant, and release abandoned locks while the parent remains running. Preserve Python ordering and cover multi-turn replay plus transient-lock observation on DTS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 5 + task/entity_lock_cancellation_test.go | 358 ++++++++++++++++++ task/entity_lock_test.go | 4 +- task/orchestrator.go | 47 ++- .../dts_lock_review_test.go | 173 +++++++++ 5 files changed, 581 insertions(+), 6 deletions(-) create mode 100644 task/entity_lock_cancellation_test.go create mode 100644 tests/durabletaskscheduler/dts_lock_review_test.go diff --git a/README.md b/README.md index 0ab22f66..bc293327 100644 --- a/README.md +++ b/README.md @@ -403,6 +403,11 @@ orchestrations to acquire overlapping multi-entity lock sets concurrently; different acquisition orders can deadlock. Keep those lock sets isolated until the SDKs share an ordering contract. +When upgrading from a pre-release build that retained canceled lock acquisitions, +complete or terminate affected orchestrations on the old workers first. Releasing +an abandoned acquisition as soon as its grant arrives changes the action sequence +relative to older histories that released it only at orchestration completion. + Entity names and operation names are matched case-insensitively using the same invariant rule as the .NET SDK, so a name resolves to the same entity in both SDKs. The DTS worker accepts only V2 entity work items (`EntityRequestV2`). diff --git a/task/entity_lock_cancellation_test.go b/task/entity_lock_cancellation_test.go new file mode 100644 index 00000000..bc6b00cd --- /dev/null +++ b/task/entity_lock_cancellation_test.go @@ -0,0 +1,358 @@ +package task + +import ( + "errors" + "testing" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" +) + +func TestLockEntitiesCancellationBeforeDispatchReplaysSubOrchestration(t *testing.T) { + registry := NewTaskRegistry() + var captured *OrchestrationContext + var nextGUID string + require.NoError(t, registry.AddOrchestratorN("cancel-lock-sub", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + ctx.Go(func(*OrchestrationContext) { cancel() }) + if release, err := child.LockEntities(api.NewEntityID("counter", "cancel")); !errors.Is(err, ErrTaskCanceled) || release != nil { + return nil, errors.New("expected canceled acquisition without a release callback") + } + nextGUID = ctx.NewGuid() + if err := ctx.CallSubOrchestrator("child").Await(nil); err != nil { + return nil, err + } + return "done", ctx.WaitForSingleEvent("finish", -1).Await(nil) + })) + history := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-lock-sub", "instance", nil, nil, nil, nil), + } + first := executeOrchestrationTurn(t, registry, "instance", nil, history) + require.Len(t, first.Actions, 1) + require.Equal(t, int32(1), first.Actions[0].Id, "cancellation must not recycle the lock action ID") + sub := first.Actions[0].GetCreateSubOrchestration() + require.NotNil(t, sub) + require.Empty(t, captured.pendingEntityTasks) + require.False(t, captured.IsInCriticalSection()) + require.Equal(t, uint64(2), captured.newGuidCounter) + expectedGUID := nextGUID + + history = append(history, helpers.NewSubOrchestrationCreatedEvent(1, sub.Name, sub.Version, sub.Input, sub.InstanceId, nil)) + replay := executeOrchestrationTurn(t, registry, "instance", history, nil) + require.Empty(t, replay.Actions, "the canceled lock must not block the recorded sub-orchestration") + require.Empty(t, captured.pendingEntityTasks) + require.False(t, captured.IsInCriticalSection()) + require.Equal(t, expectedGUID, nextGUID) + require.Equal(t, uint64(2), captured.newGuidCounter, "cancellation must not recycle the lock GUID") + + completed := &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{TaskScheduledId: 1}, + }, + } + second := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{completed}) + require.Empty(t, second.Actions) + history = append(history, completed) + third := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{helpers.NewEventRaisedEvent("finish", nil)}) + require.Len(t, third.Actions, 1) + require.Equal(t, int32(2), third.Actions[0].Id) + require.Equal(t, `"done"`, completionResult(t, third)) + require.Equal(t, expectedGUID, nextGUID) +} + +func TestLockEntitiesCanceledCommittedRequestReleasesOnLateGrant(t *testing.T) { + for _, activityWait := range []bool{false, true} { + name := "event" + if activityWait { + name = "activity" + } + t.Run(name, func(t *testing.T) { + registry := NewTaskRegistry() + entities := []api.EntityID{api.NewEntityID("counter", "a"), api.NewEntityID("counter", "b")} + var captured *OrchestrationContext + require.NoError(t, registry.AddOrchestratorN("cancel-committed", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + ctx.Go(func(ctx *OrchestrationContext) { + if err := ctx.WaitForSingleEvent("cancel", -1).Await(nil); err == nil { + cancel() + } + }) + if release, err := child.LockEntities(entities...); !errors.Is(err, ErrTaskCanceled) || release != nil { + return nil, errors.New("expected canceled acquisition without a release callback") + } + if !ctx.IsInCriticalSection() { + return nil, errors.New("abandoned acquisition must retain its critical section until granted") + } + if _, err := ctx.LockEntities(entities...); err == nil { + return nil, errors.New("a draining request must not allow another acquisition") + } + if err := ctx.CallSubOrchestrator("forbidden").Await(nil); err == nil { + return nil, errors.New("a draining request must not allow sub-orchestrations") + } + if err := ctx.CallEntity(entities[0], "get").Await(nil); err == nil { + return nil, errors.New("a draining request must not allow entity calls") + } + var err error + if activityWait { + err = ctx.CallActivity("finish").Await(nil) + } else { + err = ctx.WaitForSingleEvent("finish", -1).Await(nil) + } + if err != nil { + return nil, err + } + return !ctx.IsInCriticalSection(), nil + })) + history := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("cancel-committed", "instance", nil, nil, nil, nil), + } + first := executeOrchestrationTurn(t, registry, "instance", nil, history) + require.Len(t, first.Actions, 1) + request := first.Actions[0].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, request) + history = append(history, lockRequestHistory(first.Actions[0])) + cancelEvent := helpers.NewEventRaisedEvent("cancel", nil) + second := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{cancelEvent}) + require.True(t, captured.criticalSectionAbandoned) + require.True(t, captured.IsInCriticalSection()) + require.Empty(t, captured.pendingEntityTasks, "cleanup must not rely on the canceled task") + history = append(history, cancelEvent) + nextActionID := int32(1) + finishEvent := helpers.NewEventRaisedEvent("finish", nil) + if activityWait { + require.Len(t, second.Actions, 1) + require.NotNil(t, second.Actions[0].GetScheduleTask()) + history = append(history, helpers.NewTaskScheduledEvent(1, "finish", nil, nil, nil)) + nextActionID++ + finishEvent = helpers.NewTaskCompletedEvent(1, nil) + } else { + require.Empty(t, second.Actions, "cancellation must not release before the full grant") + } + + grant := lockGrantedHistory(request.CriticalSectionId) + for _, replayGrant := range []bool{false, true} { + var oldEvents, newEvents []*protos.HistoryEvent + if replayGrant { + oldEvents = append(append([]*protos.HistoryEvent(nil), history...), grant, grant) + } else { + oldEvents, newEvents = history, []*protos.HistoryEvent{grant, grant} + } + third := executeOrchestrationTurn(t, registry, "instance", oldEvents, newEvents) + require.Len(t, third.Actions, len(entities), "late grant must release once while the parent is still running") + require.False(t, captured.IsInCriticalSection()) + releasedHistory := append(append([]*protos.HistoryEvent(nil), history...), grant) + for index, action := range third.Actions { + require.Equal(t, nextActionID+int32(index), action.Id) + unlock := action.GetSendEntityMessage().GetEntityUnlockSent() + require.NotNil(t, unlock) + require.Equal(t, request.CriticalSectionId, unlock.CriticalSectionId) + require.Equal(t, entities[index].String(), unlock.TargetInstanceId.GetValue()) + releasedHistory = append(releasedHistory, lockUnlockHistory(action)) + } + replay := executeOrchestrationTurn(t, registry, "instance", releasedHistory, []*protos.HistoryEvent{grant}) + require.Empty(t, replay.Actions, "recorded unlocks and duplicate grants must replay without new actions") + last := executeOrchestrationTurn(t, registry, "instance", releasedHistory, []*protos.HistoryEvent{finishEvent}) + require.Len(t, last.Actions, 1) + require.Equal(t, nextActionID+int32(len(entities)), last.Actions[0].Id) + require.Equal(t, "true", completionResult(t, last)) + } + }) + } +} + +func TestLockEntitiesCancellationRecognizesUnprocessedRequestMarker(t *testing.T) { + registry := NewTaskRegistry() + var captured *OrchestrationContext + require.NoError(t, registry.AddOrchestratorN("marker-lookahead", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + child, cancel := ctx.WithCancel() + ctx.Go(func(ctx *OrchestrationContext) { + if err := ctx.WaitForSingleEvent("cancel", -1).Await(nil); err == nil { + cancel() + } + }) + if _, err := child.LockEntities(api.NewEntityID("counter", "marker")); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("expected cancellation") + } + return nil, ctx.WaitForSingleEvent("finish", -1).Await(nil) + })) + start := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("marker-lookahead", "instance", nil, nil, nil, nil), + } + first := executeOrchestrationTurn(t, registry, "instance", nil, start) + require.Len(t, first.Actions, 1) + marker := lockRequestHistory(first.Actions[0]) + cancelEvent := helpers.NewEventRaisedEvent("cancel", nil) + history := append(append([]*protos.HistoryEvent(nil), start...), cancelEvent, marker) + for _, split := range []int{0, len(start), len(history) - 1, len(history)} { + replay := executeOrchestrationTurn(t, registry, "instance", history[:split], history[split:]) + require.Empty(t, replay.Actions, "request marker after cancellation must still match, split=%d", split) + require.True(t, captured.criticalSectionRequestCommitted) + require.True(t, captured.criticalSectionAbandoned) + require.Empty(t, captured.pendingEntityTasks) + } +} + +func TestHistoricalEntityLockRequestMatchesAcquisition(t *testing.T) { + for _, test := range []struct { + name string + actionID int32 + sectionID string + matches bool + }{ + {"matching", 3, "current", true}, + {"different action", 4, "current", false}, + {"different section", 3, "previous", false}, + } { + t.Run(test.name, func(t *testing.T) { + marker := lockRequestHistory(helpers.NewEntityLockRequestedAction(test.actionID, test.sectionID, "instance", []string{"@counter@a"})) + for _, old := range []bool{false, true} { + ctx := newTestOrchestrationContext(NewTaskRegistry(), "instance", nil, nil) + if old { + ctx.oldEvents = []*protos.HistoryEvent{marker} + } else { + ctx.newEvents = []*protos.HistoryEvent{marker} + } + require.Equal(t, test.matches, ctx.hasHistoricalEntityLockRequest(3, "current")) + } + }) + } +} + +func TestLockEntitiesGrantBeforeAppliedCancellationWins(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("grant-before-cancel", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + ctx.Go(func(*OrchestrationContext) { + cancel() + if child.scope.isCanceled() { + panic("cancellation applied before the scheduler boundary") + } + // Complete the pending acquisition in the same scheduler step as + // the cancellation request, before the queued cancellation applies. + if err := ctx.onEntityLockGranted(&protos.EntityLockGrantedEvent{CriticalSectionId: ctx.criticalSectionID}); err != nil { + panic(err) + } + }) + release, err := child.LockEntities(api.NewEntityID("counter", "granted")) + if err != nil { + return nil, err + } + if !child.scope.isCanceled() || !ctx.IsInCriticalSection() || ctx.criticalSectionAbandoned { + return nil, errors.New("completed acquisition must win over applied cancellation") + } + release() + release() + return "granted", nil + })) + response := executeOrchestrationTurn(t, registry, "instance", nil, []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("grant-before-cancel", "instance", nil, nil, nil, nil), + }) + require.Len(t, response.Actions, 3) + require.NotNil(t, response.Actions[0].GetSendEntityMessage().GetEntityLockRequested()) + require.NotNil(t, response.Actions[1].GetSendEntityMessage().GetEntityUnlockSent()) + require.Equal(t, `"granted"`, completionResult(t, response)) +} + +func TestLockEntitiesStaleGrantAndReleaseDoNotAffectNewSection(t *testing.T) { + registry := NewTaskRegistry() + var captured *OrchestrationContext + require.NoError(t, registry.AddOrchestratorN("successive-locks", func(ctx *OrchestrationContext) (any, error) { + captured = ctx + entity := api.NewEntityID("counter", "successive") + oldRelease, err := ctx.LockEntities(entity) + if err != nil { + return nil, err + } + // Simulate engine-owned cleanup without consuming the user's callback. + ctx.releaseCriticalSection(ctx.criticalSectionID) + release, err := ctx.LockEntities(entity) + if err != nil { + return nil, err + } + defer release() + oldRelease() + oldRelease() + if !ctx.IsInCriticalSection() { + return nil, errors.New("stale callback released the new critical section") + } + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + release() + release() + return "done", nil + })) + history := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("successive-locks", "instance", nil, nil, nil, nil), + } + first := executeOrchestrationTurn(t, registry, "instance", nil, history) + require.Len(t, first.Actions, 1) + oldSectionID := first.Actions[0].GetSendEntityMessage().GetEntityLockRequested().CriticalSectionId + history = append(history, lockRequestHistory(first.Actions[0]), lockGrantedHistory(oldSectionID)) + second := executeOrchestrationTurn(t, registry, "instance", history, nil) + require.Len(t, second.Actions, 2) + require.NotNil(t, second.Actions[0].GetSendEntityMessage().GetEntityUnlockSent()) + nextRequest := second.Actions[1].GetSendEntityMessage().GetEntityLockRequested() + require.NotNil(t, nextRequest) + require.NotEqual(t, oldSectionID, nextRequest.CriticalSectionId) + history = append(history, lockUnlockHistory(second.Actions[0]), lockRequestHistory(second.Actions[1])) + + stale := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{lockGrantedHistory(oldSectionID)}) + require.Empty(t, stale.Actions) + require.Equal(t, nextRequest.CriticalSectionId, captured.criticalSectionID) + require.Nil(t, captured.criticalSectionAvailable, "stale grant must not complete the new acquisition") + require.Len(t, captured.pendingEntityTasks, 1) + + history = append(history, lockGrantedHistory(nextRequest.CriticalSectionId)) + replay := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{lockGrantedHistory(oldSectionID)}) + require.Empty(t, replay.Actions, "stale releases and replay teardown must not unlock the new section") + require.Equal(t, nextRequest.CriticalSectionId, captured.criticalSectionID) + require.NotNil(t, captured.criticalSectionAvailable) + last := executeOrchestrationTurn(t, registry, "instance", history, []*protos.HistoryEvent{helpers.NewEventRaisedEvent("finish", nil)}) + require.Len(t, last.Actions, 2) + require.Equal(t, nextRequest.CriticalSectionId, last.Actions[0].GetSendEntityMessage().GetEntityUnlockSent().CriticalSectionId) + require.Equal(t, `"done"`, completionResult(t, last)) +} + +func lockRequestHistory(action *protos.OrchestratorAction) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: action.Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityLockRequested{ + EntityLockRequested: action.GetSendEntityMessage().GetEntityLockRequested(), + }, + } +} + +func lockGrantedHistory(criticalSectionID string) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: -1, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityLockGranted{ + EntityLockGranted: &protos.EntityLockGrantedEvent{CriticalSectionId: criticalSectionID}, + }, + } +} + +func lockUnlockHistory(action *protos.OrchestratorAction) *protos.HistoryEvent { + return &protos.HistoryEvent{ + EventId: action.Id, + Timestamp: timestamppb.Now(), + EventType: &protos.HistoryEvent_EntityUnlockSent{ + EntityUnlockSent: action.GetSendEntityMessage().GetEntityUnlockSent(), + }, + } +} diff --git a/task/entity_lock_test.go b/task/entity_lock_test.go index 06edfc59..58f8792c 100644 --- a/task/entity_lock_test.go +++ b/task/entity_lock_test.go @@ -18,8 +18,10 @@ func TestLockEntitiesSortsAndDeduplicatesEveryPermutation(t *testing.T) { api.NewEntityID("account", "b"), api.NewEntityID("account", "a"), api.NewEntityID("cart", "z"), + api.NewEntityID("counter", "k"), + api.NewEntityID("counter2", "k"), } - expected := []string{"@account@a", "@account@b", "@cart@z"} + expected := []string{"@account@a", "@account@b", "@cart@z", "@counter2@k", "@counter@k"} random := rand.New(rand.NewSource(42)) for iteration := 0; iteration < 100; iteration++ { diff --git a/task/orchestrator.go b/task/orchestrator.go index f240488f..1b395d8b 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -89,6 +89,7 @@ type OrchestrationContext struct { criticalSectionLocks []string criticalSectionAvailable map[string]bool criticalSectionRequestCommitted bool + criticalSectionAbandoned bool } // callSubOrchestratorOptions is a struct that holds the options for the CallSubOrchestrator orchestrator method. @@ -1123,6 +1124,8 @@ func (ctx *OrchestrationContext) SignalEntity(entityID api.EntityID, operationNa } // LockEntities acquires an ordered critical section over a set of entities. +// If cancellation follows a committed request, critical-section restrictions +// remain in effect until the eventual grant is received and automatically released. func (ctx *OrchestrationContext) LockEntities(entityIDs ...api.EntityID) (func(), error) { engine := ctx.engineContext() if engine.isTerminated || ctx.scope.isCanceled() { @@ -1162,13 +1165,25 @@ func (ctx *OrchestrationContext) LockEntities(entityIDs ...api.EntityID) (func() engine.pendingActions[action.Id] = action engine.criticalSectionID = criticalSectionID engine.criticalSectionLocks = append([]string(nil), lockSet...) - engine.criticalSectionRequestCommitted = engine.IsReplaying + engine.criticalSectionRequestCommitted = false lockTask := newTaskInScope(engine, ctx.scope) engine.pendingEntityTasks[criticalSectionID] = lockTask if err := lockTask.Await(nil); err != nil { - if !engine.criticalSectionRequestCommitted { - delete(engine.pendingActions, action.Id) - engine.clearCriticalSection() + if engine.criticalSectionID == criticalSectionID { + delete(engine.pendingEntityTasks, criticalSectionID) + // Coroutines run before subsequent history markers. A request may + // already be committed even though its marker has not been replayed. + if !engine.criticalSectionRequestCommitted { + engine.criticalSectionRequestCommitted = engine.hasHistoricalEntityLockRequest(action.Id, criticalSectionID) + } + if engine.criticalSectionRequestCommitted { + // The lock set is acquired sequentially. Keep the critical + // section reserved until the full grant makes releasing safe. + engine.criticalSectionAbandoned = true + } else { + delete(engine.pendingActions, action.Id) + engine.clearCriticalSection() + } } return nil, err } @@ -1527,12 +1542,33 @@ func (ctx *OrchestrationContext) onEntityLockRequested(eventID int32, event *pro message.GetEntityLockRequested().CriticalSectionId != event.CriticalSectionId { return fmt.Errorf("entity lock request %q does not match pending action %d", event.CriticalSectionId, eventID) } - ctx.criticalSectionRequestCommitted = true + if ctx.criticalSectionID == event.CriticalSectionId { + ctx.criticalSectionRequestCommitted = true + } delete(ctx.pendingActions, eventID) return nil } +func (ctx *OrchestrationContext) hasHistoricalEntityLockRequest(actionID int32, criticalSectionID string) bool { + for _, history := range [][]*protos.HistoryEvent{ctx.oldEvents, ctx.newEvents} { + for _, event := range history { + if requested := event.GetEntityLockRequested(); requested != nil && + event.EventId == actionID && requested.CriticalSectionId == criticalSectionID { + return true + } + } + } + return false +} + func (ctx *OrchestrationContext) onEntityLockGranted(event *protos.EntityLockGrantedEvent) error { + if ctx.criticalSectionID != event.CriticalSectionId { + return nil + } + if ctx.criticalSectionAbandoned { + ctx.releaseCriticalSection(event.CriticalSectionId) + return nil + } if task := ctx.takePendingEntityTask(event.CriticalSectionId); task != nil { task.complete(nil) } @@ -1747,6 +1783,7 @@ func (ctx *OrchestrationContext) clearCriticalSection() { ctx.criticalSectionLocks = nil ctx.criticalSectionAvailable = nil ctx.criticalSectionRequestCommitted = false + ctx.criticalSectionAbandoned = false } func (ctx *OrchestrationContext) getNextSequenceNumber() int32 { diff --git a/tests/durabletaskscheduler/dts_lock_review_test.go b/tests/durabletaskscheduler/dts_lock_review_test.go new file mode 100644 index 00000000..72e92e7e --- /dev/null +++ b/tests/durabletaskscheduler/dts_lock_review_test.go @@ -0,0 +1,173 @@ +package durabletaskscheduler_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" +) + +func TestDTSEmulatorEntityLockCancellationBeforeDispatchReplays(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("DTSLockReviewChild", func(ctx *task.OrchestrationContext) (any, error) { + return "child", ctx.CreateTimer(100 * time.Millisecond).Await(nil) + })) + require.NoError(t, registry.AddOrchestratorN("DTSLockReviewBeforeDispatch", func(ctx *task.OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + ctx.Go(func(*task.OrchestrationContext) { cancel() }) + if release, err := child.LockEntities(api.NewEntityID("lockreview", string(ctx.ID))); !errors.Is(err, task.ErrTaskCanceled) || release != nil { + return nil, errors.New("expected canceled acquisition without a release callback") + } + var output string + if err := ctx.CallSubOrchestrator("DTSLockReviewChild").Await(&output); err != nil { + return nil, err + } + // Force another replay after the sub-orchestration completes. + if err := ctx.CreateTimer(100 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + return output, nil + })) + client, _, _ := startEmulatorClientAndWorker(t, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + id, err := client.ScheduleNewOrchestration(ctx, "DTSLockReviewBeforeDispatch", api.WithInstanceID(uniqueInstanceID("go-lock-cancel-staged"))) + require.NoError(t, err) + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + result, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, result.RuntimeStatus, "%+v", result.FailureDetails) + require.Equal(t, `"child"`, result.SerializedOutput) + history := fetchHistory(t, ctx, client, id, "") + require.Zero(t, countEvents(history, api.HistoryEventEntityLockRequested)) + require.Zero(t, countEvents(history, api.HistoryEventEntityUnlockSent)) + require.Equal(t, 1, countEvents(history, api.HistoryEventSubOrchestrationInstanceCreated)) + require.Equal(t, 1, countEvents(history, api.HistoryEventSubOrchestrationInstanceCompleted)) +} + +func TestDTSEmulatorEntityLockCanceledRequestReleasesAfterLateGrant(t *testing.T) { + registry := task.NewTaskRegistry() + key := uuid.NewString() + entities := []api.EntityID{api.NewEntityID("lockreview", key+"-a"), api.NewEntityID("lockreview", key+"-b")} + require.NoError(t, registry.AddEntityN("lockreview", func(*task.EntityContext) (any, error) { + return "available", nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSLockReviewHolder", func(ctx *task.OrchestrationContext) (any, error) { + release, err := ctx.LockEntities(entities[1]) + if err != nil { + return nil, err + } + defer release() + ctx.SetCustomStatus(`"holding"`) + return nil, ctx.WaitForSingleEvent("release", -1).Await(nil) + })) + require.NoError(t, registry.AddOrchestratorN("DTSLockReviewCanceled", func(ctx *task.OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + ctx.Go(func(ctx *task.OrchestrationContext) { + if err := ctx.WaitForSingleEvent("cancel", -1).Await(nil); err == nil { + cancel() + } + }) + if release, err := child.LockEntities(entities...); !errors.Is(err, task.ErrTaskCanceled) || release != nil { + return nil, errors.New("expected canceled acquisition without a release callback") + } + if !ctx.IsInCriticalSection() { + return nil, errors.New("canceled acquisition must retain its critical section until granted") + } + ctx.SetCustomStatus(`"canceled"`) + if err := ctx.WaitForSingleEvent("finish", -1).Await(nil); err != nil { + return nil, err + } + return !ctx.IsInCriticalSection(), nil + })) + require.NoError(t, registry.AddOrchestratorN("DTSLockReviewProbe", func(ctx *task.OrchestrationContext) (any, error) { + release, err := ctx.LockEntities(entities...) + if err != nil { + return nil, err + } + defer release() + var output string + if err := ctx.CallEntity(entities[0], "get").Await(&output); err != nil { + return nil, err + } + return output, nil + })) + client, _, _ := startEmulatorClientAndWorker(t, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + holderID, err := client.ScheduleNewOrchestration(ctx, "DTSLockReviewHolder", api.WithInstanceID(uniqueInstanceID("go-lock-holder"))) + require.NoError(t, err) + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{holderID}) }) + require.Eventually(t, func() bool { + metadata, err := client.FetchOrchestrationMetadata(ctx, holderID, api.WithFetchPayloads(true)) + return err == nil && metadata.SerializedCustomStatus == `"holding"` + }, 20*time.Second, 100*time.Millisecond) + + waiterID, err := client.ScheduleNewOrchestration(ctx, "DTSLockReviewCanceled", api.WithInstanceID(uniqueInstanceID("go-lock-canceled"))) + require.NoError(t, err) + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{waiterID}) }) + // Wait until acquisition has reached the first entity but is blocked by + // the holder at the second. Lock-only entities are transient and cannot be + // observed through GetEntity. + require.Eventually(t, func() bool { + page, err := client.QueryEntities(ctx, api.EntityQuery{ + InstanceIDStartsWith: entities[0].String(), + IncludeTransient: true, + }) + if err != nil || page == nil { + return false + } + for _, metadata := range page.Entities { + if metadata != nil && metadata.InstanceID == entities[0] && metadata.LockedBy == string(waiterID) { + return true + } + } + return false + }, 20*time.Second, 100*time.Millisecond) + require.NoError(t, client.RaiseEvent(ctx, waiterID, "cancel")) + require.Eventually(t, func() bool { + metadata, err := client.FetchOrchestrationMetadata(ctx, waiterID, api.WithFetchPayloads(true)) + return err == nil && metadata.SerializedCustomStatus == `"canceled"` + }, 20*time.Second, 100*time.Millisecond) + canceledHistory := fetchHistory(t, ctx, client, waiterID, "") + require.Equal(t, 1, countEvents(canceledHistory, api.HistoryEventEntityLockRequested)) + require.Zero(t, countEvents(canceledHistory, api.HistoryEventEntityLockGranted)) + require.Zero(t, countEvents(canceledHistory, api.HistoryEventEntityUnlockSent)) + + require.NoError(t, client.RaiseEvent(ctx, holderID, "release")) + holder, err := client.WaitForOrchestrationCompletion(ctx, holderID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, holder.RuntimeStatus) + require.Eventually(t, func() bool { + history, err := client.GetOrchestrationHistory(ctx, waiterID, api.HistoryQuery{}) + return err == nil && countEvents(history.Events, api.HistoryEventEntityUnlockSent) == len(entities) + }, 20*time.Second, 100*time.Millisecond) + + // A separate orchestration must be able to use the locks while the + // canceled acquisition's parent is still waiting for its finish event. + probeID, err := client.ScheduleNewOrchestration(ctx, "DTSLockReviewProbe", api.WithInstanceID(uniqueInstanceID("go-lock-probe"))) + require.NoError(t, err) + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{probeID}) }) + probe, err := client.WaitForOrchestrationCompletion(ctx, probeID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, probe.RuntimeStatus, "%+v", probe.FailureDetails) + require.Equal(t, `"available"`, probe.SerializedOutput) + waiter, err := client.FetchOrchestrationMetadata(ctx, waiterID) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_RUNNING, waiter.RuntimeStatus) + + require.NoError(t, client.RaiseEvent(ctx, waiterID, "finish")) + result, err := client.WaitForOrchestrationCompletion(ctx, waiterID, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, result.RuntimeStatus, "%+v", result.FailureDetails) + require.Equal(t, "true", result.SerializedOutput) + replayedHistory := fetchHistory(t, ctx, client, waiterID, "") + require.Equal(t, 1, countEvents(replayedHistory, api.HistoryEventEntityLockGranted)) + require.Equal(t, len(entities), countEvents(replayedHistory, api.HistoryEventEntityUnlockSent)) +} From e6eefbd06cf423d377b94eaad66cdc0f718bbb75 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:23:28 -0700 Subject: [PATCH 20/26] fix: stream history exports with bounded atomic uploads Preserve history limits and execution identity while encoding incrementally, commit bounded Azure blocks only after clean source completion, encode identifier metadata reversibly, and update export consumers. Document Python-compatible processing counters rather than promising distinct-execution totals. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- exporthistory/README.md | 48 +++- exporthistory/activities.go | 148 ++++++---- exporthistory/activities_test.go | 65 +++-- exporthistory/models.go | 45 +-- exporthistory/storage.go | 108 +++++-- exporthistory/storage_azurite_test.go | 83 +++++- exporthistory/storage_test.go | 148 ++++++++-- exporthistory/streaming_test.go | 269 ++++++++++++++++++ internal/historyconv/query.go | 56 +++- internal/historyconv/query_test.go | 28 ++ samples/exporthistory/exporthistory.go | 12 +- .../dts_export_history_test.go | 9 +- 12 files changed, 845 insertions(+), 174 deletions(-) create mode 100644 exporthistory/streaming_test.go diff --git a/exporthistory/README.md b/exporthistory/README.md index d76b5faa..447509b0 100644 --- a/exporthistory/README.md +++ b/exporthistory/README.md @@ -34,6 +34,16 @@ exports each instance's history in bounded parallel windows, and commits a checkpoint back to the entity. A batch job completes when the task hub reports no more pages; a continuous job idles for a minute and lists again. +### Progress counters + +`ScannedInstances` and `ExportedInstances` are cumulative processing totals, not +counts of distinct instances or executions. This retains the Python SDK's +counter semantics. A continuous job preserves the last opaque cursor when a +terminal page has no next token. If the source returns that page again, its +entries and successful exports are counted again, even when no new instance has +completed. Retries of blob writes remain idempotent, but that does not make these +counters unique. Do not use them as exact unique-export or billing totals. + ### Lifecycle | Operation | From | To | @@ -103,10 +113,16 @@ exactly the gzip stream the name promises. Declaring the compression as the download while others would not, leaving a reader unable to tell what it received. -JSONL objects carry one `api.HistoryEvent` per line. Every object carries -`instanceId` and `schemaVersion` metadata, plus `executionId` when the task hub -returns one. The name is derived deterministically, so re-exporting an instance -overwrites its object instead of duplicating it. +JSONL objects carry one `api.HistoryEvent` per line; JSON objects contain an +array of the same events. Azure objects carry `schemaVersion`, +`instanceIdBase64`, and `executionIdBase64` metadata. The identifier values are +always UTF-8 bytes encoded as **unpadded RFC 4648 base64url** +(`base64.RawURLEncoding` in Go), even for ASCII identifiers. Decode those +suffixed keys to recover the original IDs. Raw identifiers are never sent as +Azure metadata headers; the history body and object-name hash still use the +original IDs. Metadata keys may be returned with different casing by Azure. +The name is derived deterministically, so re-exporting an instance overwrites +its object instead of duplicating it. When no destination is supplied, a job writes to the client's configured container under the prefix `-/`. @@ -129,9 +145,18 @@ worker, err := durabletaskscheduler.NewWorker(options, registry, logger, ``` `Source` supplies the three management reads the export performs: -`ListInstanceIDs`, `FetchOrchestrationMetadata`, and `GetOrchestrationHistory`. +`ListInstanceIDs`, `FetchOrchestrationMetadata`, and `StreamOrchestrationHistory`. `*client.TaskHubGrpcClient` and the Durable Task Scheduler client satisfy it. +The exporter pins the read to metadata's execution ID and incrementally checks +`HistoryQuery.MaxEvents`, the aggregate approximate byte limit, and the observed +`ExecutionStarted` identity. It does not retain an event list or a whole +serialized/compressed history. Per export, memory consists of the current source +chunk/event, its encoded JSON, gzip state when needed, a single 1 MiB upload +block, and Azure's bounded block-ID list (at most 50,000 entries). Parallel +exports each own that budget; large individual events/source chunks still +require memory. + `Store` is a narrow interface with a single `Write` method. `AzureBlobHistoryStore` is the production implementation; supply your own to export elsewhere. It is deliberately separate from `payload.AzureBlobStore`, whose large-payload @@ -140,6 +165,19 @@ validation is at least as strict as that store's: an `AccountURL` carrying userinfo, a query string, or a fragment is rejected outright, and plaintext HTTP is confined to loopback endpoints behind `AllowInsecureHTTP`. +`ExportObject.Content` is now a single-use `io.Reader`, not `[]byte`. Custom +stores must consume it synchronously, honor cancellation, and publish only +after a clean EOF: a read error must leave an existing object unchanged. The +Azure implementation stages blocks sequentially and commits the block list +only after successful source validation and gzip finalization. On source or +upload failure, the opposite side is canceled and joined. SDK transport retries +rewind only the current block; an activity retry opens a fresh pinned history +stream, never reuses a consumed reader. Failed attempts can leave uncommitted +Azure blocks for service-managed expiration, but do not replace committed data. +Custom history sources must deliver events serially and stop on handler errors +or context cancellation. Update preview source/store implementations and blob +metadata readers when upgrading. + ### Versioning Every system task is registered unversioned so it stays reachable when an diff --git a/exporthistory/activities.go b/exporthistory/activities.go index d180a343..077cb0a6 100644 --- a/exporthistory/activities.go +++ b/exporthistory/activities.go @@ -1,16 +1,18 @@ package exporthistory import ( + "compress/gzip" "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" - "strings" + "io" "time" "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/historyconv" "github.com/microsoft/durabletask-go/task" ) @@ -26,12 +28,14 @@ type HistorySource interface { id api.InstanceID, opts ...api.FetchOrchestrationMetadataOptions, ) (*api.OrchestrationMetadata, error) - // GetOrchestrationHistory returns the instance's durable history. - GetOrchestrationHistory( + // StreamOrchestrationHistory delivers events serially, stopping when the + // handler returns an error or ctx is canceled. + StreamOrchestrationHistory( ctx context.Context, id api.InstanceID, query api.HistoryQuery, - ) (*api.OrchestrationHistory, error) + handler api.HistoryEventHandler, + ) error } // exportRuntime carries the worker-side dependencies shared by the export @@ -157,45 +161,58 @@ func (r *exportRuntime) exportInstance(ctx context.Context, input ExportRequest) query := r.historyPage query.ExecutionID = metadata.ExecutionID - history, err := r.source.GetOrchestrationHistory(ctx, instanceID, query) - if err != nil { - return ExportResult{}, fmt.Errorf("failed to read instance %s history: %w", input.InstanceID, err) - } - if history == nil { - return ExportResult{}, fmt.Errorf("instance %s returned no history", input.InstanceID) - } - if history.ExecutionID != metadata.ExecutionID { - return ExportResult{}, fmt.Errorf("instance %s history execution ID %q does not match metadata execution %q", - input.InstanceID, history.ExecutionID, metadata.ExecutionID) - } - - content, contentType, err := serializeHistory(history.Events, input.Format) - if err != nil { - return ExportResult{}, err - } - completedAt := completionTimestamp(metadata) name := blobObjectName(completedAt, input.InstanceID, input.Format) path := input.Destination.BlobPath(name) object := ExportObject{ Container: input.Destination.Container, Name: path, - Content: content, - ContentType: contentType, + ContentType: input.Format.ContentType(), Metadata: map[string]string{ "instanceId": input.InstanceID, - "executionId": history.ExecutionID, + "executionId": metadata.ExecutionID, "schemaVersion": input.Format.SchemaVersion, }, } - if err := r.store.Write(ctx, object); err != nil { - return ExportResult{}, err + streamCtx, cancel := context.WithCancel(ctx) + defer cancel() + reader, writer := io.Pipe() + defer reader.Close() //nolint:errcheck // PipeReader.Close always returns nil. + object.Content = reader + type streamResult struct { + count int + err error + } + done := make(chan streamResult, 1) + go func() { + count, streamErr := writeHistory(streamCtx, writer, input.Format, query, func(handler api.HistoryEventHandler) error { + return r.source.StreamOrchestrationHistory(streamCtx, instanceID, query, handler) + }) + if streamErr != nil { + // Readers interpret EOF as success. Keep a source's EOF opaque on + // the pipe; the original error is returned through done below. + _ = writer.CloseWithError(errors.New("history export producer failed: " + streamErr.Error())) + cancel() + } else { + _ = writer.Close() + } + done <- streamResult{count: count, err: streamErr} + }() + uploadErr := r.store.Write(streamCtx, object) + // An uploader may fail without reading anything. Wake a blocked pipe write + // AND a source blocked in its next receive, then join the producer. + cancel() + _ = reader.CloseWithError(io.ErrClosedPipe) + streamed := <-done + if err := errors.Join(streamed.err, uploadErr); err != nil { + return ExportResult{}, fmt.Errorf("failed to export instance %s history: %w", input.InstanceID, err) } return ExportResult{ - InstanceID: input.InstanceID, - Success: true, - BlobPath: path, - EventCount: len(history.Events), + InstanceID: input.InstanceID, + ExecutionID: metadata.ExecutionID, + Success: true, + BlobPath: path, + EventCount: streamed.count, }, nil } @@ -217,44 +234,69 @@ func blobObjectName(completedAt time.Time, instanceID string, format ExportForma return hex.EncodeToString(digest[:]) + "." + format.FileExtension() } -// serializeHistory renders events in the requested format and returns the object -// body with the content type to store it under. +// writeHistory validates and serializes one event at a time. Only successful +// end-of-stream validation permits the JSON closing bracket or gzip trailer. // // A JSONL object is gzip-compressed and stored as an opaque gzip file: its name // ends in .jsonl.gz and its content type is application/gzip, with no // Content-Encoding. Declaring the compression as a content coding instead would // make some clients transparently decompress the download while the object name // still promises gzip bytes, so readers could not tell what they received. -func serializeHistory( - events []*api.HistoryEvent, +func writeHistory( + ctx context.Context, + output io.Writer, format ExportFormat, -) (content []byte, contentType string, err error) { + query api.HistoryQuery, + stream func(api.HistoryEventHandler) error, +) (int, error) { + var compressed *gzip.Writer if format.Kind == ExportFormatJSON { - if events == nil { - events = []*api.HistoryEvent{} + if _, err := io.WriteString(output, "["); err != nil { + return 0, err } - payload, err := json.Marshal(events) + } else { + var err error + compressed, err = gzip.NewWriterLevel(output, gzip.BestCompression) if err != nil { - return nil, "", fmt.Errorf("failed to serialize orchestration history: %w", err) + return 0, err } - return payload, format.ContentType(), nil + output = compressed } - - var builder strings.Builder - for i, event := range events { - if event == nil { - continue + first := true + _, count, err := historyconv.StreamValidated(query, stream, func(event *api.HistoryEvent) error { + if err := ctx.Err(); err != nil { + return err } - line, err := json.Marshal(event) + encoded, err := json.Marshal(event) if err != nil { - return nil, "", fmt.Errorf("failed to serialize orchestration history event %d: %w", i, err) + return fmt.Errorf("failed to serialize orchestration history event: %w", err) } - builder.Write(line) - builder.WriteByte('\n') - } - compressed, err := gzipContent([]byte(builder.String())) + if format.Kind == ExportFormatJSON && !first { + if _, err := io.WriteString(output, ","); err != nil { + return err + } + } + first = false + if _, err := output.Write(encoded); err != nil { + return err + } + if format.Kind == ExportFormatJSONL { + _, err = io.WriteString(output, "\n") + } + return err + }) if err != nil { - return nil, "", err + return count, err + } + if err := ctx.Err(); err != nil { + return count, err + } + if compressed != nil { + if err := compressed.Close(); err != nil { + return count, fmt.Errorf("finish export compression: %w", err) + } + } else { + _, err = io.WriteString(output, "]") } - return compressed, format.ContentType(), nil + return count, err } diff --git a/exporthistory/activities_test.go b/exporthistory/activities_test.go index 7d2730fa..bffc07bb 100644 --- a/exporthistory/activities_test.go +++ b/exporthistory/activities_test.go @@ -1,6 +1,7 @@ package exporthistory import ( + "bytes" "context" "encoding/json" "errors" @@ -105,6 +106,22 @@ func (s *fakeSource) GetOrchestrationHistory( return history, nil } +func (s *fakeSource) StreamOrchestrationHistory(ctx context.Context, id api.InstanceID, query api.HistoryQuery, handler api.HistoryEventHandler) error { + history, err := s.GetOrchestrationHistory(ctx, id, query) + if err != nil { + return err + } + if history == nil { + return errors.New("no history") + } + for _, event := range history.Events { + if err := handler(event); err != nil { + return err + } + } + return nil +} + func (s *fakeSource) addInstance(instanceID string, status api.OrchestrationStatus, events int) { s.mu.Lock() defer s.mu.Unlock() @@ -128,6 +145,12 @@ func (s *fakeSource) addInstance(instanceID string, status api.OrchestrationStat Timestamp: completedAt, }) } + if events > 0 { + history.Events[0].Type = api.HistoryEventExecutionStarted + history.Events[0].ExecutionStarted = &api.HistoryExecutionStartedEvent{ + InstanceID: api.InstanceID(instanceID), ExecutionID: history.ExecutionID, + } + } s.history[instanceID] = history } @@ -294,10 +317,10 @@ func TestExportInstanceHistoryActivity(t *testing.T) { decompressed := decompressGzip(t, object.Content) lines := strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") require.Len(t, lines, 3) - for _, line := range lines { + for i, line := range lines { var event api.HistoryEvent require.NoError(t, json.Unmarshal([]byte(line), &event)) - assert.Equal(t, api.HistoryEventOrchestratorStarted, event.Type) + assert.Equal(t, source.history["instance-1"].Events[i].Type, event.Type) } }) @@ -398,7 +421,7 @@ func TestExportInstanceHistoryActivity(t *testing.T) { instanceID string message string }{ - {"history-error", "failed to read instance history-error history"}, + {"history-error", "failed to export instance history-error history"}, {"metadata-error", "failed to read instance metadata-error metadata"}, {"store-error", "upload rejected"}, } @@ -476,19 +499,29 @@ func TestSerializeHistory(t *testing.T) { assert.JSONEq(t, `[]`, string(content)) }) - t.Run("nil events are skipped rather than serialized as null", func(t *testing.T) { + t.Run("nil events are rejected", func(t *testing.T) { events := []*api.HistoryEvent{ - {Type: api.HistoryEventExecutionStarted}, + {Type: api.HistoryEventOrchestratorStarted}, nil, {Type: api.HistoryEventExecutionCompleted}, } - content, _, err := serializeHistory(events, DefaultExportFormat()) - require.NoError(t, err) - decompressed := decompressGzip(t, content) - lines := strings.Split(strings.TrimRight(string(decompressed), "\n"), "\n") - require.Len(t, lines, 2) - assert.NotContains(t, string(decompressed), "null") + _, _, err := serializeHistory(events, DefaultExportFormat()) + require.ErrorContains(t, err, "must not be nil") + }) +} + +// Only tests buffer serialized output; the production path streams it. +func serializeHistory(events []*api.HistoryEvent, format ExportFormat) ([]byte, string, error) { + var output bytes.Buffer + _, err := writeHistory(context.Background(), &output, format, api.HistoryQuery{}, func(handler api.HistoryEventHandler) error { + for _, event := range events { + if err := handler(event); err != nil { + return err + } + } + return nil }) + return output.Bytes(), format.ContentType(), err } func TestBlobObjectName(t *testing.T) { @@ -629,11 +662,9 @@ func TestExportInstanceHistoryRejectsExecutionIdentity(t *testing.T) { source.addInstance("instance-1", api.RUNTIME_STATUS_COMPLETED, 1) source.metadata["instance-1"].ExecutionID = test.metadataID source.history["instance-1"].ExecutionID = test.historyID - // Identity validation must run before attempting to serialize content. - details := &api.FailureDetails{} - details.InnerFailure = details source.history["instance-1"].Events = []*api.HistoryEvent{{ - TaskFailed: &api.HistoryTaskFailureEvent{FailureDetails: details}, + Type: api.HistoryEventExecutionStarted, + ExecutionStarted: &api.HistoryExecutionStartedEvent{ExecutionID: test.historyID}, }} store := newMemoryStore() runtime := newTestRuntime(source, store) @@ -643,7 +674,7 @@ func TestExportInstanceHistoryRejectsExecutionIdentity(t *testing.T) { Format: DefaultExportFormat(), })) assert.ErrorContains(t, err, "execution") - assert.Zero(t, store.writeCount()) + assert.Zero(t, store.count()) assert.Empty(t, store.snapshot()) if test.readHistory { assert.Equal(t, test.metadataID, source.historyQuery.ExecutionID) @@ -758,7 +789,7 @@ func TestExportInstanceHistoryPinsTheClientCollector(t *testing.T) { if test.wantError { assert.ErrorContains(t, err, "execution") assert.Nil(t, result) - assert.Zero(t, store.writeCount()) + assert.Zero(t, store.count()) } else { require.NoError(t, err) require.True(t, result.(ExportResult).Success) diff --git a/exporthistory/models.go b/exporthistory/models.go index 68fc629b..b419ca50 100644 --- a/exporthistory/models.go +++ b/exporthistory/models.go @@ -314,16 +314,18 @@ type ExportJobConfiguration struct { // ExportJobState is the persisted entity state of an export job. type ExportJobState struct { - Status ExportJobStatus `json:"Status"` - Config *ExportJobConfiguration `json:"Config,omitempty"` - Checkpoint *ExportCheckpoint `json:"Checkpoint,omitempty"` - CreatedAt *time.Time `json:"CreatedAt,omitempty"` - LastModifiedAt *time.Time `json:"LastModifiedAt,omitempty"` - LastCheckpointTime *time.Time `json:"LastCheckpointTime,omitempty"` - LastError string `json:"LastError,omitempty"` - ScannedInstances int64 `json:"ScannedInstances"` - ExportedInstances int64 `json:"ExportedInstances"` - OrchestratorInstanceID string `json:"OrchestratorInstanceId,omitempty"` + Status ExportJobStatus `json:"Status"` + Config *ExportJobConfiguration `json:"Config,omitempty"` + Checkpoint *ExportCheckpoint `json:"Checkpoint,omitempty"` + CreatedAt *time.Time `json:"CreatedAt,omitempty"` + LastModifiedAt *time.Time `json:"LastModifiedAt,omitempty"` + LastCheckpointTime *time.Time `json:"LastCheckpointTime,omitempty"` + LastError string `json:"LastError,omitempty"` + // ScannedInstances is cumulative scan progress, including repeated pages. + ScannedInstances int64 `json:"ScannedInstances"` + // ExportedInstances counts successful exports, including re-exports; it is not a distinct-execution count. + ExportedInstances int64 `json:"ExportedInstances"` + OrchestratorInstanceID string `json:"OrchestratorInstanceId,omitempty"` // RunToken identifies the job's current run generation. Every Create mints // a new one, so a run that started before the job was deleted and recreated // carries a stale token and is fenced out of the new generation's state. @@ -341,11 +343,13 @@ type ExportJobDescription struct { // OrchestratorInstanceID identifies the current generation, reserved during // Create before its Run signal is delivered. Recreation assigns a new ID. OrchestratorInstanceID string - ScannedInstances int64 - ExportedInstances int64 - LastError string - Checkpoint *ExportCheckpoint - LastCheckpointTime time.Time + // ScannedInstances includes entries counted again when a page is re-scanned. + ScannedInstances int64 + // ExportedInstances includes re-exports, matching ExportJobState's processing-total semantics. + ExportedInstances int64 + LastError string + Checkpoint *ExportCheckpoint + LastCheckpointTime time.Time } // ExportJobQuery filters a single page of export jobs. Status and creation-time @@ -457,11 +461,12 @@ type ExportRequest struct { // failures are collected rather than thrown so a batch can report every failing // instance at once. type ExportResult struct { - InstanceID string `json:"InstanceId"` - Success bool `json:"Success"` - Error string `json:"Error,omitempty"` - BlobPath string `json:"BlobPath,omitempty"` - EventCount int `json:"EventCount,omitempty"` + InstanceID string `json:"InstanceId"` + ExecutionID string `json:"ExecutionId,omitempty"` + Success bool `json:"Success"` + Error string `json:"Error,omitempty"` + BlobPath string `json:"BlobPath,omitempty"` + EventCount int `json:"EventCount,omitempty"` } // TerminalStatuses returns the orchestration runtime statuses an export job diff --git a/exporthistory/storage.go b/exporthistory/storage.go index f24e0bb9..07cef975 100644 --- a/exporthistory/storage.go +++ b/exporthistory/storage.go @@ -2,9 +2,11 @@ package exporthistory import ( "bytes" - "compress/gzip" "context" + "encoding/base64" + "errors" "fmt" + "io" "net" "net/url" "strings" @@ -13,10 +15,12 @@ import ( "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" ) // ExportObject is one serialized orchestration history ready to be persisted. @@ -26,7 +30,7 @@ type ExportObject struct { // Name is the full object path inside the container, prefix included. Name string // Content is the object body, already compressed when the format requires it. - Content []byte + Content io.Reader // ContentType is the MIME type of Content. A compressed format reports the // compressed type, such as application/gzip, rather than declaring the // compression as a separate content coding, so a reader always receives the @@ -45,6 +49,10 @@ type ExportObject struct { // Implementations must be safe for concurrent use and should treat a repeated // write of the same object name as an overwrite, since an activity retry can // re-export an instance whose object already exists. +// Write must consume Content synchronously, respect ctx, and publish only after +// a clean EOF. A read error must abort the write, leaving an existing object +// unchanged. Content is single-use; retrying an export opens a fresh history +// stream rather than replaying this reader. type Store interface { Write(ctx context.Context, object ExportObject) error } @@ -84,7 +92,7 @@ type AzureBlobHistoryStore struct { // Narrow hooks stand in for the two *azblob.Client calls whose ordering and // retry behavior matter. Both are nil in production. createContainerHook func(ctx context.Context, container string) error - uploadBlobHook func(ctx context.Context, container, name string, body []byte, options *azblob.UploadBufferOptions) error + uploadBlobHook func(ctx context.Context, container, name string, body io.Reader, options *blockblob.CommitBlockListOptions) error waitHook func(ctx context.Context, d time.Duration) error } @@ -105,6 +113,9 @@ const ( containerBeingDeletedAttempts = 9 containerBeingDeletedInitialBackoff = 250 * time.Millisecond containerBeingDeletedMaxBackoff = 8 * time.Second + // Bound resident upload buffers independently of the history size and of + // future Azure SDK defaults. Each concurrent export owns this budget. + historyUploadBlockSize = 1024 * 1024 ) // NewAzureBlobHistoryStore constructs a production Azure Blob Storage export store. @@ -195,13 +206,20 @@ func (s *AzureBlobHistoryStore) Write(ctx context.Context, object ExportObject) } metadata := make(map[string]*string, len(object.Metadata)) for key, value := range object.Metadata { + // Azure metadata travels in HTTP headers, which cannot contain arbitrary + // Unicode identifiers. The suffixed keys explicitly identify unpadded + // RFC 4648 base64url, including for identifiers that were already ASCII. + if strings.EqualFold(key, "instanceId") || strings.EqualFold(key, "executionId") { + key += "Base64" + value = base64.RawURLEncoding.EncodeToString([]byte(value)) + } metadata[key] = to.Ptr(value) } headers := &blob.HTTPHeaders{} if object.ContentType != "" { headers.BlobContentType = to.Ptr(object.ContentType) } - options := &azblob.UploadBufferOptions{HTTPHeaders: headers, Metadata: metadata} + options := &blockblob.CommitBlockListOptions{HTTPHeaders: headers, Metadata: metadata} if err := s.uploadBlob(ctx, object.Container, object.Name, object.Content, options); err != nil { // A container deleted between initialization and upload must not wedge // the store: forget the exact initialization this write relied on so the @@ -305,16 +323,75 @@ func (s *AzureBlobHistoryStore) uploadBlob( ctx context.Context, container string, name string, - body []byte, - options *azblob.UploadBufferOptions, + body io.Reader, + options *blockblob.CommitBlockListOptions, ) error { if s.uploadBlobHook != nil { return s.uploadBlobHook(ctx, container, name, body, options) } - _, err := s.client.UploadBuffer(ctx, container, name, body, options) + if body == nil { + body = strings.NewReader("") + } + client := s.client.ServiceClient().NewContainerClient(container).NewBlockBlobClient(name) + buffer := make([]byte, historyUploadBlockSize) + blockIDs := make([]string, 0) + prefix := newCompactUUID() + for { + if err := ctx.Err(); err != nil { + return err + } + n, err := readHistoryBlock(ctx, body, buffer) + if err != nil && err != io.EOF { //nolint:errorlint // Only exact EOF authorizes committing; wrapped EOF is a source failure. + return err + } + if n > 0 { + if len(blockIDs) == 50000 { + return errors.New("export exceeds Azure's 50000-block limit") + } + blockID := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%s-%08d", prefix, len(blockIDs)))) + // Stage synchronously so a failed upload returns immediately even + // if the producer is blocked receiving the next history chunk. + // SDK retries rewind this one block, never the consumed history. + _, stageErr := client.StageBlock(ctx, blockID, streaming.NopCloser(bytes.NewReader(buffer[:n])), nil) + if stageErr != nil { + return stageErr + } + blockIDs = append(blockIDs, blockID) + } + if err != nil { + break + } + } + _, err := client.CommitBlockList(ctx, blockIDs, options) return err } +// Unlike io.ReadFull, preserve a reader's error even when it accompanies a full +// block. Only an actual EOF authorizes committing; ErrUnexpectedEOF from a +// truncated source must never be confused with a clean partial final block. +func readHistoryBlock(ctx context.Context, reader io.Reader, block []byte) (int, error) { + size, emptyReads := 0, 0 + for size < len(block) { + if err := ctx.Err(); err != nil { + return size, err + } + n, err := reader.Read(block[size:]) + size += n + if err != nil { + return size, err + } + if n > 0 { + emptyReads = 0 + } else { + emptyReads++ + if emptyReads == 100 { + return size, io.ErrNoProgress + } + } + } + return size, nil +} + func (s *AzureBlobHistoryStore) waitForRetry(ctx context.Context, d time.Duration) error { if s.waitHook != nil { return s.waitHook(ctx, d) @@ -329,23 +406,6 @@ func (s *AzureBlobHistoryStore) waitForRetry(ctx context.Context, d time.Duratio } } -// gzipContent compresses content with the deterministic settings the JSONL -// export format expects. -func gzipContent(content []byte) ([]byte, error) { - var buffer bytes.Buffer - writer, err := gzip.NewWriterLevel(&buffer, gzip.BestCompression) - if err != nil { - return nil, fmt.Errorf("create gzip writer: %w", err) - } - if _, err := writer.Write(content); err != nil { - return nil, fmt.Errorf("compress export content: %w", err) - } - if err := writer.Close(); err != nil { - return nil, fmt.Errorf("finish export compression: %w", err) - } - return buffer.Bytes(), nil -} - // validBlobContainerName mirrors the Azure Blob container naming rules. func validBlobContainerName(name string) bool { if len(name) < 3 || len(name) > 63 || name[0] == '-' || name[len(name)-1] == '-' { diff --git a/exporthistory/storage_azurite_test.go b/exporthistory/storage_azurite_test.go index 1dfb1e06..0e747fcf 100644 --- a/exporthistory/storage_azurite_test.go +++ b/exporthistory/storage_azurite_test.go @@ -1,9 +1,12 @@ package exporthistory import ( + "bytes" "context" "crypto/rand" + "encoding/base64" "encoding/hex" + "errors" "io" "os" "strings" @@ -52,7 +55,7 @@ func TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects(t *testing.T) { ctx := context.Background() events := []*api.HistoryEvent{ - {Type: api.HistoryEventExecutionStarted}, + {Type: api.HistoryEventExecutionStarted, ExecutionStarted: &api.HistoryExecutionStartedEvent{ExecutionID: "execution-1"}}, {Type: api.HistoryEventExecutionCompleted}, } content, contentType, err := serializeHistory(events, DefaultExportFormat()) @@ -60,7 +63,7 @@ func TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects(t *testing.T) { object := ExportObject{ Container: container, Name: "batch-job/" + strings.Repeat("a", 64) + ".jsonl.gz", - Content: content, + Content: bytes.NewReader(content), ContentType: contentType, Metadata: map[string]string{"instanceId": "instance-1", "schemaVersion": DefaultSchemaVersion}, } @@ -71,7 +74,7 @@ func TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects(t *testing.T) { // and no client transparently decompresses it. assert.Equal(t, "application/gzip", derefString(properties.ContentType)) assert.Empty(t, derefString(properties.ContentEncoding)) - assert.Equal(t, "instance-1", metadataValue(properties.Metadata, "instanceId")) + assert.Equal(t, base64.RawURLEncoding.EncodeToString([]byte("instance-1")), metadataValue(properties.Metadata, "instanceIdBase64")) assert.Equal(t, DefaultSchemaVersion, metadataValue(properties.Metadata, "schemaVersion")) // Downloading always yields exactly the gzip bytes that were uploaded. @@ -85,12 +88,80 @@ func TestAzureBlobHistoryStoreAzuriteWritesCompressedObjects(t *testing.T) { assert.Contains(t, lines[1], string(api.HistoryEventExecutionCompleted)) // Re-exporting the same instance overwrites its object rather than failing. - object.Content = []byte("second write") + object.Content = strings.NewReader("second write") object.ContentType = "text/plain" require.NoError(t, store.Write(ctx, object)) assert.Equal(t, []byte("second write"), downloadAzuriteBlob(t, store, container, object.Name)) } +func TestAzureBlobHistoryStoreAzuriteUnicodeIdentifiers(t *testing.T) { + store, container := newAzuriteExportStore(t) + source := newFakeSource() + instanceID, executionID := "日本語-café-😀", "実行-é-🚀" + source.addInstance(instanceID, api.RUNTIME_STATUS_COMPLETED, 1) + source.metadata[instanceID].ExecutionID = executionID + source.history[instanceID].Events[0].ExecutionStarted.ExecutionID = executionID + request := ExportRequest{ + InstanceID: instanceID, Destination: ExportDestination{Container: container}, Format: DefaultExportFormat(), + } + result, err := newTestRuntime(source, store).exportInstance(context.Background(), request) + require.NoError(t, err) + require.True(t, result.Success) + properties := azuriteBlobProperties(t, store, container, result.BlobPath) + for key, expected := range map[string]string{"instanceId": instanceID, "executionId": executionID} { + require.Empty(t, metadataValue(properties.Metadata, key), "raw identifiers must never be sent as headers") + decoded, err := base64.RawURLEncoding.DecodeString(metadataValue(properties.Metadata, key+"Base64")) + require.NoError(t, err) + require.Equal(t, expected, string(decoded)) + } + content := downloadAzuriteBlob(t, store, container, result.BlobPath) + require.Contains(t, string(decompressGzip(t, content)), instanceID) + require.Contains(t, string(decompressGzip(t, content)), executionID) + require.Equal(t, blobObjectName(completionTimestamp(source.metadata[instanceID]), instanceID, request.Format), result.BlobPath) +} + +func TestAzureBlobHistoryStoreAzuriteSourceFailurePreservesBlob(t *testing.T) { + store, container := newAzuriteExportStore(t) + base := streamTestSource() + request := streamTestRequest() + request.Destination.Container = container + request.Format.Kind = ExportFormatJSON + result, err := newTestRuntime(base, store).exportInstance(context.Background(), request) + require.NoError(t, err) + original := downloadAzuriteBlob(t, store, container, result.BlobPath) + err = store.Write(context.Background(), ExportObject{ + Container: container, + Name: result.BlobPath, + Content: &terminalErrorReader{ + content: strings.Repeat("x", historyUploadBlockSize), + err: io.ErrUnexpectedEOF, + }, + }) + require.ErrorIs(t, err, io.ErrUnexpectedEOF) + require.Equal(t, original, downloadAzuriteBlob(t, store, container, result.BlobPath), + "reader errors returned alongside a full block must prevent commit") + + for _, failure := range []error{errors.New("source failed after staged block"), io.EOF, io.ErrUnexpectedEOF} { + source := streamingSource{fakeSource: base, stream: func(_ context.Context, handler api.HistoryEventHandler) error { + if err := handler(base.history["subject"].Events[0]); err != nil { + return err + } + if err := handler(&api.HistoryEvent{Generic: &api.HistoryPayloadEvent{ + SerializedInput: strings.Repeat("x", 2*historyUploadBlockSize), + }}); err != nil { + return err + } + return failure + }} + result, err := newTestRuntime(source, store).exportInstance(context.Background(), request) + require.ErrorIs(t, err, failure) + require.False(t, result.Success) + path := blobObjectName(completionTimestamp(base.metadata["subject"]), "subject", request.Format) + require.Equal(t, original, downloadAzuriteBlob(t, store, container, path), + "a source error after staging blocks must not replace the committed blob") + } +} + // TestAzureBlobHistoryStoreAzuriteRejectsDisallowedContainer keeps a job from writing // outside the containers the worker was configured for, against real storage. func TestAzureBlobHistoryStoreAzuriteRejectsDisallowedContainer(t *testing.T) { @@ -98,7 +169,7 @@ func TestAzureBlobHistoryStoreAzuriteRejectsDisallowedContainer(t *testing.T) { err := store.Write(context.Background(), ExportObject{ Container: randomContainerName(t), Name: "object.jsonl.gz", - Content: []byte("payload"), + Content: strings.NewReader("payload"), }) require.ErrorIs(t, err, ErrValidation) assert.Contains(t, err.Error(), "is not allowed by this worker") @@ -130,7 +201,7 @@ func TestAzureBlobHistoryStoreAzuriteWritesAcrossAllowedContainers(t *testing.T) require.NoError(t, store.Write(ctx, ExportObject{ Container: container, Name: "prefix/object.json", - Content: []byte(`[]`), + Content: strings.NewReader(`[]`), Metadata: map[string]string{"instanceId": "instance-1"}, }), container) assert.Equal(t, []byte(`[]`), downloadAzuriteBlob(t, store, container, "prefix/object.json")) diff --git a/exporthistory/storage_test.go b/exporthistory/storage_test.go index 153a22be..86aa3a7e 100644 --- a/exporthistory/storage_test.go +++ b/exporthistory/storage_test.go @@ -4,19 +4,24 @@ import ( "bytes" "compress/gzip" "context" + "encoding/base64" "errors" "io" "maps" + "net/http" + "net/http/httptest" "net/url" "strings" "sync" + "sync/atomic" "testing" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + "github.com/microsoft/durabletask-go/api" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -41,7 +46,7 @@ const azuriteConnectionString = "DefaultEndpointsProtocol=http;AccountName=devst // pipeline without Azure Storage. type memoryStore struct { mu sync.Mutex - objects map[string]ExportObject + objects map[string]storedExportObject writes int failure error // failFor fails the write for a specific instance ID, so a test can drive a @@ -49,8 +54,13 @@ type memoryStore struct { failFor map[string]error } +type storedExportObject struct { + ExportObject + Content []byte +} + func newMemoryStore() *memoryStore { - return &memoryStore{objects: make(map[string]ExportObject), failFor: make(map[string]error)} + return &memoryStore{objects: make(map[string]storedExportObject), failFor: make(map[string]error)} } func (s *memoryStore) Write(_ context.Context, object ExportObject) error { @@ -63,7 +73,12 @@ func (s *memoryStore) Write(_ context.Context, object ExportObject) error { if err, ok := s.failFor[object.Metadata["instanceId"]]; ok { return err } - s.objects[object.Container+"/"+object.Name] = object + content, err := io.ReadAll(object.Content) + if err != nil { + return err + } + object.Content = nil + s.objects[object.Container+"/"+object.Name] = storedExportObject{ExportObject: object, Content: content} return nil } @@ -79,7 +94,7 @@ func (s *memoryStore) writeCount() int { return s.writes } -func (s *memoryStore) snapshot() map[string]ExportObject { +func (s *memoryStore) snapshot() map[string]storedExportObject { s.mu.Lock() defer s.mu.Unlock() return maps.Clone(s.objects) @@ -291,7 +306,7 @@ func TestAzureBlobHistoryStoreRejectsDisallowedContainers(t *testing.T) { }) require.NoError(t, err) store.createContainerHook = func(context.Context, string) error { return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } @@ -319,7 +334,7 @@ func TestAzureBlobHistoryStoreAllowAnyContainer(t *testing.T) { }) require.NoError(t, err) store.createContainerHook = func(context.Context, string) error { return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } require.NoError(t, store.Write(context.Background(), ExportObject{Container: "anything", Name: "a"})) @@ -343,7 +358,7 @@ func TestAzureBlobHistoryStoreCreatesEachContainerOnce(t *testing.T) { creates[container]++ return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } @@ -389,7 +404,7 @@ func TestAzureBlobHistoryStoreRetriesContainerBeingDeleted(t *testing.T) { waits = append(waits, d) return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } @@ -411,7 +426,7 @@ func TestAzureBlobHistoryStoreTreatsExistingContainerAsSuccess(t *testing.T) { return &azcore.ResponseError{ErrorCode: string(bloberror.ContainerAlreadyExists)} } uploaded := 0 - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { uploaded++ return nil } @@ -436,7 +451,7 @@ func TestAzureBlobHistoryStoreRetriesAfterContainerCreationFailure(t *testing.T) } return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } @@ -460,7 +475,7 @@ func TestAzureBlobHistoryStoreForgetsDeletedContainer(t *testing.T) { return nil } uploads := 0 - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { uploads++ if uploads == 2 { return &azcore.ResponseError{ErrorCode: string(bloberror.ContainerNotFound)} @@ -492,7 +507,7 @@ func TestAzureBlobHistoryStoreReportsCreationFailureToWaiters(t *testing.T) { return creationFailure } uploads := 0 - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { uploads++ return nil } @@ -533,7 +548,7 @@ func TestAzureBlobHistoryStoreInvalidationKeepsNewerInitialization(t *testing.T) creates++ return nil } - store.uploadBlobHook = func(context.Context, string, string, []byte, *azblob.UploadBufferOptions) error { + store.uploadBlobHook = func(context.Context, string, string, io.Reader, *blockblob.CommitBlockListOptions) error { return nil } @@ -562,24 +577,26 @@ func TestAzureBlobHistoryStoreSendsHeadersAndMetadata(t *testing.T) { require.NoError(t, err) store.createContainerHook = func(context.Context, string) error { return nil } - var captured *azblob.UploadBufferOptions + var captured *blockblob.CommitBlockListOptions var capturedBody []byte var capturedName string store.uploadBlobHook = func( _ context.Context, _ string, name string, - body []byte, - options *azblob.UploadBufferOptions, + body io.Reader, + options *blockblob.CommitBlockListOptions, ) error { - capturedName, capturedBody, captured = name, body, options - return nil + capturedName, captured = name, options + var err error + capturedBody, err = io.ReadAll(body) + return err } require.NoError(t, store.Write(context.Background(), ExportObject{ Container: "primary", Name: "prefix/object.jsonl.gz", - Content: []byte("body"), + Content: strings.NewReader("body"), ContentType: "application/gzip", Metadata: map[string]string{"instanceId": "abc"}, })) @@ -591,20 +608,91 @@ func TestAzureBlobHistoryStoreSendsHeadersAndMetadata(t *testing.T) { // The store never declares a content coding, so no reader transparently // decompresses an object whose name promises gzip bytes. assert.Nil(t, captured.HTTPHeaders.BlobContentEncoding) - require.NotNil(t, captured.Metadata["instanceId"]) - assert.Equal(t, "abc", *captured.Metadata["instanceId"]) + require.NotNil(t, captured.Metadata["instanceIdBase64"]) + assert.Equal(t, base64.RawURLEncoding.EncodeToString([]byte("abc")), *captured.Metadata["instanceIdBase64"]) + assert.NotContains(t, captured.Metadata, "instanceId") } -func TestGzipContentRoundTrip(t *testing.T) { - payload := []byte(strings.Repeat("history event\n", 512)) - compressed, err := gzipContent(payload) +func TestAzureBlobHistoryStoreFailedBlockCancelsProducer(t *testing.T) { + var blocks, commits atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Query().Get("comp") { + case "block": + blocks.Add(1) + w.Header().Set("x-ms-error-code", "AuthorizationPermissionMismatch") + w.WriteHeader(http.StatusForbidden) + case "blocklist": + commits.Add(1) + w.WriteHeader(http.StatusCreated) + default: + t.Errorf("unexpected blob request: %s", r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + defer server.Close() + store, err := NewAzureBlobHistoryStore(AzureBlobHistoryStoreOptions{ + ConnectionString: "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=" + + placeholderAccountKey + ";BlobEndpoint=" + server.URL + "/devstoreaccount1;", + ContainerName: "container", AllowInsecureHTTP: true, + }) require.NoError(t, err) - assert.Less(t, len(compressed), len(payload)) - assert.Equal(t, payload, decompressGzip(t, compressed)) + store.createContainerHook = func(context.Context, string) error { return nil } + base := streamTestSource() + stopped := make(chan struct{}) + source := streamingSource{fakeSource: base, stream: func(ctx context.Context, handler api.HistoryEventHandler) error { + defer close(stopped) + if err := handler(base.history["subject"].Events[0]); err != nil { + return err + } + if err := handler(&api.HistoryEvent{Generic: &api.HistoryPayloadEvent{ + SerializedInput: strings.Repeat("x", 2*historyUploadBlockSize), + }}); err != nil { + return err + } + <-ctx.Done() + return ctx.Err() + }} + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + request := streamTestRequest() + request.Format.Kind = ExportFormatJSON + _, err = newTestRuntime(source, store).exportInstance(ctx, request) + require.Error(t, err) + var responseErr *azcore.ResponseError + require.ErrorAs(t, err, &responseErr) + require.Equal(t, http.StatusForbidden, responseErr.StatusCode) + require.NoError(t, ctx.Err(), "upload failure must not wait for a source receive or outer deadline") + <-stopped + require.Equal(t, int32(1), blocks.Load()) + require.Zero(t, commits.Load(), "failed blocks must never be committed") +} - empty, err := gzipContent(nil) - require.NoError(t, err) - assert.Empty(t, decompressGzip(t, empty)) +type terminalErrorReader struct { + content string + err error +} + +func (r *terminalErrorReader) Read(p []byte) (int, error) { + n := copy(p, r.content) + r.content = r.content[n:] + return n, r.err +} + +func TestReadHistoryBlockPreservesErrors(t *testing.T) { + ctx := context.Background() + for _, size := range []int{3, 8} { + for _, failure := range []error{io.ErrUnexpectedEOF, errors.New("source failed")} { + reader := &terminalErrorReader{content: strings.Repeat("x", size), err: failure} + n, err := readHistoryBlock(ctx, reader, make([]byte, 8)) + require.Equal(t, size, n) + require.ErrorIs(t, err, failure, "a source error accompanying data must survive") + } + } + n, err := readHistoryBlock(ctx, strings.NewReader("end"), make([]byte, 8)) + require.Equal(t, 3, n) + require.ErrorIs(t, err, io.EOF, "a clean partial final block keeps its EOF") + _, err = readHistoryBlock(ctx, &terminalErrorReader{}, make([]byte, 8)) + require.ErrorIs(t, err, io.ErrNoProgress) } func decompressGzip(t *testing.T, content []byte) []byte { diff --git a/exporthistory/streaming_test.go b/exporthistory/streaming_test.go new file mode 100644 index 00000000..e0154113 --- /dev/null +++ b/exporthistory/streaming_test.go @@ -0,0 +1,269 @@ +package exporthistory + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "runtime" + "strings" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type streamingSource struct { + *fakeSource + stream func(context.Context, api.HistoryEventHandler) error +} + +func (s streamingSource) StreamOrchestrationHistory(ctx context.Context, _ api.InstanceID, _ api.HistoryQuery, handler api.HistoryEventHandler) error { + return s.stream(ctx, handler) +} + +type streamingStore func(context.Context, ExportObject) error + +func (s streamingStore) Write(ctx context.Context, object ExportObject) error { return s(ctx, object) } + +func streamTestRequest() ExportRequest { + return ExportRequest{ + InstanceID: "subject", Destination: ExportDestination{Container: "container"}, Format: DefaultExportFormat(), + } +} + +func streamTestSource() *fakeSource { + source := newFakeSource() + source.addInstance("subject", api.RUNTIME_STATUS_COMPLETED, 1) + return source +} + +func TestExportStreamingFailuresNeverPublish(t *testing.T) { + sentinel := errors.New("source unavailable") + for _, format := range []ExportFormatKind{ExportFormatJSON, ExportFormatJSONL} { + for _, failure := range []error{sentinel, io.EOF, io.ErrUnexpectedEOF} { + t.Run(format.String()+"/"+failure.Error(), func(t *testing.T) { + base := streamTestSource() + source := streamingSource{fakeSource: base, stream: func(_ context.Context, handler api.HistoryEventHandler) error { + if err := handler(base.history["subject"].Events[0]); err != nil { + return err + } + return failure + }} + store := newMemoryStore() + request := streamTestRequest() + request.Format.Kind = format + result, err := newTestRuntime(source, store).exportInstance(context.Background(), request) + require.ErrorIs(t, err, failure) + assert.False(t, result.Success) + assert.Zero(t, store.count(), "partial output must never become visible") + }) + } + } +} + +func TestExportStreamingCancelsBothDirections(t *testing.T) { + t.Run("upload failure cancels a blocked source", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + entered := make(chan struct{}) + stopped := make(chan struct{}) + source := streamingSource{fakeSource: streamTestSource(), stream: func(ctx context.Context, _ api.HistoryEventHandler) error { + close(entered) + <-ctx.Done() + close(stopped) + return ctx.Err() + }} + failure := errors.New("upload rejected") + store := streamingStore(func(context.Context, ExportObject) error { + <-entered + return failure + }) + _, err := newTestRuntime(source, store).exportInstance(ctx, streamTestRequest()) + require.ErrorIs(t, err, failure) + require.NoError(t, ctx.Err(), "producer must be canceled without waiting for the outer deadline") + <-stopped + }) + + t.Run("source failure cancels a blocked uploader", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + failure := errors.New("source failed") + source := streamingSource{fakeSource: streamTestSource(), stream: func(context.Context, api.HistoryEventHandler) error { + return failure + }} + store := streamingStore(func(ctx context.Context, _ ExportObject) error { + <-ctx.Done() + return ctx.Err() + }) + _, err := newTestRuntime(source, store).exportInstance(ctx, streamTestRequest()) + require.ErrorIs(t, err, failure) + require.NoError(t, ctx.Err()) + }) + + t.Run("outer cancellation joins both sides", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + entered := make(chan struct{}) + source := streamingSource{fakeSource: streamTestSource(), stream: func(ctx context.Context, _ api.HistoryEventHandler) error { + close(entered) + <-ctx.Done() + return ctx.Err() + }} + finished := make(chan error, 1) + go func() { + _, err := newTestRuntime(source, newMemoryStore()).exportInstance(ctx, streamTestRequest()) + finished <- err + }() + <-entered + cancel() + select { + case err := <-finished: + require.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("canceled stream leaked its producer or uploader") + } + }) +} + +func TestExportStreamingRetryReopensHistory(t *testing.T) { + base := streamTestSource() + reads := 0 + source := streamingSource{fakeSource: base, stream: func(_ context.Context, handler api.HistoryEventHandler) error { + reads++ + return handler(base.history["subject"].Events[0]) + }} + store := newMemoryStore() + failure := errors.New("transient upload failure") + store.failure = failure + exporter := newTestRuntime(source, store) + _, err := exporter.exportInstance(context.Background(), streamTestRequest()) + require.ErrorIs(t, err, failure) + store.failure = nil + result, err := exporter.exportInstance(context.Background(), streamTestRequest()) + require.NoError(t, err) + require.True(t, result.Success) + require.Equal(t, 2, reads) + require.Equal(t, 1, result.EventCount) + require.Equal(t, 1, store.count()) +} + +func TestExportStreamingLimits(t *testing.T) { + for _, test := range []struct { + name string + query api.HistoryQuery + }{ + {"events", api.HistoryQuery{MaxEvents: 1}}, + {"bytes", api.HistoryQuery{MaxBytes: 1024}}, + } { + t.Run(test.name, func(t *testing.T) { + source := streamTestSource() + source.history["subject"].Events = append(source.history["subject"].Events, + &api.HistoryEvent{Generic: &api.HistoryPayloadEvent{SerializedInput: strings.Repeat("x", 4096)}}) + store := newMemoryStore() + exporter := newTestRuntime(source, store) + exporter.historyPage = test.query + result, err := exporter.exportInstance(context.Background(), streamTestRequest()) + require.ErrorIs(t, err, api.ErrHistoryLimitExceeded) + require.False(t, result.Success) + require.Zero(t, store.count()) + }) + } +} + +type failingWriter struct{ err error } + +func (w failingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestExportStreamingCloseFailure(t *testing.T) { + failure := errors.New("gzip trailer write failed") + // With no events, gzip writes its header/trailer only from Close. + _, err := writeHistory(context.Background(), failingWriter{failure}, DefaultExportFormat(), + api.HistoryQuery{}, func(api.HistoryEventHandler) error { return nil }) + require.ErrorIs(t, err, failure) + require.ErrorContains(t, err, "finish export compression") +} + +func TestExportStreamingExactJSONShape(t *testing.T) { + source := streamTestSource() + source.history["subject"].Events = append(source.history["subject"].Events, + &api.HistoryEvent{Generic: &api.HistoryPayloadEvent{SerializedInput: `"Unicode: 日本語 <&>"`}}) + for _, format := range []ExportFormatKind{ExportFormatJSON, ExportFormatJSONL} { + request := streamTestRequest() + request.Format.Kind = format + var output bytes.Buffer + store := streamingStore(func(_ context.Context, object ExportObject) error { + _, err := io.Copy(&output, object.Content) + return err + }) + result, err := newTestRuntime(source, store).exportInstance(context.Background(), request) + require.NoError(t, err) + require.Equal(t, 2, result.EventCount) + if format == ExportFormatJSON { + expected, err := json.Marshal(source.history["subject"].Events) + require.NoError(t, err) + require.Equal(t, expected, output.Bytes()) + } else { + var expected bytes.Buffer + for _, event := range source.history["subject"].Events { + encoded, err := json.Marshal(event) + require.NoError(t, err) + expected.Write(encoded) + expected.WriteByte('\n') + } + require.Equal(t, expected.Bytes(), decompressGzip(t, output.Bytes())) + } + } +} + +func TestExportStreamingMemoryDoesNotScaleWithHistory(t *testing.T) { + for _, format := range []ExportFormatKind{ExportFormatJSON, ExportFormatJSONL} { + t.Run(format.String(), func(t *testing.T) { + measure := func(events int) uint64 { + runtime.GC() + var initial runtime.MemStats + runtime.ReadMemStats(&initial) + peak := initial.HeapAlloc + base := streamTestSource() + source := streamingSource{fakeSource: base, stream: func(_ context.Context, handler api.HistoryEventHandler) error { + if err := handler(base.history["subject"].Events[0]); err != nil { + return err + } + for i := 0; i < events; i++ { + event := &api.HistoryEvent{ + Generic: &api.HistoryPayloadEvent{SerializedInput: strings.Repeat("x", 16*1024)}, + } + if err := handler(event); err != nil { + return err + } + if i%128 == 0 || i == events-1 { + runtime.GC() + var sample runtime.MemStats + runtime.ReadMemStats(&sample) + peak = max(peak, sample.HeapAlloc) + } + } + return nil + }} + store := streamingStore(func(_ context.Context, object ExportObject) error { + _, err := io.Copy(io.Discard, object.Content) + return err + }) + exporter := newTestRuntime(source, store) + exporter.historyPage.MaxBytes = api.MaxHistoryMaxBytes + request := streamTestRequest() + request.Format.Kind = format + result, err := exporter.exportInstance(context.Background(), request) + require.NoError(t, err) + require.Equal(t, events+1, result.EventCount) + return peak - initial.HeapAlloc + } + small, large := measure(128), measure(4096) + t.Logf("retained heap growth: 2 MiB history=%d, 64 MiB history=%d", small, large) + require.Less(t, large, small+8*1024*1024, "64 MiB of history must not be retained") + }) + } +} diff --git a/internal/historyconv/query.go b/internal/historyconv/query.go index 921f2eb8..d73783c7 100644 --- a/internal/historyconv/query.go +++ b/internal/historyconv/query.go @@ -33,17 +33,37 @@ func Collect( query api.HistoryQuery, stream func(api.HistoryEventHandler) error, ) (*api.OrchestrationHistory, error) { - normalized, err := api.NormalizeHistoryQuery(query) + result := &api.OrchestrationHistory{InstanceID: id} + executionID, _, err := StreamValidated(query, stream, func(event *api.HistoryEvent) error { + result.Events = append(result.Events, event) + return nil + }) if err != nil { return nil, err } - result := &api.OrchestrationHistory{InstanceID: id} + result.ExecutionID = executionID + return result, nil +} + +// StreamValidated applies the same aggregate limits and observed execution +// identity checks as Collect without retaining events. stream must invoke its +// handler serially and stop on a handler error. Success includes validation of +// the end of the stream; callers must not publish output before this returns. +func StreamValidated( + query api.HistoryQuery, + stream func(api.HistoryEventHandler) error, + handler api.HistoryEventHandler, +) (executionID string, eventCount int, err error) { + normalized, err := api.NormalizeHistoryQuery(query) + if err != nil { + return "", 0, err + } totalBytes := 0 - err = stream(func(event *api.HistoryEvent) error { + validate := func(event *api.HistoryEvent) error { if event == nil { return errors.New("history event must not be nil") } - if len(result.Events) >= normalized.MaxEvents { + if eventCount >= normalized.MaxEvents { return fmt.Errorf("%w: limit %d", api.ErrHistoryLimitExceeded, normalized.MaxEvents) } if event.ExecutionStarted != nil || event.Type == api.HistoryEventExecutionStarted { @@ -55,25 +75,35 @@ func Collect( return fmt.Errorf("history execution ID %q does not match requested execution %q", started.ExecutionID, normalized.ExecutionID) } - if result.ExecutionID != "" && started.ExecutionID != result.ExecutionID { + if executionID != "" && started.ExecutionID != executionID { return fmt.Errorf("history contains conflicting execution IDs %q and %q", - result.ExecutionID, started.ExecutionID) + executionID, started.ExecutionID) } - result.ExecutionID = started.ExecutionID + executionID = started.ExecutionID } totalBytes = addSize(totalBytes, ApproximateEventSize(event)) if totalBytes > normalized.MaxBytes { return fmt.Errorf("%w: byte limit %d", api.ErrHistoryLimitExceeded, normalized.MaxBytes) } - result.Events = append(result.Events, event) - return nil + eventCount++ + return handler(event) + } + var handlerErr error + err = stream(func(event *api.HistoryEvent) error { + if handlerErr == nil { + handlerErr = validate(event) + } + return handlerErr }) + if handlerErr != nil { + return "", eventCount, handlerErr + } if err != nil { - return nil, err + return "", eventCount, err } - if normalized.ExecutionID != "" && result.ExecutionID == "" { - return nil, fmt.Errorf("history is missing an ExecutionStarted event for requested execution %q", + if normalized.ExecutionID != "" && executionID == "" { + return "", eventCount, fmt.Errorf("history is missing an ExecutionStarted event for requested execution %q", normalized.ExecutionID) } - return result, nil + return executionID, eventCount, nil } diff --git a/internal/historyconv/query_test.go b/internal/historyconv/query_test.go index ca5630c9..a1f82e2b 100644 --- a/internal/historyconv/query_test.go +++ b/internal/historyconv/query_test.go @@ -1,6 +1,7 @@ package historyconv import ( + "errors" "strings" "testing" @@ -8,6 +9,33 @@ import ( "github.com/stretchr/testify/require" ) +func TestStreamValidatedDoesNotLoseHandlerErrors(t *testing.T) { + failure := errors.New("destination failed") + calls := 0 + _, _, err := StreamValidated(api.HistoryQuery{ExecutionID: "execution"}, func(handler api.HistoryEventHandler) error { + _ = handler(executionStarted("execution")) + _ = handler(executionStarted("execution")) + return nil // Even a misbehaving source cannot hide a handler failure. + }, func(*api.HistoryEvent) error { + calls++ + return failure + }) + require.ErrorIs(t, err, failure) + require.Equal(t, 1, calls) +} + +func TestStreamValidatedEndOfStreamIdentity(t *testing.T) { + called := false + _, _, err := StreamValidated(api.HistoryQuery{ExecutionID: "execution"}, func(handler api.HistoryEventHandler) error { + return handler(&api.HistoryEvent{Type: api.HistoryEventOrchestratorStarted}) + }, func(*api.HistoryEvent) error { + called = true + return nil + }) + require.True(t, called, "events are delivered without waiting for the whole history") + require.ErrorContains(t, err, "missing an ExecutionStarted") +} + func collectEvents(query api.HistoryQuery, events ...*api.HistoryEvent) (*api.OrchestrationHistory, error) { return Collect("instance", query, func(handler api.HistoryEventHandler) error { for _, event := range events { diff --git a/samples/exporthistory/exporthistory.go b/samples/exporthistory/exporthistory.go index e52df632..2c1f8431 100644 --- a/samples/exporthistory/exporthistory.go +++ b/samples/exporthistory/exporthistory.go @@ -7,6 +7,7 @@ import ( "compress/gzip" "context" "crypto/rand" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -534,10 +535,15 @@ func downloadHistories( if item.Name == nil { continue } - instanceID := metadataValue(item.Metadata, "instanceId") - if instanceID == "" { - return nil, fmt.Errorf("blob %s has no instanceId metadata", *item.Name) + encodedID := metadataValue(item.Metadata, "instanceIdBase64") + if encodedID == "" { + return nil, fmt.Errorf("blob %s has no instanceIdBase64 metadata", *item.Name) } + decodedID, err := base64.RawURLEncoding.DecodeString(encodedID) + if err != nil { + return nil, fmt.Errorf("decode instance ID metadata on blob %s: %w", *item.Name, err) + } + instanceID := string(decodedID) response, err := client.DownloadStream(ctx, container, *item.Name, nil) if err != nil { return nil, err diff --git a/tests/durabletaskscheduler/dts_export_history_test.go b/tests/durabletaskscheduler/dts_export_history_test.go index 83fec467..83fbe503 100644 --- a/tests/durabletaskscheduler/dts_export_history_test.go +++ b/tests/durabletaskscheduler/dts_export_history_test.go @@ -5,6 +5,7 @@ import ( "compress/gzip" "context" "crypto/rand" + "encoding/base64" "encoding/hex" "encoding/json" "fmt" @@ -336,11 +337,13 @@ func listExportedInstances( assert.True(t, strings.HasSuffix(name, ".jsonl.gz"), name) instanceID := "" for key, value := range item.Metadata { - if strings.EqualFold(key, "instanceId") && value != nil { - instanceID = *value + if strings.EqualFold(key, "instanceIdBase64") && value != nil { + decoded, err := base64.RawURLEncoding.DecodeString(*value) + require.NoError(t, err, "invalid instanceIdBase64 metadata") + instanceID = string(decoded) } } - require.NotEmpty(t, instanceID, "exported object %s has no instanceId metadata", name) + require.NotEmpty(t, instanceID, "exported object %s has no instanceIdBase64 metadata", name) // The object is an opaque gzip file, so nothing transparently // decompresses it and the download is always the gzip stream. assert.Equal(t, "application/gzip", derefBlobString(item.Properties.ContentType), name) From 3e8cc74047a7ed067802c49a8e81c382f5a8c2be Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:38:17 -0700 Subject: [PATCH 21/26] fix: guard transport retries and orchestration dispatch Restrict UNAVAILABLE retries to read-only RPCs and Hello, reject malformed rewind envelopes, and skip callbacks canceled before their first resume. Cover accepted-event acknowledgement loss with real gRPC and canceled coroutine replay through DTS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- durabletaskscheduler/README.md | 9 ++- durabletaskscheduler/authentication_test.go | 13 +++- durabletaskscheduler/connection.go | 12 +++- durabletaskscheduler/retry_safety_test.go | 59 +++++++++++++++++++ task/cancel_test.go | 7 ++- task/coroutine.go | 4 ++ task/coroutine_cancellation_test.go | 58 ++++++++++++++++++ task/orchestrator.go | 1 + task/rewind.go | 2 +- task/rewind_test.go | 6 +- .../dts_coroutine_review_test.go | 59 +++++++++++++++++++ 11 files changed, 222 insertions(+), 8 deletions(-) create mode 100644 durabletaskscheduler/retry_safety_test.go create mode 100644 task/coroutine_cancellation_test.go create mode 100644 tests/durabletaskscheduler/dts_coroutine_review_test.go diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index fd0742ad..ef6f46c7 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -69,9 +69,12 @@ whitespace or newlines. Unset worker IDs default to `Options.HelloTimeout` (default 30 seconds) bounds the fail-fast `Hello` handshake for both `NewClient` and the worker connection factory; the caller's context still applies when it is shorter. Client channels use a default gRPC -service config that retries `UNAVAILABLE` up to five attempts with a 50 ms -initial backoff, 250 ms cap, and multiplier 2. Worker channels do not, because -the worker owns its own reconnect loop. +service config that retries `UNAVAILABLE` for the idempotent `Hello` handshake +and read-only RPCs up to five attempts with a 50 ms initial backoff, 250 ms cap, +and multiplier 2. Mutations such as `RaiseEvent`, `SignalEntity`, and instance +lifecycle operations are not automatically retried: a failed acknowledgement +can follow a successful service-side change. Worker channels do not use this +policy, because the worker owns its own reconnect loop. Individual gRPC messages are bounded to 64 MiB by default through `Options.MaxReceiveMessageSize` and `Options.MaxSendMessageSize`. The worker uses a diff --git a/durabletaskscheduler/authentication_test.go b/durabletaskscheduler/authentication_test.go index b0adf69a..b562d5b7 100644 --- a/durabletaskscheduler/authentication_test.go +++ b/durabletaskscheduler/authentication_test.go @@ -687,7 +687,18 @@ func TestRetryServiceConfigDefaults(t *testing.T) { require.NoError(t, json.Unmarshal([]byte(retryServiceConfig), &parsed)) require.Len(t, parsed.MethodConfig, 1) method := parsed.MethodConfig[0] - require.Equal(t, []map[string]any{{}}, method.Name) + var methods []string + for _, selector := range method.Name { + require.Len(t, selector, 2) + require.Equal(t, "TaskHubSidecarService", selector["service"]) + name, ok := selector["method"].(string) + require.True(t, ok) + methods = append(methods, name) + } + require.ElementsMatch(t, []string{ + "Hello", "GetInstance", "WaitForInstanceStart", "WaitForInstanceCompletion", + "QueryInstances", "ListInstanceIds", "StreamInstanceHistory", "GetEntity", "QueryEntities", + }, methods) require.Equal(t, 5, method.RetryPolicy.MaxAttempts) require.Equal(t, "0.050s", method.RetryPolicy.InitialBackoff) require.Equal(t, "0.250s", method.RetryPolicy.MaxBackoff) diff --git a/durabletaskscheduler/connection.go b/durabletaskscheduler/connection.go index d487884c..bf8fbb51 100644 --- a/durabletaskscheduler/connection.go +++ b/durabletaskscheduler/connection.go @@ -50,7 +50,17 @@ type schedulerCredentialRefresh struct { const retryServiceConfig = `{ "methodConfig": [{ - "name": [{}], + "name": [ + {"service": "TaskHubSidecarService", "method": "Hello"}, + {"service": "TaskHubSidecarService", "method": "GetInstance"}, + {"service": "TaskHubSidecarService", "method": "WaitForInstanceStart"}, + {"service": "TaskHubSidecarService", "method": "WaitForInstanceCompletion"}, + {"service": "TaskHubSidecarService", "method": "QueryInstances"}, + {"service": "TaskHubSidecarService", "method": "ListInstanceIds"}, + {"service": "TaskHubSidecarService", "method": "StreamInstanceHistory"}, + {"service": "TaskHubSidecarService", "method": "GetEntity"}, + {"service": "TaskHubSidecarService", "method": "QueryEntities"} + ], "retryPolicy": { "maxAttempts": 5, "initialBackoff": "0.050s", diff --git a/durabletaskscheduler/retry_safety_test.go b/durabletaskscheduler/retry_safety_test.go new file mode 100644 index 00000000..bf0b6fa8 --- /dev/null +++ b/durabletaskscheduler/retry_safety_test.go @@ -0,0 +1,59 @@ +package durabletaskscheduler + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +type retrySafetyServer struct { + protos.UnimplementedTaskHubSidecarServiceServer + reads atomic.Int32 + acceptedEvents atomic.Int32 +} + +func (*retrySafetyServer) Hello(context.Context, *emptypb.Empty) (*emptypb.Empty, error) { + return &emptypb.Empty{}, nil +} + +func (s *retrySafetyServer) GetInstance(context.Context, *protos.GetInstanceRequest) (*protos.GetInstanceResponse, error) { + if s.reads.Add(1) <= 3 { + return nil, status.Error(codes.Unavailable, "temporary read failure") + } + return &protos.GetInstanceResponse{Exists: false}, nil +} + +func (s *retrySafetyServer) RaiseEvent(context.Context, *protos.RaiseEventRequest) (*protos.RaiseEventResponse, error) { + s.acceptedEvents.Add(1) + // Returning without response headers reproduces a trailers-only failure + // after the server has already accepted the non-idempotent operation. + return nil, status.Error(codes.Unavailable, "event accepted but acknowledgement failed") +} + +func TestClientRetriesReadsWithoutDuplicatingAcceptedEvents(t *testing.T) { + server := new(retrySafetyServer) + listener, stop := startBufconnServer(t, server) + defer stop() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + client, err := NewClient(ctx, insecureBufconnOptions(t, listener), api.DefaultLogger()) + require.NoError(t, err) + defer func() { require.NoError(t, client.Close()) }() + + _, err = client.FetchOrchestrationMetadata(ctx, "instance") + require.ErrorIs(t, err, api.ErrInstanceNotFound) + require.EqualValues(t, 4, server.reads.Load()) + + err = client.RaiseEvent(ctx, "instance", "event", api.WithEventPayload("once")) + require.Equal(t, codes.Unavailable, status.Code(err)) + require.ErrorContains(t, err, "event accepted but acknowledgement failed") + require.EqualValues(t, 1, server.acceptedEvents.Load()) +} diff --git a/task/cancel_test.go b/task/cancel_test.go index cba212d8..4def47c9 100644 --- a/task/cancel_test.go +++ b/task/cancel_test.go @@ -486,9 +486,14 @@ func TestAlreadyCanceledSelectDoesNotWaitOrConsume(t *testing.T) { runner.Go(func(*OrchestrationContext) { defer done.Done() defer func() { canceled = isTaskCanceled(recover()) }() + // Start the callback before cancellation; an unstarted + // callback in a canceled scope must not run at all. + cancel() + if err := child.WaitForSingleEvent("cancel-boundary", -1).Await(nil); !errors.Is(err, ErrTaskCanceled) { + panic("cancellation was not applied at the scheduler boundary") + } selector.Select(OnEvent(channel, func(int) { invoked = true })) }) - cancel() done.Wait(ctx) value, received, err := channel.TryReceiveErr() _, again, nextErr := channel.TryReceiveErr() diff --git a/task/coroutine.go b/task/coroutine.go index dacc9b87..1f04b309 100644 --- a/task/coroutine.go +++ b/task/coroutine.go @@ -72,6 +72,10 @@ func (c *coroutine) run() { return } + if c.scope.isCanceled() { + c.sendSignal(coroutineSignal{kind: coroutineCanceled}) + return + } c.fn() c.sendSignal(coroutineSignal{kind: coroutineFinished}) } diff --git a/task/coroutine_cancellation_test.go b/task/coroutine_cancellation_test.go new file mode 100644 index 00000000..63a34d39 --- /dev/null +++ b/task/coroutine_cancellation_test.go @@ -0,0 +1,58 @@ +package task + +import ( + "errors" + "testing" + + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/stretchr/testify/require" +) + +func TestCanceledScopeSkipsUnstartedCoroutines(t *testing.T) { + for _, ordering := range []string{"cancel-before-enqueue", "cancel-after-enqueue", "already-canceled"} { + t.Run(ordering, func(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("canceled-callback", func(ctx *OrchestrationContext) (any, error) { + child, cancel := ctx.WithCancel() + if ordering != "cancel-after-enqueue" { + cancel() + } + if ordering == "already-canceled" { + if err := child.WaitForSingleEvent("never", -1).Await(nil); !errors.Is(err, ErrTaskCanceled) { + return nil, errors.New("cancellation was not applied") + } + } + ran, siblingRan := false, false + child.Go(func(ctx *OrchestrationContext) { + ran = true + ctx.SetCustomStatus("canceled callback ran") + }) + cancel() + group := ctx.NewWaitGroup() + group.Add(1) + ctx.Go(func(*OrchestrationContext) { + siblingRan = true + group.Done() + }) + group.Wait(ctx) + return !ran && siblingRan, nil + })) + events := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("canceled-callback", "instance", nil, nil, nil, nil), + } + for _, replay := range []bool{false, true} { + var oldEvents, newEvents []*protos.HistoryEvent + if replay { + oldEvents = events + } else { + newEvents = events + } + response := executeOrchestrationTurn(t, registry, "instance", oldEvents, newEvents) + require.Equal(t, "true", completionResult(t, response), "replay=%t", replay) + require.Empty(t, response.GetCustomStatus().GetValue(), "replay=%t", replay) + } + }) + } +} diff --git a/task/orchestrator.go b/task/orchestrator.go index 1b395d8b..2727fa2b 100644 --- a/task/orchestrator.go +++ b/task/orchestrator.go @@ -658,6 +658,7 @@ func (ctx *OrchestrationContext) newFailedTask(engine *OrchestrationContext, err // Go starts a coroutine that is cooperatively scheduled with the orchestration. // Only one orchestration coroutine runs at a time, in monotonically increasing ID order. +// A callback whose scope is canceled before it starts is not invoked. func (ctx *OrchestrationContext) Go(fn func(ctx *OrchestrationContext)) { if fn == nil { panic("orchestration coroutine function must be non-nil") diff --git a/task/rewind.go b/task/rewind.go index c3f4909f..ba10beb2 100644 --- a/task/rewind.go +++ b/task/rewind.go @@ -26,7 +26,7 @@ func isRewindRequest(oldEvents, newEvents []*protos.HistoryEvent) bool { // Keep this transformation aligned with durabletask-python's // _OrchestrationExecutor._build_rewind_result and the DTS backend contract. func buildRewindResult(id api.InstanceID, oldEvents, newEvents []*protos.HistoryEvent) (*ExecutionResults, error) { - if len(newEvents) != 2 || newEvents[1].GetExecutionRewound() == nil { + if len(newEvents) != 2 || newEvents[0].GetOrchestratorStarted() == nil || newEvents[1].GetExecutionRewound() == nil { return nil, fmt.Errorf("rewind requires exactly two new events: orchestrator started and execution rewound") } rewound := newEvents[1].GetExecutionRewound() diff --git a/task/rewind_test.go b/task/rewind_test.go index d5a5c401..40387607 100644 --- a/task/rewind_test.go +++ b/task/rewind_test.go @@ -157,11 +157,15 @@ func TestRewindMalformedRequests(t *testing.T) { for _, events := range [][]*protos.HistoryEvent{ {rewindEvent("")}, {rewindEvent(""), helpers.NewOrchestratorStartedEvent()}, + {nil, rewindEvent("")}, + {new(protos.HistoryEvent), rewindEvent("")}, + {helpers.NewTaskCompletedEvent(0, nil), rewindEvent("")}, {helpers.NewOrchestratorStartedEvent(), rewindEvent(""), helpers.NewEventRaisedEvent("extra", nil)}, } { - _, err := NewTaskExecutor(NewTaskRegistry()).ExecuteOrchestrator(context.Background(), "instance", + result, err := NewTaskExecutor(NewTaskRegistry()).ExecuteOrchestrator(context.Background(), "instance", []*protos.HistoryEvent{failedCompletionEvent()}, events, nil) require.ErrorContains(t, err, "rewind requires exactly two new events") + require.Nil(t, result) } require.False(t, isRewindRequest([]*protos.HistoryEvent{rewindEvent(""), failedCompletionEvent()}, nil)) require.False(t, isRewindRequest(nil, []*protos.HistoryEvent{rewindEvent("")})) diff --git a/tests/durabletaskscheduler/dts_coroutine_review_test.go b/tests/durabletaskscheduler/dts_coroutine_review_test.go new file mode 100644 index 00000000..de133829 --- /dev/null +++ b/tests/durabletaskscheduler/dts_coroutine_review_test.go @@ -0,0 +1,59 @@ +package durabletaskscheduler_test + +import ( + "context" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" +) + +func TestDTSCanceledScopeSkipsUnstartedCoroutines(t *testing.T) { + name := "DTSReviewCanceledCallback-" + uuid.NewString() + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + var cancelFirst bool + if err := ctx.GetInput(&cancelFirst); err != nil { + return nil, err + } + child, cancel := ctx.WithCancel() + if cancelFirst { + cancel() + } + ran, siblingRan := false, false + child.Go(func(ctx *task.OrchestrationContext) { + ran = true + ctx.SetCustomStatus("canceled callback ran") + }) + cancel() + group := ctx.NewWaitGroup() + group.Add(1) + ctx.Go(func(*task.OrchestrationContext) { + siblingRan = true + group.Done() + }) + group.Wait(ctx) + if err := ctx.CreateTimer(20 * time.Millisecond).Await(nil); err != nil { + return nil, err + } + return !ran && siblingRan, nil + })) + client, _, _ := startEmulatorClientAndWorker(t, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + for _, cancelFirst := range []bool{false, true} { + id := uniqueInstanceID("go-canceled-callback") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithInput(cancelFirst)) + require.NoError(t, err) + result, err := client.WaitForOrchestrationCompletion(ctx, id, api.WithFetchPayloads(true)) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, result.RuntimeStatus, "%+v", result.FailureDetails) + require.Equal(t, "true", result.SerializedOutput) + require.Empty(t, result.SerializedCustomStatus) + } +} From cbfd2b6f8e423df674311c0ace497b198b1f901b Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Thu, 10 Sep 2026 15:58:03 -0700 Subject: [PATCH 22/26] fix: bound current-or-older work item filters Share runtime version comparison with route generation, exclude future versions without wildcard fallthrough, and preserve compatible current/default legacy fallback. Reject unrepresentable automatic wildcard routes and verify older workers never abandon newer work on DTS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- client/current_or_older_filter_test.go | 84 +++++++++++++++ client/grpc_worker.go | 40 +++++-- client/grpc_worker_test.go | 4 +- durabletaskscheduler/README.md | 10 +- internal/helpers/versioning.go | 62 +++++++++++ task/versioning.go | 58 +--------- task/versioning_test.go | 2 +- .../dts_current_or_older_review_test.go | 101 ++++++++++++++++++ 8 files changed, 293 insertions(+), 68 deletions(-) create mode 100644 client/current_or_older_filter_test.go create mode 100644 internal/helpers/versioning.go create mode 100644 tests/durabletaskscheduler/dts_current_or_older_review_test.go diff --git a/client/current_or_older_filter_test.go b/client/current_or_older_filter_test.go new file mode 100644 index 00000000..aa968f66 --- /dev/null +++ b/client/current_or_older_filter_test.go @@ -0,0 +1,84 @@ +package client + +import ( + "slices" + "strings" + "testing" + + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" +) + +func TestCurrentOrOlderAutoFiltersBoundRegisteredVersions(t *testing.T) { + tests := []struct { + name string + worker string + defaultVersion string + versions []string + want []string + }{ + {"numeric", "2.0", "", []string{"", "1.0", "2.0", "2.0.0", "10.0"}, []string{"", "1.0", "2.0"}}, + {"opaque", "v2", "", []string{"", "v1", "V2", "v3"}, []string{"", "v1", "V2"}}, + {"unversioned fallback", "2.0", "", []string{""}, []string{"", "2.0"}}, + {"older default fallback", "2.0", "1.0", []string{""}, []string{"", "1.0", "2.0"}}, + {"equivalent default fallback", "v2", "V2", []string{""}, []string{"", "v2"}}, + {"newer default excluded", "2.0", "3.0", []string{""}, []string{"", "2.0"}}, + {"versioned handler disables fallback", "2.0", "1.0", []string{"", "2.0"}, []string{"", "2.0"}}, + {"newer only", "2.0", "", []string{"3.0"}, nil}, + {"unversioned worker", "", "1.0", []string{"", "1.0"}, []string{""}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + registrations := make([]task.TaskRegistration, 0, len(test.versions)) + for _, version := range test.versions { + registrations = append(registrations, task.TaskRegistration{Name: "mixed", Version: version}) + } + filters := workItemFiltersFromRegistry(task.TaskRegistrySnapshot{ + Orchestrators: registrations, + Activities: registrations, + }, &task.VersioningOptions{ + Version: test.worker, DefaultVersion: test.defaultVersion, MatchStrategy: task.VersionMatchCurrentOrOlder, + }, nil, nil) + require.Equal(t, len(test.want) == 0, filters.RejectAllOrchestrations) + require.Equal(t, filters.RejectAllOrchestrations, filters.RejectAllActivities) + if len(test.want) == 0 { + require.Empty(t, filters.Orchestrations) + require.Empty(t, filters.Activities) + } else { + require.Equal(t, []WorkItemFilter{{Name: "mixed", Versions: test.want}}, filters.Orchestrations) + require.Equal(t, filters.Orchestrations, filters.Activities) + } + normalized, err := cloneWorkItemFilters(filters) + require.NoError(t, err) + wire := workItemFiltersToProto(normalized) + require.Len(t, wire.Orchestrations, 1) + require.Len(t, wire.Activities, 1) + if len(test.want) == 0 { + require.Equal(t, helpers.RejectAllWorkItemFilterName, wire.Orchestrations[0].Name) + require.Equal(t, helpers.RejectAllWorkItemFilterName, wire.Activities[0].Name) + } else { + require.Len(t, wire.Orchestrations[0].Versions, len(test.want)) + require.Len(t, wire.Activities[0].Versions, len(test.want)) + } + for _, version := range append(slices.Clone(test.versions), test.worker, test.defaultVersion, "future") { + want := slices.ContainsFunc(test.want, func(allowed string) bool { return strings.EqualFold(allowed, version) }) + require.Equal(t, want, matchesWorkItemFilters(normalized, true, "mixed", version), version) + require.Equal(t, want, matchesWorkItemFilters(normalized, false, "mixed", version), version) + } + }) + } +} + +func TestCurrentOrOlderAutoFiltersRejectWildcardRegistrations(t *testing.T) { + for _, snapshot := range []task.TaskRegistrySnapshot{ + {Orchestrators: []task.TaskRegistration{{Name: "*"}}}, + {Activities: []task.TaskRegistration{{Name: "*", Version: "1.0"}}}, + } { + err := validateAutoFilters(snapshot, &task.VersioningOptions{ + Version: "2.0", MatchStrategy: task.VersionMatchCurrentOrOlder, + }) + require.ErrorContains(t, err, "require named registrations") + require.NoError(t, validateAutoFilters(snapshot, nil)) + } +} diff --git a/client/grpc_worker.go b/client/grpc_worker.go index d1fa6110..dd64e9e3 100644 --- a/client/grpc_worker.go +++ b/client/grpc_worker.go @@ -606,7 +606,7 @@ func newTaskHubGrpcWorker( } } if options.autoWorkItemFilters && !options.workItemFiltersConfigured { - if err := validateStrictAutoFilters(snapshot, options.versioning); err != nil { + if err := validateAutoFilters(snapshot, options.versioning); err != nil { return nil, err } options.workItemFilters = workItemFiltersFromRegistry( @@ -666,21 +666,35 @@ func workItemFiltersFromRegistry( snapshot.Orchestrators, versioning, allowedUnversionedOrchestrators) activities := taskRegistrationsToFilters( snapshot.Activities, versioning, allowedUnversionedActivities) + // Nil denotes a wildcard registration; an empty non-nil list admits no work. return &WorkItemFilters{ Orchestrations: orchestrations, Activities: activities, Entities: entities, - RejectAllOrchestrations: len(snapshot.Orchestrators) == 0, - RejectAllActivities: len(snapshot.Activities) == 0, + RejectAllOrchestrations: orchestrations != nil && len(orchestrations) == 0, + RejectAllActivities: activities != nil && len(activities) == 0, RejectAllEntities: len(snapshot.Entities) == 0, } } -func validateStrictAutoFilters( +func validateAutoFilters( snapshot task.TaskRegistrySnapshot, versioning *task.VersioningOptions, ) error { - if versioning == nil || versioning.MatchStrategy != task.VersionMatchStrict { + if versioning == nil { + return nil + } + if versioning.MatchStrategy == task.VersionMatchCurrentOrOlder { + for _, registrations := range [][]task.TaskRegistration{snapshot.Orchestrators, snapshot.Activities} { + for _, registration := range registrations { + if registration.Name == "*" { + return fmt.Errorf("automatic current-or-older filters require named registrations; configure explicit work-item filters for wildcard handlers") + } + } + } + return nil + } + if versioning.MatchStrategy != task.VersionMatchStrict { return nil } if err := validateStrictRegistrations("orchestrator", snapshot.Orchestrators, versioning.Version); err != nil { @@ -791,6 +805,7 @@ func taskRegistrationsToFilters( group.versions[strings.ToLower(registration.Version)] = registration.Version } filters := make([]WorkItemFilter, 0, len(groups)) + currentOrOlder := versioning != nil && versioning.MatchStrategy == task.VersionMatchCurrentOrOlder for _, group := range groups { if versioning != nil && versioning.MatchStrategy == task.VersionMatchStrict { if _, allowed := allowedUnversioned[strings.ToLower(group.name)]; allowed { @@ -815,16 +830,27 @@ func taskRegistrationsToFilters( versions := make([]string, 0, len(group.versions)) _, hasUnversioned := group.versions[""] for normalized, version := range group.versions { - if normalized != "" { + if normalized != "" && (!currentOrOlder || helpers.CompareTaskVersions(version, versioning.Version) <= 0) { versions = append(versions, version) } } + if currentOrOlder && hasUnversioned && len(group.versions) == 1 { + for _, version := range []string{versioning.Version, versioning.DefaultVersion} { + if version != "" && helpers.CompareTaskVersions(version, versioning.Version) <= 0 && + !slices.ContainsFunc(versions, func(existing string) bool { return strings.EqualFold(existing, version) }) { + versions = append(versions, version) + } + } + } slices.SortFunc(versions, func(left, right string) int { return strings.Compare(strings.ToLower(left), strings.ToLower(right)) }) - if hasUnversioned && len(versions) > 0 { + if hasUnversioned && (len(versions) > 0 || currentOrOlder) { versions = append([]string{""}, versions...) } + if currentOrOlder && len(versions) == 0 { + continue + } filters = append(filters, WorkItemFilter{Name: group.name, Versions: versions}) } slices.SortFunc(filters, func(left, right WorkItemFilter) int { diff --git a/client/grpc_worker_test.go b/client/grpc_worker_test.go index 1582daf6..6efb1ae7 100644 --- a/client/grpc_worker_test.go +++ b/client/grpc_worker_test.go @@ -722,7 +722,7 @@ func TestWorkItemFiltersFromRegistryMatchVersionedFallbackRules(t *testing.T) { }, nil, nil) require.Equal(t, []string{"v3"}, strict.Orchestrations[0].Versions) require.Equal(t, []string{"v3"}, strict.Activities[0].Versions) - require.Error(t, validateStrictAutoFilters(registry.Snapshot(), &task.VersioningOptions{ + require.Error(t, validateAutoFilters(registry.Snapshot(), &task.VersioningOptions{ Version: "v3", MatchStrategy: task.VersionMatchStrict, })) @@ -783,7 +783,7 @@ func TestStrictAutoFiltersValidateNamedRegistrationsWithWildcard(t *testing.T) { require.NoError(t, registry.AddOrchestratorN("*", func(*task.OrchestrationContext) (any, error) { return nil, nil })) - require.Error(t, validateStrictAutoFilters(registry.Snapshot(), &task.VersioningOptions{ + require.Error(t, validateAutoFilters(registry.Snapshot(), &task.VersioningOptions{ Version: "v2", MatchStrategy: task.VersionMatchStrict, })) diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index ef6f46c7..9d5bc0d9 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -209,8 +209,14 @@ opaque version strings, but DTS applications should use numeric versions such as Use `client.WithAutoWorkItemFilters()` to derive filters from the registry, or `client.WithWorkItemFilters` for an explicit override. Local enforcement is a fallback for services that ignore the filter request. `CurrentOrOlder` ranges -cannot be represented by the protocol filter and are therefore enforced by the -worker. Service-side +cannot be represented by the protocol filter, so automatic filters advertise +only registered versions at or below the worker version, including an explicit +empty version for unversioned registrations. An unversioned-only handler also +advertises the current worker version and a compatible configured default version, +which it can resolve through fallback. +Register every older version that the worker should receive; unversioned +fallback does not advertise an open range. +Wildcard handlers require explicit filters with this strategy. Service-side filters can leave a task pending indefinitely when no worker advertises it, whereas unfiltered delivery produces a deterministic task-not-found failure. Auto-generated filters reject task kinds with no registrations and validate diff --git a/internal/helpers/versioning.go b/internal/helpers/versioning.go new file mode 100644 index 00000000..d299b57f --- /dev/null +++ b/internal/helpers/versioning.go @@ -0,0 +1,62 @@ +package helpers + +import ( + "strconv" + "strings" +) + +// CompareTaskVersions uses numeric comparison for two to four nonnegative +// integer components, otherwise case-insensitive lexical comparison. +// An empty version sorts before every nonempty version. +func CompareTaskVersions(left, right string) int { + left = strings.TrimSpace(left) + right = strings.TrimSpace(right) + switch { + case left == "" && right == "": + return 0 + case left == "": + return -1 + case right == "": + return 1 + } + + leftParts, leftOK := numericVersion(left) + rightParts, rightOK := numericVersion(right) + if leftOK && rightOK { + maxParts := max(len(leftParts), len(rightParts)) + for i := 0; i < maxParts; i++ { + leftValue := -1 + if i < len(leftParts) { + leftValue = leftParts[i] + } + rightValue := -1 + if i < len(rightParts) { + rightValue = rightParts[i] + } + if leftValue < rightValue { + return -1 + } + if leftValue > rightValue { + return 1 + } + } + return 0 + } + return strings.Compare(strings.ToLower(left), strings.ToLower(right)) +} + +func numericVersion(version string) ([]int, bool) { + parts := strings.Split(version, ".") + if len(parts) < 2 || len(parts) > 4 { + return nil, false + } + values := make([]int, len(parts)) + for i, part := range parts { + value, err := strconv.Atoi(part) + if err != nil || value < 0 { + return nil, false + } + values[i] = value + } + return values, true +} diff --git a/task/versioning.go b/task/versioning.go index 083bec51..0cd341dd 100644 --- a/task/versioning.go +++ b/task/versioning.go @@ -2,11 +2,10 @@ package task import ( "fmt" - "strconv" - "strings" "time" "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" ) // VersionMatchStrategy determines which task versions a worker accepts. @@ -102,7 +101,7 @@ func (o *VersioningOptions) check(taskVersion string) error { if o.MatchStrategy == VersionMatchNone { return nil } - comparison := compareVersions(taskVersion, o.Version) + comparison := helpers.CompareTaskVersions(taskVersion, o.Version) switch o.MatchStrategy { case VersionMatchStrict: if comparison == 0 { @@ -149,56 +148,3 @@ func (*versionConfigurationError) NonRetriable() bool { func (*versionConfigurationError) Is(target error) bool { return target == api.ErrVersionMismatch } - -func compareVersions(left, right string) int { - left = strings.TrimSpace(left) - right = strings.TrimSpace(right) - switch { - case left == "" && right == "": - return 0 - case left == "": - return -1 - case right == "": - return 1 - } - - leftParts, leftOK := numericVersion(left) - rightParts, rightOK := numericVersion(right) - if leftOK && rightOK { - maxParts := max(len(leftParts), len(rightParts)) - for i := 0; i < maxParts; i++ { - leftValue := -1 - if i < len(leftParts) { - leftValue = leftParts[i] - } - rightValue := -1 - if i < len(rightParts) { - rightValue = rightParts[i] - } - if leftValue < rightValue { - return -1 - } - if leftValue > rightValue { - return 1 - } - } - return 0 - } - return strings.Compare(strings.ToLower(left), strings.ToLower(right)) -} - -func numericVersion(version string) ([]int, bool) { - parts := strings.Split(version, ".") - if len(parts) < 2 || len(parts) > 4 { - return nil, false - } - values := make([]int, len(parts)) - for i, part := range parts { - value, err := strconv.Atoi(part) - if err != nil || value < 0 { - return nil, false - } - values[i] = value - } - return values, true -} diff --git a/task/versioning_test.go b/task/versioning_test.go index 5c0aebe4..d07c8d4e 100644 --- a/task/versioning_test.go +++ b/task/versioning_test.go @@ -26,7 +26,7 @@ func TestCompareVersionsMatchesDurableTaskRules(t *testing.T) { {"preview-b", "preview-a", 1}, } for _, test := range tests { - got := compareVersions(test.left, test.right) + got := helpers.CompareTaskVersions(test.left, test.right) if got < 0 { got = -1 } else if got > 0 { diff --git a/tests/durabletaskscheduler/dts_current_or_older_review_test.go b/tests/durabletaskscheduler/dts_current_or_older_review_test.go new file mode 100644 index 00000000..9e447994 --- /dev/null +++ b/tests/durabletaskscheduler/dts_current_or_older_review_test.go @@ -0,0 +1,101 @@ +package durabletaskscheduler_test + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/google/uuid" + "github.com/microsoft/durabletask-go/api" + durabletaskclient "github.com/microsoft/durabletask-go/client" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +func TestDTSCurrentOrOlderFiltersDoNotClaimNewerWork(t *testing.T) { + options := emulatorOptions(t) + mixed := "DTSReviewOlder-" + uuid.NewString() + legacy := mixed + "-legacy" + oldRegistry := task.NewTaskRegistry() + for _, version := range []string{"", "1.0", "2.0", "10.0"} { + require.NoError(t, oldRegistry.AddOrchestratorNVersion(mixed, version, func(*task.OrchestrationContext) (any, error) { + return "old:" + version, nil + })) + } + require.NoError(t, oldRegistry.AddOrchestratorN(legacy, func(*task.OrchestrationContext) (any, error) { + return "legacy", nil + })) + var abandoned atomic.Int32 + oldOptions := *options + oldOptions.Versioning = &task.VersioningOptions{ + Version: "2.0", DefaultVersion: "1.0", MatchStrategy: task.VersionMatchCurrentOrOlder, + } + oldOptions.UnaryInterceptors = []grpc.UnaryClientInterceptor{ + func(ctx context.Context, method string, request, reply any, connection *grpc.ClientConn, invoke grpc.UnaryInvoker, callOptions ...grpc.CallOption) error { + if method == protos.TaskHubSidecarService_AbandonTaskOrchestratorWorkItem_FullMethodName { + abandoned.Add(1) + } + return invoke(ctx, method, request, reply, connection, callOptions...) + }, + } + client, _ := startEmulatorWithOptions(t, &oldOptions, oldRegistry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + run := func(name, version, expected string) { + id := uniqueInstanceID("go-current-or-older") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithVersion(version)) + require.NoError(t, err) + result, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + var output string + require.NoError(t, result.ReadOutput(&output)) + require.Equal(t, expected, output) + } + for _, version := range []string{"", "1.0", "2.0"} { + run(mixed, version, "old:"+version) + } + for _, version := range []string{"", "1.0", "2.0"} { + run(legacy, version, "legacy") + } + defaultID := uniqueInstanceID("go-default-older") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{defaultID}) }) + _, err := client.ScheduleNewOrchestration(ctx, legacy, api.WithInstanceID(defaultID)) + require.NoError(t, err) + defaulted, err := client.WaitForOrchestrationCompletion(ctx, defaultID) + require.NoError(t, err) + require.Equal(t, "1.0", defaulted.Version) + var defaultOutput string + require.NoError(t, defaulted.ReadOutput(&defaultOutput)) + require.Equal(t, "legacy", defaultOutput) + var pending []api.InstanceID + for _, name := range []string{mixed, legacy} { + id := uniqueInstanceID("go-newer-worker") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id), api.WithVersion("10.0")) + require.NoError(t, err) + pending = append(pending, id) + } + require.Never(t, func() bool { return abandoned.Load() != 0 }, 500*time.Millisecond, 10*time.Millisecond) + + newOptions := *options + newOptions.Versioning = &task.VersioningOptions{Version: "10.0", MatchStrategy: task.VersionMatchStrict} + newRegistry := task.NewTaskRegistry() + for _, name := range []string{mixed, legacy} { + require.NoError(t, newRegistry.AddOrchestratorNVersion(name, "10.0", func(*task.OrchestrationContext) (any, error) { + return "new:10.0", nil + })) + } + startEmulatorWithOptions(t, &newOptions, newRegistry, durabletaskclient.WithAutoWorkItemFilters()) + for _, id := range pending { + result, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + var output string + require.NoError(t, result.ReadOutput(&output)) + require.Equal(t, "new:10.0", output) + } + require.Zero(t, abandoned.Load()) +} From 6deb2b304be694e99ae1ca5f0079acd16c1fe4fc Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:22:32 -0700 Subject: [PATCH 23/26] fix: reject unsafe rewind histories before replacement Reject additional durable events after an operation failure instead of removing handled failures required by retained work. Preserve the existing protocol for supported histories, surface worker rejection, and cover rewrite-to-replay plus unchanged rejected history on DTS. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 7 +- client/grpc_worker_processor.go | 1 + client/grpc_worker_rewind_test.go | 30 +++++ durabletaskscheduler/README.md | 12 +- task/rewind.go | 26 ++-- task/rewind_test.go | 119 +++++++++++++++--- tests/durabletaskscheduler/dts_rewind_test.go | 77 ++++++++++++ 7 files changed, 243 insertions(+), 29 deletions(-) diff --git a/README.md b/README.md index bc293327..5f95f205 100644 --- a/README.md +++ b/README.md @@ -346,9 +346,10 @@ The SDK does not expose skip-graceful-termination operations. enqueues recovery. Observe a **new execution ID** and its final status separately; an immediate completion wait can still return the previous failed execution. The worker replaces failed history using the same protocol as the Python SDK, -and DTS recursively rewinds failed children. Activity retry-policy histories -containing retry timers are not supported by the pinned Python rewind algorithm -or this implementation. See the [rewind sample](./samples/rewind) and +and DTS recursively rewinds failed children. The Go worker rejects replacement +when additional durable events follow a failed activity or child operation; +handled failures followed by more work, retry timers, and some concurrent +histories require a new orchestration instead. See the [rewind sample](./samples/rewind) and [rewind behavior and limitations](./durabletaskscheduler/README.md#rewind). To read a long history, use `StreamOrchestrationHistory`. This method reads the history one part at a time. If you buffer the history instead, the SDK applies a validated event cap. diff --git a/client/grpc_worker_processor.go b/client/grpc_worker_processor.go index f1882076..3e9ae4c8 100644 --- a/client/grpc_worker_processor.go +++ b/client/grpc_worker_processor.go @@ -247,6 +247,7 @@ func (w *TaskHubGrpcWorker) processOrchestration( } switch { case err != nil: + w.logger.Errorf("%s: orchestration execution failed: %v", request.InstanceId, err) response.Actions = []*protos.OrchestratorAction{helpers.NewCompleteOrchestrationAction( -1, protos.OrchestrationStatus_ORCHESTRATION_STATUS_FAILED, diff --git a/client/grpc_worker_rewind_test.go b/client/grpc_worker_rewind_test.go index 52d81ce9..b30e84aa 100644 --- a/client/grpc_worker_rewind_test.go +++ b/client/grpc_worker_rewind_test.go @@ -99,3 +99,33 @@ func TestWorkerRewindOversizedHistoryFailsExplicitly(t *testing.T) { require.Equal(t, string(api.ErrorTypeOrchestratorResponseTooLarge), completion.GetFailureDetails().GetErrorType()) require.True(t, completion.GetFailureDetails().GetIsNonRetriable()) } + +func TestWorkerRewindRejectsHandledFailureWithoutReplacement(t *testing.T) { + client := new(fakeSchedulerClient) + worker := newFakeWorker(t, client) + worker.processOrchestration(context.Background(), client, "token", &protos.OrchestratorRequest{ + InstanceId: "instance", + PastEvents: []*protos.HistoryEvent{ + helpers.NewExecutionStartedEvent("workflow", "instance", nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "handled", nil, nil, nil), + helpers.NewTaskFailedEvent(0, nil), + helpers.NewTaskScheduledEvent(1, "good", nil, nil, nil), + helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"kept"`)), + {EventType: &protos.HistoryEvent_ExecutionCompleted{ + ExecutionCompleted: &protos.ExecutionCompletedEvent{OrchestrationStatus: api.RUNTIME_STATUS_FAILED}, + }}, + }, + NewEvents: []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + {EventType: &protos.HistoryEvent_ExecutionRewound{ExecutionRewound: &protos.ExecutionRewoundEvent{}}}, + }, + }) + require.Zero(t, client.orchestrationAbandons) + require.Len(t, client.orchestrationCompletions, 1) + response := client.orchestrationCompletions[0] + require.Len(t, response.Actions, 1) + require.Nil(t, response.Actions[0].GetRewindOrchestration()) + completion := response.Actions[0].GetCompleteOrchestration() + require.Equal(t, api.RUNTIME_STATUS_FAILED, completion.GetOrchestrationStatus()) + require.Contains(t, completion.GetFailureDetails().GetErrorMessage(), "rewind cannot remove an operation failure") +} diff --git a/durabletaskscheduler/README.md b/durabletaskscheduler/README.md index 9d5bc0d9..15002a3d 100644 --- a/durabletaskscheduler/README.md +++ b/durabletaskscheduler/README.md @@ -179,9 +179,15 @@ status/output; `WaitForOrchestrationCompletion` can initially return the old failed execution. The [runnable sample](../samples/rewind) demonstrates this and cleans up only its own instance. -**Limitations:** the pinned Python algorithm retains timer events between -activity retry attempts. Rewinding an activity that used a retry policy can -therefore produce a replay nondeterminism error; that case is not supported. +**Limitations:** the Go worker conservatively rejects replacement history when +anything other than orchestration lifecycle markers follows the first failed +activity or child operation. This includes handled failures followed by more +work, retry timers, and some concurrent executions. Removing those failures +could prevent retained downstream work from replaying. The worker reports the +unsupported history in its logs without emitting replacement history; the +instance may retain its original failure metadata. Enqueue success does not +imply that this worker-side validation succeeded. This guard is not a +general proof of replay safety and does not infer whether a failure was handled. Rewind does not repair unchanged failing application code. Replacement history uses the normal large-payload handling and orchestration response-size bound; oversized responses fail explicitly rather than being silently truncated. diff --git a/task/rewind.go b/task/rewind.go index ba10beb2..8be8d6d5 100644 --- a/task/rewind.go +++ b/task/rewind.go @@ -29,21 +29,29 @@ func buildRewindResult(id api.InstanceID, oldEvents, newEvents []*protos.History if len(newEvents) != 2 || newEvents[0].GetOrchestratorStarted() == nil || newEvents[1].GetExecutionRewound() == nil { return nil, fmt.Errorf("rewind requires exactly two new events: orchestrator started and execution rewound") } + failedTasks := make(map[int32]struct{}) + failureSeen := false + for _, event := range oldEvents { + // ponytail: only lifecycle markers may follow one operation failure; + // broader recovery needs dependency-aware history selection. + if failureSeen && event.GetOrchestratorStarted() == nil && + event.GetOrchestratorCompleted() == nil && event.GetExecutionCompleted() == nil { + return nil, fmt.Errorf("%w: rewind cannot remove an operation failure followed by other durable events (event %d)", + api.ErrFeatureNotSupported, event.GetEventId()) + } + if failed := event.GetTaskFailed(); failed != nil { + failedTasks[failed.TaskScheduledId] = struct{}{} + failureSeen = true + } else if event.GetSubOrchestrationInstanceFailed() != nil { + failureSeen = true + } + } rewound := newEvents[1].GetExecutionRewound() executionID, err := uuid.NewRandom() if err != nil { return nil, fmt.Errorf("failed to generate rewind execution ID: %w", err) } - failedTasks := make(map[int32]struct{}) - for _, events := range [][]*protos.HistoryEvent{oldEvents, newEvents} { - for _, event := range events { - if failed := event.GetTaskFailed(); failed != nil { - failedTasks[failed.TaskScheduledId] = struct{}{} - } - } - } - history := make([]*protos.HistoryEvent, 0, len(oldEvents)+len(newEvents)) for _, events := range [][]*protos.HistoryEvent{oldEvents, newEvents} { for _, event := range events { diff --git a/task/rewind_test.go b/task/rewind_test.go index 40387607..646500ee 100644 --- a/task/rewind_test.go +++ b/task/rewind_test.go @@ -39,25 +39,19 @@ func TestRewindHistoryPreservesSuccessfulWork(t *testing.T) { start, helpers.NewTaskScheduledEvent(0, "good", nil, nil, nil), helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"kept"`)), - helpers.NewTaskScheduledEvent(1, "bad", nil, nil, nil), - helpers.NewTaskFailedEvent(1, &protos.TaskFailureDetails{ErrorMessage: "failed"}), - {EventId: 2, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ - SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{InstanceId: "failed-child", Name: "child"}, - }}, - {EventType: &protos.HistoryEvent_SubOrchestrationInstanceFailed{ - SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{TaskScheduledId: 2}, - }}, - {EventId: 3, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + {EventId: 1, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{InstanceId: "successful-child", Name: "child"}, }}, {EventType: &protos.HistoryEvent_SubOrchestrationInstanceCompleted{ - SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{TaskScheduledId: 3, Result: wrapperspb.String(`"child-kept"`)}, + SubOrchestrationInstanceCompleted: &protos.SubOrchestrationInstanceCompletedEvent{TaskScheduledId: 1, Result: wrapperspb.String(`"child-kept"`)}, }}, - helpers.NewTimerCreatedEvent(4, start.Timestamp), - {EventType: &protos.HistoryEvent_TimerFired{TimerFired: &protos.TimerFiredEvent{TimerId: 4}}}, + helpers.NewTimerCreatedEvent(2, start.Timestamp), + {EventType: &protos.HistoryEvent_TimerFired{TimerFired: &protos.TimerFiredEvent{TimerId: 2}}}, {EventType: &protos.HistoryEvent_GenericEvent{GenericEvent: &protos.GenericEvent{Data: wrapperspb.String("audit")}}}, - {EventType: &protos.HistoryEvent_OrchestratorCompleted{OrchestratorCompleted: &protos.OrchestratorCompletedEvent{}}}, rewindEvent(""), + helpers.NewTaskScheduledEvent(3, "bad", nil, nil, nil), + helpers.NewTaskFailedEvent(3, &protos.TaskFailureDetails{ErrorMessage: "failed"}), + {EventType: &protos.HistoryEvent_OrchestratorCompleted{OrchestratorCompleted: &protos.OrchestratorCompletedEvent{}}}, failedCompletionEvent(), } newEvents := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")} @@ -77,7 +71,9 @@ func TestRewindHistoryPreservesSuccessfulWork(t *testing.T) { action := result.Response.Actions[0] require.EqualValues(t, -1, action.Id) history := action.GetRewindOrchestration().GetNewHistory() - want := []*protos.HistoryEvent{old[0], proto.CloneOf(start), old[2], old[3], old[6], old[8], old[9], old[10], old[11], old[12], old[13], old[14], newEvents[0], newEvents[1]} + want := append([]*protos.HistoryEvent(nil), old[:10]...) + want[1] = proto.CloneOf(start) + want = append(want, old[12], newEvents[0], newEvents[1]) require.Len(t, history, len(want)) newID := history[1].GetExecutionStarted().GetOrchestrationInstance().GetExecutionId().GetValue() require.Len(t, newID, 32) @@ -170,3 +166,98 @@ func TestRewindMalformedRequests(t *testing.T) { require.False(t, isRewindRequest([]*protos.HistoryEvent{rewindEvent(""), failedCompletionEvent()}, nil)) require.False(t, isRewindRequest(nil, []*protos.HistoryEvent{rewindEvent("")})) } + +func TestRewindReplacementReplaysSuccessfulPrefix(t *testing.T) { + registry := NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("workflow", func(ctx *OrchestrationContext) (any, error) { + var good, recovered string + if err := ctx.CallActivity("good").Await(&good); err != nil { + return nil, err + } + if err := ctx.CallActivity("bad", WithActivityInput(good)).Await(&recovered); err != nil { + return nil, err + } + return good + ":" + recovered, nil + })) + executor := NewTaskExecutor(registry) + history := []*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("workflow", "instance", nil, nil, nil, nil), + helpers.NewTaskScheduledEvent(0, "good", nil, nil, nil), + helpers.NewTaskCompletedEvent(0, wrapperspb.String(`"kept"`)), + helpers.NewTaskScheduledEvent(1, "bad", nil, wrapperspb.String(`"kept"`), nil), + helpers.NewTaskFailedEvent(1, nil), failedCompletionEvent(), + } + rewrite, err := executor.ExecuteOrchestrator(context.Background(), "instance", history, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")}, nil) + require.NoError(t, err) + replacement := rewrite.Response.Actions[0].GetRewindOrchestration().GetNewHistory() + newEvents := []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")} + replay, err := executor.ExecuteOrchestrator(context.Background(), "instance", replacement, newEvents, nil) + require.NoError(t, err) + require.Len(t, replay.Response.Actions, 1) + retried := replay.Response.Actions[0] + require.EqualValues(t, 1, retried.Id) + require.Equal(t, "bad", retried.GetScheduleTask().GetName()) + require.Equal(t, `"kept"`, retried.GetScheduleTask().GetInput().GetValue()) + replacement = append(replacement, newEvents...) + replacement = append(replacement, helpers.NewTaskScheduledEvent(retried.Id, "bad", nil, retried.GetScheduleTask().Input, nil)) + completed, err := executor.ExecuteOrchestrator(context.Background(), "instance", replacement, + []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), helpers.NewTaskCompletedEvent(retried.Id, wrapperspb.String(`"recovered"`))}, nil) + require.NoError(t, err) + require.Len(t, completed.Response.Actions, 1) + require.Equal(t, `"kept:recovered"`, completed.Response.Actions[0].GetCompleteOrchestration().GetResult().GetValue()) +} + +func TestRewindRejectsHistoryAfterAFailure(t *testing.T) { + firstFailure := helpers.NewTaskFailedEvent(0, &protos.TaskFailureDetails{ErrorMessage: "handled"}) + for _, test := range []struct { + name string + events []*protos.HistoryEvent + }{ + {"handled activity", []*protos.HistoryEvent{ + helpers.NewTaskScheduledEvent(0, "handled", nil, nil, nil), firstFailure, + helpers.NewTaskScheduledEvent(1, "good", nil, nil, nil), helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"kept"`)), + helpers.NewTaskScheduledEvent(2, "terminal", nil, nil, nil), helpers.NewTaskFailedEvent(2, nil), + }}, + {"handled child", []*protos.HistoryEvent{ + {EventId: 0, EventType: &protos.HistoryEvent_SubOrchestrationInstanceCreated{ + SubOrchestrationInstanceCreated: &protos.SubOrchestrationInstanceCreatedEvent{Name: "child", InstanceId: "child"}, + }}, + {EventType: &protos.HistoryEvent_SubOrchestrationInstanceFailed{ + SubOrchestrationInstanceFailed: &protos.SubOrchestrationInstanceFailedEvent{TaskScheduledId: 0}, + }}, + helpers.NewTaskScheduledEvent(1, "good", nil, nil, nil), helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"kept"`)), + }}, + {"handled activity then local error", []*protos.HistoryEvent{ + helpers.NewTaskScheduledEvent(0, "handled", nil, nil, nil), firstFailure, + helpers.NewTaskScheduledEvent(1, "good", nil, nil, nil), helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"kept"`)), + }}, + {"concurrent success after failure", []*protos.HistoryEvent{ + helpers.NewTaskScheduledEvent(0, "bad", nil, nil, nil), helpers.NewTaskScheduledEvent(1, "good", nil, nil, nil), + firstFailure, helpers.NewTaskCompletedEvent(1, wrapperspb.String(`"kept"`)), + }}, + {"multiple failures", []*protos.HistoryEvent{ + helpers.NewTaskScheduledEvent(0, "bad", nil, nil, nil), helpers.NewTaskScheduledEvent(1, "also-bad", nil, nil, nil), + firstFailure, helpers.NewTaskFailedEvent(1, nil), + }}, + {"retry timer", []*protos.HistoryEvent{ + helpers.NewTaskScheduledEvent(0, "bad", nil, nil, nil), firstFailure, + helpers.NewTimerCreatedEvent(1, helpers.NewOrchestratorStartedEvent().Timestamp), + }}, + } { + t.Run(test.name, func(t *testing.T) { + history := append([]*protos.HistoryEvent{ + helpers.NewOrchestratorStartedEvent(), + helpers.NewExecutionStartedEvent("workflow", "instance", nil, nil, nil, nil), + }, test.events...) + history = append(history, failedCompletionEvent()) + original := proto.CloneOf(&protos.OrchestratorRequest{PastEvents: history}) + result, err := NewTaskExecutor(NewTaskRegistry()).ExecuteOrchestrator(context.Background(), "instance", + history, []*protos.HistoryEvent{helpers.NewOrchestratorStartedEvent(), rewindEvent("")}, nil) + require.ErrorIs(t, err, api.ErrFeatureNotSupported) + require.Nil(t, result) + require.True(t, proto.Equal(original, &protos.OrchestratorRequest{PastEvents: history})) + }) + } +} diff --git a/tests/durabletaskscheduler/dts_rewind_test.go b/tests/durabletaskscheduler/dts_rewind_test.go index 0c83eac2..9cefe13c 100644 --- a/tests/durabletaskscheduler/dts_rewind_test.go +++ b/tests/durabletaskscheduler/dts_rewind_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "sync/atomic" "testing" "time" @@ -13,8 +14,10 @@ import ( "github.com/microsoft/durabletask-go/api" durabletaskclient "github.com/microsoft/durabletask-go/client" "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" "github.com/microsoft/durabletask-go/task" "github.com/stretchr/testify/require" + "google.golang.org/grpc" ) func TestDTSRewindRecovery(t *testing.T) { @@ -142,6 +145,80 @@ func TestDTSRewindRecovery(t *testing.T) { } } +func TestDTSRewindRejectsHandledFailure(t *testing.T) { + name := "GoRewindHandled_" + uuid.NewString() + registry := task.NewTaskRegistry() + var calls atomic.Int32 + for _, activity := range []string{"handled", "good", "terminal"} { + require.NoError(t, registry.AddActivityN(name+activity, func(task.ActivityContext) (any, error) { + calls.Add(1) + if activity != "good" { + return nil, errors.New(activity) + } + return "kept", nil + })) + } + require.NoError(t, registry.AddOrchestratorN(name, func(ctx *task.OrchestrationContext) (any, error) { + if err := ctx.CallActivity(name + "handled").Await(nil); err == nil { + return nil, errors.New("expected the handled failure") + } + if err := ctx.CallActivity(name + "good").Await(nil); err != nil { + return nil, err + } + return nil, ctx.CallActivity(name + "terminal").Await(nil) + })) + rejected := make(chan struct{}, 1) + options := emulatorOptions(t) + options.UnaryInterceptors = []grpc.UnaryClientInterceptor{ + func(ctx context.Context, method string, request, reply any, connection *grpc.ClientConn, invoke grpc.UnaryInvoker, callOptions ...grpc.CallOption) error { + err := invoke(ctx, method, request, reply, connection, callOptions...) + if response, ok := request.(*protos.OrchestratorResponse); ok && err == nil { + for _, action := range response.Actions { + if strings.Contains(action.GetCompleteOrchestration().GetFailureDetails().GetErrorMessage(), "rewind cannot remove an operation failure") { + select { + case rejected <- struct{}{}: + default: + } + } + } + } + return err + }, + } + client, _ := startEmulatorWithOptions(t, options, registry, durabletaskclient.WithAutoWorkItemFilters()) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + id := uniqueInstanceID("go-rewind-handled") + t.Cleanup(func() { cleanupRewindInstances(t, client, []api.InstanceID{id}) }) + _, err := client.ScheduleNewOrchestration(ctx, name, api.WithInstanceID(id)) + require.NoError(t, err) + failed, err := client.WaitForOrchestrationCompletion(ctx, id) + require.NoError(t, err) + require.Equal(t, api.RUNTIME_STATUS_FAILED, failed.RuntimeStatus) + require.EqualValues(t, 3, calls.Load()) + require.NoError(t, client.RewindInstance(ctx, id)) + select { + case <-rejected: + case <-ctx.Done(): + t.Fatal("worker did not report the rejected rewind:", ctx.Err()) + } + history, err := client.GetOrchestrationHistory(ctx, id, api.HistoryQuery{}) + require.NoError(t, err) + require.Equal(t, failed.ExecutionID, history.ExecutionID, "unsupported history must not be replaced") + require.EqualValues(t, 3, calls.Load(), "no activity should rerun after rejection") + failureCount, successCount := 0, 0 + for _, event := range history.Events { + if event.TaskFailed != nil { + failureCount++ + } + if event.TaskCompleted != nil && event.TaskCompleted.SerializedResult == `"kept"` { + successCount++ + } + } + require.Equal(t, 2, failureCount) + require.Equal(t, 1, successCount) +} + func waitForRewindRecovery(ctx context.Context, client *durabletaskscheduler.Client, id api.InstanceID, oldExecution string) (*api.OrchestrationMetadata, error) { ctx, cancel := context.WithTimeout(ctx, 90*time.Second) defer cancel() From 674dcb6bc05907eff37b5ca6e2c26b61925cc355 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:22:32 -0700 Subject: [PATCH 24/26] fix: order sample cleanup and tolerate delayed timers Retain and confirm schedule creation before deletion, preserve worker lifetime for entity cleanup, drain before deleting payload storage, and bound Run cleanup while preserving errors. Validate logical timer deadlines rather than a fixed physical chunk count; add focused protocol failure and delay regressions. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- samples/entity/entity.go | 16 +- samples/entity/entity_test.go | 100 ++++++++++ samples/largepayloads/largepayloads.go | 10 +- samples/largepayloads/largepayloads_test.go | 183 ++++++++++++++++++ samples/scheduledtasks/scheduledtasks.go | 46 ++++- samples/scheduledtasks/scheduledtasks_test.go | 144 ++++++++++++++ samples/timers/timers.go | 35 ++-- samples/timers/timers_test.go | 84 ++++++++ samples/worker/worker.go | 31 +-- samples/worker/worker_test.go | 156 +++++++++++++++ 10 files changed, 766 insertions(+), 39 deletions(-) create mode 100644 samples/entity/entity_test.go create mode 100644 samples/scheduledtasks/scheduledtasks_test.go create mode 100644 samples/timers/timers_test.go create mode 100644 samples/worker/worker_test.go diff --git a/samples/entity/entity.go b/samples/entity/entity.go index ed7124a8..4b531e86 100644 --- a/samples/entity/entity.go +++ b/samples/entity/entity.go @@ -25,13 +25,13 @@ const ( var factoryStats = &factoryRecorder{} func main() { - if err := run(); err != nil { + if err := run(context.Background()); err != nil { log.Fatal(err) } fmt.Println("SAMPLE_OK entity") } -func run() (err error) { +func run(ctx context.Context) (err error) { registry := task.NewTaskRegistry() if err := registry.AddEntityN(counterEntityName, CounterEntity); err != nil { return err @@ -65,9 +65,10 @@ func run() (err error) { return err } - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() - app, err := dtssample.Start(ctx, registry) + // Entity delete signals still need a worker after the scenario times out. + app, err := dtssample.Start(context.WithoutCancel(ctx), registry) if err != nil { return err } @@ -86,7 +87,7 @@ func run() (err error) { startedID := dtssample.NewInstanceID("entity-started") ids := []api.InstanceID{counterWorkflowID, transferID, startedID} defer func() { - err = errors.Join(err, deleteEntities(app.Client, entities...), dtssample.Cleanup(app.Client, ids...), app.Shutdown()) + err = errors.Join(err, dtssample.Cleanup(app.Client, ids...), deleteEntities(app.Client, entities...), app.Shutdown()) }() if err := verifyRawCounterSignalsAndQueries(ctx, app.Client, counterID); err != nil { @@ -528,11 +529,16 @@ func deleteEntities(client *durabletaskscheduler.Client, entityIDs ...api.Entity ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() var failures []error + var signaled []api.EntityID for _, id := range entityIDs { if err := client.SignalEntity(ctx, id, "delete"); err != nil { failures = append(failures, fmt.Errorf("delete %s: %w", id, err)) continue } + signaled = append(signaled, id) + } + // Submit every delete before one entity's completion wait can exhaust the budget. + for _, id := range signaled { if _, err := waitForEntity(ctx, client, id, func(metadata *api.EntityMetadata) (bool, error) { return metadata == nil || !metadata.HasState, nil }); err != nil { diff --git a/samples/entity/entity_test.go b/samples/entity/entity_test.go new file mode 100644 index 00000000..4bd29f4b --- /dev/null +++ b/samples/entity/entity_test.go @@ -0,0 +1,100 @@ +package main + +import ( + "context" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" +) + +func TestEntityCleanupAfterScenarioDeadline(t *testing.T) { + var terminated, waited, purged, deleted, read atomic.Int32 + workerStarted := make(chan struct{}) + var workerCtx context.Context + server := grpc.NewServer( + grpc.UnaryInterceptor(func(ctx context.Context, request any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + switch request := request.(type) { + case *emptypb.Empty: + return &emptypb.Empty{}, nil + case *protos.SignalEntityRequest: + if request.Name != "delete" { + <-ctx.Done() + return nil, status.FromContextError(ctx.Err()).Err() + } + if purged.Load() != 3 { + t.Error("entities were deleted before owned orchestrations were cleaned up") + } + <-workerStarted + if workerCtx.Err() != nil { + t.Error("worker stopped before entity delete signals") + } + deleted.Add(1) + return &protos.SignalEntityResponse{}, nil + case *protos.GetInstanceRequest: + runtimeStatus := api.RUNTIME_STATUS_RUNNING + if info.FullMethod == protos.TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName { + waited.Add(1) + runtimeStatus = api.RUNTIME_STATUS_TERMINATED + } + return &protos.GetInstanceResponse{ + Exists: true, + OrchestrationState: &protos.OrchestrationState{ + InstanceId: request.InstanceId, OrchestrationStatus: runtimeStatus, + }, + }, nil + case *protos.TerminateRequest: + terminated.Add(1) + return &protos.TerminateResponse{}, nil + case *protos.PurgeInstancesRequest: + if waited.Load() != terminated.Load() { + t.Error("owned orchestration purged before termination completed") + } + purged.Add(1) + return &protos.PurgeInstancesResponse{DeletedInstanceCount: 1}, nil + case *protos.GetEntityRequest: + if deleted.Load() != 6 { + t.Error("entity delete wait began before all delete signals were submitted") + } + if read.Add(1) == 1 { + return nil, status.Error(codes.InvalidArgument, "injected entity delete wait failure") + } + return &protos.GetEntityResponse{}, nil + default: + return handler(ctx, request) + } + }), + grpc.StreamInterceptor(func(_ any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, _ grpc.StreamHandler) error { + workerCtx = stream.Context() + close(workerStarted) + <-stream.Context().Done() + return nil + }), + ) + protos.RegisterTaskHubSidecarServiceServer(server, &protos.UnimplementedTaskHubSidecarServiceServer{}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(server.Stop) + go func() { _ = server.Serve(listener) }() + t.Setenv(dtssample.ConnectionStringVariable, "Endpoint=http://"+listener.Addr().String()+";TaskHub=test;Authentication=None") + + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + err = run(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "injected entity delete wait failure") + require.EqualValues(t, 3, terminated.Load()) + require.EqualValues(t, 3, waited.Load()) + require.EqualValues(t, 3, purged.Load()) + require.EqualValues(t, 6, deleted.Load()) + require.EqualValues(t, 6, read.Load()) +} diff --git a/samples/largepayloads/largepayloads.go b/samples/largepayloads/largepayloads.go index 766b2829..9e362b6d 100644 --- a/samples/largepayloads/largepayloads.go +++ b/samples/largepayloads/largepayloads.go @@ -114,7 +114,15 @@ func run() (err error) { } var ownedIDs []api.InstanceID defer func() { - err = errors.Join(err, dtssample.Cleanup(app.Client, ownedIDs...), cleanupStorage(), app.Shutdown()) + cleanupErr := dtssample.Cleanup(app.Client, ownedIDs...) + if shutdownErr := app.Shutdown(); shutdownErr != nil { + // A shutdown timeout cancels processing without confirming that uploads drained. + err = errors.Join(err, cleanupErr, fmt.Errorf( + "storage container %s retained because worker shutdown did not confirm drain: %w", + settings.container, shutdownErr)) + return + } + err = errors.Join(err, cleanupErr, cleanupStorage()) }() content := strings.Repeat("large-payloads-sample-", 512) diff --git a/samples/largepayloads/largepayloads_test.go b/samples/largepayloads/largepayloads_test.go index 0edabf29..871a5066 100644 --- a/samples/largepayloads/largepayloads_test.go +++ b/samples/largepayloads/largepayloads_test.go @@ -3,16 +3,199 @@ package main import ( "bytes" "compress/gzip" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "io" + "net" "net/http" "net/http/httptest" "strconv" "strings" + "sync" + "sync/atomic" "testing" + "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/wrapperspb" ) +func TestStorageCleanupWaitsForAcceptedUploads(t *testing.T) { + for _, shutdownTimesOut := range []bool{false, true} { + name := "drained before deletion" + if shutdownTimesOut { + name = "shutdown timeout retains container" + } + t.Run(name, func(t *testing.T) { + uploadStarted := make(chan struct{}) + uploadFinished := make(chan struct{}) + releaseUpload := make(chan struct{}) + release := sync.OnceFunc(func() { close(releaseUpload) }) + var uploads, waits atomic.Int32 + var purged, completed, deleted atomic.Bool + storage := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodPut: + _, _ = io.Copy(io.Discard, r.Body) + if r.URL.Query().Get("restype") != "container" && uploads.Add(1) == 2 { + // The worker's output upload is accepted but not yet acknowledged. + close(uploadStarted) + <-releaseUpload + close(uploadFinished) + } + w.WriteHeader(http.StatusCreated) + case http.MethodDelete: + if !purged.Load() || !completed.Load() { + t.Error("storage deleted before owned orchestration cleanup and worker drain") + } + deleted.Store(true) + w.WriteHeader(http.StatusAccepted) + default: + t.Errorf("unexpected storage request: %s %s", r.Method, r.URL) + w.WriteHeader(http.StatusBadRequest) + } + })) + t.Cleanup(storage.Close) + t.Cleanup(release) + t.Setenv("AZURE_STORAGE_CONNECTION_STRING", + "AccountName=account;AccountKey=ZmFrZS1hY2NvdW50LWtleQ==;BlobEndpoint="+storage.URL+"/account") + t.Setenv("DTS_SAMPLE_ALLOW_INSECURE_STORAGE", "1") + + content := strings.Repeat("accepted-payload", 32) + digest := sha256.Sum256([]byte(content)) + input, err := json.Marshal(sampleInput{Content: content, SHA256: hex.EncodeToString(digest[:])}) + require.NoError(t, err) + items := make(chan *protos.WorkItem, 1) + intakeStopped := make(chan struct{}) + server := grpc.NewServer( + grpc.UnaryInterceptor(func(ctx context.Context, request any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + switch request := request.(type) { + case *emptypb.Empty: + return &emptypb.Empty{}, nil + case *protos.CreateInstanceRequest: + items <- &protos.WorkItem{ + CompletionToken: "upload", + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "ValidateLargePayload", + Input: wrapperspb.String(string(input)), + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: request.InstanceId}, + }}, + } + return &protos.CreateInstanceResponse{InstanceId: request.InstanceId}, nil + case *protos.GetInstanceRequest: + runtimeStatus := api.RUNTIME_STATUS_RUNNING + if info.FullMethod == protos.TaskHubSidecarService_WaitForInstanceCompletion_FullMethodName { + if waits.Add(1) == 1 { + select { + case <-uploadStarted: + return nil, status.Error(codes.InvalidArgument, "injected scenario failure") + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } + } + runtimeStatus = api.RUNTIME_STATUS_TERMINATED + } + return &protos.GetInstanceResponse{ + Exists: true, + OrchestrationState: &protos.OrchestrationState{ + InstanceId: request.InstanceId, OrchestrationStatus: runtimeStatus, + }, + }, nil + case *protos.TerminateRequest: + return &protos.TerminateResponse{}, nil + case *protos.PurgeInstancesRequest: + purged.Store(true) + return &protos.PurgeInstancesResponse{DeletedInstanceCount: 1}, nil + case *protos.ActivityResponse: + completed.Store(true) + return &protos.CompleteTaskResponse{}, nil + case *protos.AbandonActivityTaskRequest: + return &protos.AbandonActivityTaskResponse{}, nil + default: + return handler(ctx, request) + } + }), + grpc.StreamInterceptor(func(_ any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, _ grpc.StreamHandler) error { + select { + case item := <-items: + if err := stream.SendMsg(item); err != nil { + return err + } + case <-stream.Context().Done(): + return nil + } + <-stream.Context().Done() + close(intakeStopped) + return nil + }), + ) + protos.RegisterTaskHubSidecarServiceServer(server, &protos.UnimplementedTaskHubSidecarServiceServer{}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(server.Stop) + go func() { _ = server.Serve(listener) }() + t.Setenv(dtssample.ConnectionStringVariable, "Endpoint=http://"+listener.Addr().String()+";TaskHub=test;Authentication=None") + + done := make(chan error, 1) + go func() { + done <- run() + close(done) + }() + t.Cleanup(func() { + release() + select { + case <-done: + case <-time.After(35 * time.Second): + t.Error("sample did not finish after releasing its upload") + } + }) + select { + case <-intakeStopped: + case <-time.After(5 * time.Second): + t.Fatal("sample did not reach worker shutdown with an accepted upload") + } + require.True(t, purged.Load()) + require.False(t, deleted.Load(), "intake cancellation alone does not drain accepted uploads") + if !shutdownTimesOut { + release() + } + select { + case err := <-done: + require.ErrorContains(t, err, "injected scenario failure") + if shutdownTimesOut { + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "storage container dtgolarge") + require.ErrorContains(t, err, "retained because worker shutdown did not confirm drain") + require.False(t, deleted.Load()) + } else { + require.NotErrorIs(t, err, context.DeadlineExceeded) + require.True(t, completed.Load()) + require.True(t, deleted.Load()) + } + case <-time.After(35 * time.Second): + t.Fatal("sample did not bound worker shutdown") + } + release() + select { + case <-uploadFinished: + case <-time.After(time.Second): + t.Fatal("released upload did not finish") + } + }) + } +} + func TestStoredCompressionIsIndependentOfDownloadDecompression(t *testing.T) { const hash = "expected-payload-hash" content := []byte(strings.Repeat(hash, 10)) diff --git a/samples/scheduledtasks/scheduledtasks.go b/samples/scheduledtasks/scheduledtasks.go index 7418286d..128898ef 100644 --- a/samples/scheduledtasks/scheduledtasks.go +++ b/samples/scheduledtasks/scheduledtasks.go @@ -40,14 +40,14 @@ type scheduledOutput struct { } func main() { - if err := run(); err != nil { + if err := run(context.Background()); err != nil { log.Fatal(err) } fmt.Println("SAMPLE_OK scheduledtasks") } -func run() (err error) { - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute) +func run(ctx context.Context) (err error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) defer cancel() createdFrom := time.Now().UTC().Add(-time.Second) runID := string(dtssample.NewInstanceID("scheduledtasks")) @@ -63,17 +63,28 @@ func run() (err error) { if err := durabletaskscheduler.RegisterScheduledTasks(registry); err != nil { return err } - app, err := dtssample.Start(ctx, registry, durabletaskscheduler.WithScheduledTasks(), durabletaskclient.WithAutoWorkItemFilters()) + // Durable schedule deletion still needs a worker after the scenario times out. + app, err := dtssample.Start(context.WithoutCancel(ctx), registry, durabletaskscheduler.WithScheduledTasks(), durabletaskclient.WithAutoWorkItemFilters()) if err != nil { return err } var targetIDs []api.InstanceID var handle *durabletaskscheduler.ScheduleClient + creationConfirmed := false defer func() { var cleanupErr error if handle != nil { deleteCtx, stopDelete := context.WithTimeout(context.Background(), 15*time.Second) - cleanupErr = errors.Join(cleanupErr, handle.Delete(deleteCtx)) + if !creationConfirmed { + if confirmErr := waitForScheduleCreation(deleteCtx, handle); confirmErr != nil { + cleanupErr = fmt.Errorf("schedule %s creation outcome is unknown; cleanup incomplete: %w", scheduleID, confirmErr) + } else { + creationConfirmed = true + } + } + if creationConfirmed { + cleanupErr = errors.Join(cleanupErr, handle.Delete(deleteCtx)) + } stopDelete() } queryCtx, stopQuery := context.WithTimeout(context.Background(), 15*time.Second) @@ -86,7 +97,12 @@ func run() (err error) { }() schedules := app.Client.ScheduledTasks() - handle, err = schedules.Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ + // Retain ownership even if Create is accepted but its completion wait fails. + handle, err = schedules.GetScheduleClient(scheduleID) + if err != nil { + return err + } + err = handle.Create(ctx, durabletaskscheduler.ScheduleCreationOptions{ ScheduleID: scheduleID, OrchestrationName: scheduledTargetName, TypedOrchestrationInput: scheduledInput{Run: runID, Phase: "initial", FailFirst: true}, @@ -106,6 +122,7 @@ func run() (err error) { if err != nil { return err } + creationConfirmed = true description, err := schedules.Get(ctx, scheduleID) if err != nil { return err @@ -236,6 +253,23 @@ func run() (err error) { return err } +func waitForScheduleCreation(ctx context.Context, handle *durabletaskscheduler.ScheduleClient) error { + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + for { + // A unique owned ID must exist before Delete can safely follow its + // potentially still-queued Create operation. + if _, err := handle.Describe(ctx); !errors.Is(err, durabletaskscheduler.ErrScheduleNotFound) { + return err + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} + func scheduledTargetOrchestrator(ctx *task.OrchestrationContext) (any, error) { var input scheduledInput if err := ctx.GetInput(&input); err != nil { diff --git a/samples/scheduledtasks/scheduledtasks_test.go b/samples/scheduledtasks/scheduledtasks_test.go new file mode 100644 index 00000000..96aa4840 --- /dev/null +++ b/samples/scheduledtasks/scheduledtasks_test.go @@ -0,0 +1,144 @@ +package main + +import ( + "context" + "encoding/json" + "net" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/samples/internal/dtssample" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestAcceptedScheduleIsDeletedAfterScenarioDeadline(t *testing.T) { + for _, test := range []struct { + name string + deleteFails bool + lookupFails bool + }{ + {name: "late creation is deleted"}, + {name: "delete failure is preserved", deleteFails: true}, + {name: "unknown creation is reported", lookupFails: true}, + } { + t.Run(test.name, func(t *testing.T) { + var mu sync.Mutex + var operations []durabletaskscheduler.ScheduleOperationRequest + var createApplied, active bool + operationIDs := make(map[string]string) + workerStarted := make(chan struct{}) + var workerCtx context.Context + server := grpc.NewServer( + grpc.UnaryInterceptor(func(ctx context.Context, request any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + switch request := request.(type) { + case *emptypb.Empty: + return &emptypb.Empty{}, nil + case *protos.CreateInstanceRequest: + var operation durabletaskscheduler.ScheduleOperationRequest + if err := json.Unmarshal([]byte(request.GetInput().GetValue()), &operation); err != nil { + return nil, err + } + mu.Lock() + operations = append(operations, operation) + operationIDs[request.InstanceId] = operation.OperationName + if operation.OperationName == "delete" { + active = false + if !createApplied { + // A late create after deletion would leave an active schedule. + createApplied, active = true, true + } + } + mu.Unlock() + return &protos.CreateInstanceResponse{InstanceId: request.InstanceId}, nil + case *protos.GetEntityRequest: + if test.lookupFails { + return nil, status.Error(codes.InvalidArgument, "injected creation lookup failure") + } + mu.Lock() + defer mu.Unlock() + if !createApplied { + // The first read precedes execution of the queued creation. + createApplied, active = true, true + return &protos.GetEntityResponse{}, nil + } + return &protos.GetEntityResponse{Exists: active, Entity: &protos.EntityMetadata{ + InstanceId: request.InstanceId, SerializedState: wrapperspb.String(`{"Status":1}`), + }}, nil + case *protos.GetInstanceRequest: + mu.Lock() + operation := operationIDs[request.InstanceId] + mu.Unlock() + if operation == "CreateSchedule" { + // The durable create was accepted, but its caller cannot confirm it. + <-ctx.Done() + return nil, status.FromContextError(ctx.Err()).Err() + } + <-workerStarted + if workerCtx.Err() != nil { + t.Error("worker stopped before durable schedule deletion") + } + if test.deleteFails { + return nil, status.Error(codes.InvalidArgument, "injected delete wait failure") + } + return &protos.GetInstanceResponse{ + Exists: true, + OrchestrationState: &protos.OrchestrationState{ + InstanceId: request.InstanceId, OrchestrationStatus: api.RUNTIME_STATUS_COMPLETED, + }, + }, nil + case *protos.QueryInstancesRequest: + return &protos.QueryInstancesResponse{}, nil + default: + return handler(ctx, request) + } + }), + grpc.StreamInterceptor(func(_ any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, _ grpc.StreamHandler) error { + workerCtx = stream.Context() + close(workerStarted) + <-stream.Context().Done() + return nil + }), + ) + protos.RegisterTaskHubSidecarServiceServer(server, &protos.UnimplementedTaskHubSidecarServiceServer{}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(server.Stop) + go func() { _ = server.Serve(listener) }() + t.Setenv(dtssample.ConnectionStringVariable, "Endpoint=http://"+listener.Addr().String()+";TaskHub=test;Authentication=None") + + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + err = run(ctx) + require.ErrorIs(t, err, context.DeadlineExceeded) + switch { + case test.deleteFails: + require.ErrorContains(t, err, "injected delete wait failure") + case test.lookupFails: + require.ErrorContains(t, err, "injected creation lookup failure") + require.ErrorContains(t, err, "creation outcome") + default: + require.Equal(t, context.DeadlineExceeded.Error(), err.Error()) + } + mu.Lock() + defer mu.Unlock() + if test.lookupFails { + require.Len(t, operations, 1, "do not race an unconfirmed creation with deletion") + return + } + require.Len(t, operations, 2) + require.Equal(t, "CreateSchedule", operations[0].OperationName) + require.Equal(t, "delete", operations[1].OperationName) + require.Equal(t, operations[0].EntityID, operations[1].EntityID) + require.False(t, active, "a delayed creation must not reactivate the deleted schedule") + }) + } +} diff --git a/samples/timers/timers.go b/samples/timers/timers.go index b1a4f728..6d9788a3 100644 --- a/samples/timers/timers.go +++ b/samples/timers/timers.go @@ -20,7 +20,6 @@ import ( const ( maximumPhysicalTimerInterval = 2 * time.Second logicalTimerDelay = 5 * time.Second - expectedPhysicalTimers = 3 ) type timerInput struct { @@ -110,9 +109,6 @@ func run() (err error) { if !output.Deadline.Equal(output.StartedAt.Add(logicalTimerDelay)) { return fmt.Errorf("deadline = %s, want %s", output.Deadline, output.StartedAt.Add(logicalTimerDelay)) } - if output.FiredAt.Before(output.Deadline) { - return fmt.Errorf("timer fired at %s before logical deadline %s", output.FiredAt, output.Deadline) - } if output.FirstGUID == "" || output.SecondGUID == "" || output.FirstGUID == output.SecondGUID { return fmt.Errorf("unexpected deterministic GUIDs: first=%q second=%q", output.FirstGUID, output.SecondGUID) } @@ -124,23 +120,30 @@ func run() (err error) { if err != nil { return fmt.Errorf("failed to read timer orchestration history: %w", err) } - timerCreated, timerFired := 0, 0 - var lastTimerFireAt time.Time - for _, event := range history.Events { + return verifyTimerHistory(history.Events, output) +} + +func verifyTimerHistory(events []*api.HistoryEvent, output timerOutput) error { + if output.FiredAt.Before(output.Deadline) { + return fmt.Errorf("timer fired at %s before logical deadline %s", output.FiredAt, output.Deadline) + } + pending := make(map[int32]time.Time) + fired := 0 + for _, event := range events { switch event.Type { case api.HistoryEventTimerCreated: - timerCreated++ + pending[event.EventID] = event.TimerCreated.FireAt case api.HistoryEventTimerFired: - timerFired++ - lastTimerFireAt = event.TimerFired.FireAt + fireAt, ok := pending[event.TimerFired.TimerID] + if !ok || !fireAt.Equal(event.TimerFired.FireAt) { + return fmt.Errorf("timer %d fired without a matching creation record", event.TimerFired.TimerID) + } + delete(pending, event.TimerFired.TimerID) + fired++ } } - if timerCreated != expectedPhysicalTimers || timerFired != expectedPhysicalTimers { - return fmt.Errorf("physical timers created/fired = %d/%d, want %d/%d", - timerCreated, timerFired, expectedPhysicalTimers, expectedPhysicalTimers) - } - if !lastTimerFireAt.Equal(output.Deadline) { - return fmt.Errorf("last physical timer fire = %s, want logical deadline %s", lastTimerFireAt, output.Deadline) + if fired == 0 || len(pending) != 0 { + return fmt.Errorf("incomplete physical timer history: %d fired, %d pending", fired, len(pending)) } return nil } diff --git a/samples/timers/timers_test.go b/samples/timers/timers_test.go new file mode 100644 index 00000000..1ce502a1 --- /dev/null +++ b/samples/timers/timers_test.go @@ -0,0 +1,84 @@ +package main + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/internal/helpers" + "github.com/microsoft/durabletask-go/internal/historyconv" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestTimerSampleAcceptsLatePhysicalDelivery(t *testing.T) { + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("TimerOrchestrator", TimerOrchestrator)) + executor := task.NewTaskExecutor(registry, task.WithMaximumTimerInterval(maximumPhysicalTimerInterval)) + input, err := json.Marshal(timerInput{Delay: logicalTimerDelay}) + require.NoError(t, err) + start := time.Date(2026, 9, 14, 12, 0, 0, 0, time.UTC) + started := helpers.NewOrchestratorStartedEvent() + started.Timestamp = timestamppb.New(start) + history := []*protos.HistoryEvent{started, + helpers.NewExecutionStartedEvent("TimerOrchestrator", "instance", wrapperspb.String(string(input)), nil, nil, nil)} + first, err := executor.ExecuteOrchestrator(context.Background(), "instance", nil, history, nil) + require.NoError(t, err) + require.Len(t, first.Response.Actions, 1) + timer := first.Response.Actions[0] + require.Equal(t, start.Add(maximumPhysicalTimerInterval), timer.GetCreateTimer().GetFireAt().AsTime()) + history = append(history, helpers.NewTimerCreatedEvent(timer.Id, timer.GetCreateTimer().FireAt)) + + late := helpers.NewOrchestratorStartedEvent() + late.Timestamp = timestamppb.New(start.Add(6 * time.Second)) + events := []*protos.HistoryEvent{late, helpers.NewTimerFiredEvent(timer.Id, timer.GetCreateTimer().FireAt, nil)} + second, err := executor.ExecuteOrchestrator(context.Background(), "instance", history, events, nil) + require.NoError(t, err) + require.Len(t, second.Response.Actions, 1) + activity := second.Response.Actions[0] + require.Equal(t, "RecordStableObservation", activity.GetScheduleTask().GetName()) + history = append(history, events...) + history = append(history, helpers.NewTaskScheduledEvent(activity.Id, "RecordStableObservation", nil, activity.GetScheduleTask().Input, nil)) + events = []*protos.HistoryEvent{late, helpers.NewTaskCompletedEvent(activity.Id, activity.GetScheduleTask().Input)} + last, err := executor.ExecuteOrchestrator(context.Background(), "instance", history, events, nil) + require.NoError(t, err) + require.Len(t, last.Response.Actions, 1) + completion := last.Response.Actions[0].GetCompleteOrchestration() + require.NotNil(t, completion) + require.Equal(t, api.RUNTIME_STATUS_COMPLETED, completion.OrchestrationStatus) + var output timerOutput + require.NoError(t, json.Unmarshal([]byte(completion.GetResult().GetValue()), &output)) + require.Equal(t, start.Add(logicalTimerDelay), output.Deadline) + require.Equal(t, start.Add(6*time.Second), output.FiredAt) + + converter := historyconv.New(nil) + var records []*api.HistoryEvent + for _, event := range append(history, events...) { + record, err := converter.Convert(event) + require.NoError(t, err) + records = append(records, record) + } + require.NoError(t, verifyTimerHistory(records, output)) + output.FiredAt = output.Deadline.Add(-time.Second) + require.ErrorContains(t, verifyTimerHistory(records, output), "before logical deadline") +} + +func TestTimerHistoryRequiresMatchedRecords(t *testing.T) { + deadline := time.Unix(10, 0).UTC() + created := &api.HistoryEvent{Type: api.HistoryEventTimerCreated, EventID: 7, + TimerCreated: &api.HistoryTimerEvent{FireAt: deadline}} + fired := &api.HistoryEvent{Type: api.HistoryEventTimerFired, + TimerFired: &api.HistoryTimerFiredEvent{TimerID: 7, FireAt: deadline}} + output := timerOutput{Deadline: deadline, FiredAt: deadline} + require.NoError(t, verifyTimerHistory([]*api.HistoryEvent{created, fired}, output)) + for _, events := range [][]*api.HistoryEvent{nil, {created}, {fired}, {created, fired, fired}} { + require.Error(t, verifyTimerHistory(events, output)) + } + fired.TimerFired.FireAt = deadline.Add(time.Second) + require.Error(t, verifyTimerHistory([]*api.HistoryEvent{created, fired}, output)) +} diff --git a/samples/worker/worker.go b/samples/worker/worker.go index bce2beb7..18c1b895 100644 --- a/samples/worker/worker.go +++ b/samples/worker/worker.go @@ -90,7 +90,7 @@ func verifyStartConcurrencyAndDrain( registry *task.TaskRegistry, concurrencyIDs []api.InstanceID, drainID api.InstanceID, -) error { +) (err error) { activityGauge.Reset() worker, err := newSampleWorker(options, registry, 2) if err != nil { @@ -102,7 +102,7 @@ func verifyStartConcurrencyAndDrain( workerStopped := false defer func() { if !workerStopped { - shutdownWorker(worker) + err = errors.Join(err, shutdownWorker(worker)) } }() for _, id := range concurrencyIDs { @@ -165,7 +165,7 @@ func verifyStartConcurrencyAndDrain( if err := recovery.Start(ctx); err != nil { return err } - defer shutdownWorker(recovery) + defer func() { err = errors.Join(err, shutdownWorker(recovery)) }() if err := waitForWorkerOutput(ctx, client, drainID, "delay:"+string(drainID)); err != nil { return fmt.Errorf("post-shutdown work did not survive worker restart: %w", err) } @@ -179,7 +179,7 @@ func verifyRestart( client *durabletaskscheduler.Client, registry *task.TaskRegistry, id api.InstanceID, -) error { +) (err error) { worker, err := newSampleWorker(options, registry, 2) if err != nil { return err @@ -187,7 +187,7 @@ func verifyRestart( if err := worker.Start(ctx); err != nil { return err } - defer shutdownWorker(worker) + defer func() { err = errors.Join(err, shutdownWorker(worker)) }() if _, err := client.ScheduleNewOrchestration(ctx, "SampleWorkerEcho", api.WithInstanceID(id), api.WithInput("restart")); err != nil { return err @@ -205,7 +205,7 @@ func verifyRun( client *durabletaskscheduler.Client, registry *task.TaskRegistry, id api.InstanceID, -) error { +) (err error) { worker, err := newSampleWorker(options, registry, 2) if err != nil { return err @@ -215,9 +215,18 @@ func verifyRun( go func() { done <- worker.Run(runCtx) }() stopped := false defer func() { - if !stopped { - cancelRun() - <-done + cancelRun() + if stopped { + return + } + err = errors.Join(err, shutdownWorker(worker)) + waitCtx, stopWait := context.WithTimeout(context.Background(), 5*time.Second) + defer stopWait() + select { + case runErr := <-done: + err = errors.Join(err, runErr) + case <-waitCtx.Done(): + err = errors.Join(err, fmt.Errorf("wait for Run worker shutdown: %w", waitCtx.Err())) } }() if err := waitForWorkerRunning(ctx, worker); err != nil { @@ -287,10 +296,10 @@ func newSampleWorker( ) } -func shutdownWorker(worker *durabletaskclient.TaskHubGrpcWorker) { +func shutdownWorker(worker *durabletaskclient.TaskHubGrpcWorker) error { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - _ = worker.Shutdown(ctx) + return worker.Shutdown(ctx) } func workerEchoWorkflow(ctx *task.OrchestrationContext) (any, error) { diff --git a/samples/worker/worker_test.go b/samples/worker/worker_test.go new file mode 100644 index 00000000..53e11214 --- /dev/null +++ b/samples/worker/worker_test.go @@ -0,0 +1,156 @@ +package main + +import ( + "context" + "net" + "sync" + "testing" + "time" + + "github.com/microsoft/durabletask-go/api" + "github.com/microsoft/durabletask-go/durabletaskscheduler" + "github.com/microsoft/durabletask-go/internal/protos" + "github.com/microsoft/durabletask-go/task" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/emptypb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestVerifyRunBoundsCleanupOnEveryExit(t *testing.T) { + for _, test := range []struct { + name string + canceled bool + blocked bool + }{ + {name: "completed Run is not waited twice"}, + {name: "canceled before Run starts", canceled: true}, + {name: "accepted activity outlives cancellation", blocked: true}, + } { + t.Run(test.name, func(t *testing.T) { + workerStarted := make(chan struct{}) + started := make(chan struct{}) + finished := make(chan struct{}) + releaseActivity := make(chan struct{}) + release := sync.OnceFunc(func() { close(releaseActivity) }) + items := make(chan *protos.WorkItem, 1) + server := grpc.NewServer( + grpc.UnaryInterceptor(func(ctx context.Context, request any, _ *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (any, error) { + switch request := request.(type) { + case *emptypb.Empty: + return &emptypb.Empty{}, nil + case *protos.CreateInstanceRequest: + if test.blocked { + items <- &protos.WorkItem{ + CompletionToken: "activity", + Request: &protos.WorkItem_ActivityRequest{ActivityRequest: &protos.ActivityRequest{ + Name: "SampleWorkerEchoActivity", + Input: wrapperspb.String(`"run"`), + OrchestrationInstance: &protos.OrchestrationInstance{InstanceId: request.InstanceId}, + }}, + } + } + return &protos.CreateInstanceResponse{InstanceId: request.InstanceId}, nil + case *protos.GetInstanceRequest: + if test.blocked { + select { + case <-started: + return nil, status.Error(codes.InvalidArgument, "injected completion wait failure") + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } + } + select { + case <-workerStarted: + case <-ctx.Done(): + return nil, status.FromContextError(ctx.Err()).Err() + } + return &protos.GetInstanceResponse{ + Exists: true, + OrchestrationState: &protos.OrchestrationState{ + InstanceId: request.InstanceId, OrchestrationStatus: api.RUNTIME_STATUS_COMPLETED, + Output: wrapperspb.String(`"echo:run"`), + }, + }, nil + case *protos.ActivityResponse: + return &protos.CompleteTaskResponse{}, nil + case *protos.AbandonActivityTaskRequest: + return &protos.AbandonActivityTaskResponse{}, nil + default: + return handler(ctx, request) + } + }), + grpc.StreamInterceptor(func(_ any, stream grpc.ServerStream, _ *grpc.StreamServerInfo, _ grpc.StreamHandler) error { + close(workerStarted) + select { + case item := <-items: + if err := stream.SendMsg(item); err != nil { + return err + } + case <-stream.Context().Done(): + return nil + } + <-stream.Context().Done() + return nil + }), + ) + protos.RegisterTaskHubSidecarServiceServer(server, &protos.UnimplementedTaskHubSidecarServiceServer{}) + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(server.Stop) + t.Cleanup(release) + go func() { _ = server.Serve(listener) }() + options, err := durabletaskscheduler.NewOptionsFromConnectionString( + "Endpoint=http://" + listener.Addr().String() + ";TaskHub=test;Authentication=None") + require.NoError(t, err) + client, err := durabletaskscheduler.NewClient(t.Context(), options, api.DefaultLogger()) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, client.Close()) }) + registry := task.NewTaskRegistry() + require.NoError(t, registry.AddOrchestratorN("SampleWorkerEcho", workerEchoWorkflow)) + require.NoError(t, registry.AddActivityN("SampleWorkerEchoActivity", func(task.ActivityContext) (any, error) { + close(started) + defer close(finished) + // Deliberately ignore cancellation to exercise the bounded final Run wait. + <-releaseActivity + return "echo:run", nil + })) + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if test.canceled { + cancel() + } + done := make(chan error, 1) + go func() { done <- verifyRun(ctx, options, client, registry, "run-test") }() + select { + case err := <-done: + switch { + case test.canceled: + require.ErrorIs(t, err, context.Canceled) + case test.blocked: + require.ErrorContains(t, err, "injected completion wait failure") + require.ErrorIs(t, err, context.DeadlineExceeded) + require.ErrorContains(t, err, "wait for Run worker shutdown") + select { + case <-finished: + t.Fatal("activity should remain blocked until explicitly released") + default: + } + release() + select { + case <-finished: + case <-time.After(time.Second): + t.Fatal("released activity did not finish") + } + default: + require.NoError(t, err) + } + case <-time.After(15 * time.Second): + t.Fatal("verifyRun did not bound its cleanup wait") + } + }) + } +} From 69cc5bd84e61f1114172c72fc44b49481a9d0ee7 Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:32:47 -0700 Subject: [PATCH 25/26] docs: mark the Go SDK as beta Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 5f95f205..15766348 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ [![Build](https://github.com/microsoft/durabletask-go/actions/workflows/pr-validation.yml/badge.svg)](https://github.com/microsoft/durabletask-go/actions/workflows/pr-validation.yml) +> This SDK is currently in **beta**. + This SDK lets you write reliable business logic in Go. You write the logic as normal Go code. The SDK calls this logic an *orchestration*. You write the orchestrations, activities, and entities. [Azure Durable Task Scheduler](https://learn.microsoft.com/azure/azure-functions/durable/durable-task-scheduler/durable-task-scheduler) (DTS) keeps the durable state. DTS also dispatches the work and recovers the work after a failure. From a63a25c1d885dda9ef02e8d32eb2aacecfd42b6e Mon Sep 17 00:00:00 2001 From: Tomer Rosenthal <17064840+torosent@users.noreply.github.com> Date: Mon, 14 Sep 2026 09:34:44 -0700 Subject: [PATCH 26/26] docs: remove numbered feature group wording Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- README.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 15766348..c0207767 100644 --- a/README.md +++ b/README.md @@ -17,9 +17,7 @@ The SDK gives you two connections: ## Features -The features are in three groups. Group 1 has the most important features. Start with group 1. - -### Group 1: Basic features +### Basic features You need these features for almost all applications. @@ -30,7 +28,7 @@ You need these features for almost all applications. | Retries and failure handling | Retry a failed task with a policy. Read the cause from typed errors. | | Orchestration management | Start, query, wait for, terminate, suspend, resume, and purge an orchestration. | -### Group 2: Common patterns +### Common patterns Most applications use one or more of these patterns. @@ -42,7 +40,7 @@ Most applications use one or more of these patterns. | Sub-orchestrations | Call one orchestration from a different orchestration. | | Durable entities | Keep addressable state. An entity does its operations one at a time. | -### Group 3: Advanced features +### Advanced features Use these features for large systems or for special conditions.