diff --git a/.github/workflows/deploy-preview.yml b/.github/workflows/deploy-preview.yml index 42539e3d..723ac7e0 100644 --- a/.github/workflows/deploy-preview.yml +++ b/.github/workflows/deploy-preview.yml @@ -51,22 +51,6 @@ jobs: - name: Build Docker Image run: docker build -t "$TAG_BASE:$CHANNEL" --build-arg GIT_SHA=${{ github.sha }} --build-arg DOCKER_CHANNEL=$CHANNEL . - # we need to get from/to timestamp because hyperdx by default only shows the last 15 mins logs, - # so if a commit preview has been pushed more than 15 mins ago relative to whenever preview link - # gets opened by someone, it wont show the logs unless user manually increases the timeframe - # from hyperdx gui, so we set the 'from' and 'to' timestamps in the link's query parameters to - # ensure the link always opens within the exact timeframe when the job was executed - - name: Set Preview Link From Timestamp - run: | - # set from timestamps in envs - FROM_TIME=$(date +%s) - - # set in github envs as milliseconds - echo "FROM_TIME=${FROM_TIME}000" >> $GITHUB_ENV - - # log the timestamp for debug purposes - echo from timestamp: $FROM_TIME \($(date -d @$FROM_TIME)\) - - name: Create GitHub Deployment id: deployment uses: bobheadxi/deployments@v1 @@ -82,7 +66,6 @@ jobs: echo "PREVIEW_ROUNDS=${{ vars.PREVIEW_ROUNDS }}" >> .env echo "TRACER_SERVICE_NAME=github-preview-${{ github.sha }}" >> .env echo "RPC_URL=${{ secrets.PREVIEW_RPC_URL }}" >> .env - echo "HYPERDX_API_KEY=${{ secrets.PREVIEW_HYPERDX_API_KEY }}" >> .env echo "BOT_WALLET_PRIVATEKEY=${{ secrets.PREVIEW_BOT_WALLET_PRIVATEKEY }}" >> .env echo "MAX_RATIO=${{ vars.PREVIEW_MAX_RATIO }}" >> .env echo "SUBGRAPH=${{ vars.PREVIEW_SUBGRAPH }}" >> .env @@ -99,26 +82,15 @@ jobs: - name: Run Rain Solver Docker Image run: docker run --rm --env-file .env $TAG_BASE:$CHANNEL - # set to timestamp after bot execution has completed - - name: Set Preview Link To Timestamps - run: | - # set to timestamps in envs - TO_TIME=$(date +%s) - - # set in github envs as milliseconds - echo "TO_TIME=${TO_TIME}000" >> $GITHUB_ENV - - # log the timestamp for debug purposes - echo to timestamp: $TO_TIME \($(date -d @$TO_TIME)\) - - name: Update Deployment Status + if: always() && steps.deployment.outputs.deployment_id != '' uses: bobheadxi/deployments@v1 with: step: finish token: ${{ secrets.GITHUB_TOKEN }} env: preview - status: success + status: ${{ job.status }} ref: ${{ github.sha }} override: false deployment_id: ${{ steps.deployment.outputs.deployment_id }} - env_url: https://www.hyperdx.io/search?q=%28%28service%3A%22github-preview-${{ github.sha }}%22%29%29&from=${{ env.FROM_TIME }}&to=${{ env.TO_TIME }} + env_url: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} diff --git a/DiagOrder.md b/DiagOrder.md index 57c4ecd3..d7c68057 100644 --- a/DiagOrder.md +++ b/DiagOrder.md @@ -4,7 +4,7 @@ - add the block number at which the debugging should take place. - replace the `from` address with the transaction sender address, ie msg.sender. - replace the `to` address with the arb contract address on the desired network. -- replace the `data` with the calldata without leading 0x taken from otel (hyperdx). +- replace the `data` with the calldata without leading 0x taken from the trace in VictoriaTraces. - save the file and now you can run the following command to get the traces: ```bash forge script diag/DiagOrder.sol:DiagOrder -vvvvv diff --git a/README.md b/README.md index 65aadb32..880fb84e 100644 --- a/README.md +++ b/README.md @@ -97,13 +97,25 @@ CONFIG= # Git branch to track for docker compose DOCKER_CHANNEL=master -# api key for heyperDx platfomr to send spans to, if not set will send traces to localhost -HYPERDX_API_KEY="" +# Full OTLP HTTP trace URL. Leave empty to print spans to the console. +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="" # trace/spans service name, defaults to "rain-solver" if not set TRACER_SERVICE_NAME="" ``` +## Tracing + +The solver sends gzip-compressed OTLP HTTP traces when +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` or `OTEL_EXPORTER_OTLP_ENDPOINT` is set. +The trace-specific endpoint takes precedence and must include the full path. +The generic endpoint is a base URL; the OpenTelemetry SDK appends `/v1/traces`. +Without either endpoint, spans go to the console, including in GitHub previews. +No telemetry credential is required for the private VictoriaTraces endpoint. + +For the Base solver deployment and verification steps, see +[the VictoriaTraces runbook](VictoriaTraces.md). + ## List of supported dexes as external liquidity (decentralized exchanges) - `BalancerV3` - `Stabull` diff --git a/VictoriaTraces.md b/VictoriaTraces.md new file mode 100644 index 00000000..030a2cdf --- /dev/null +++ b/VictoriaTraces.md @@ -0,0 +1,47 @@ +# Base solver tracing + +The solver records round results, order diagnostics, events, and exceptions as +OpenTelemetry spans. VictoriaTraces stores these spans. This change does not +forward Docker stdout to VictoriaLogs or create alert rules. + +## Rollout + +1. Apply the `rain.devops` tailnet policy granting `tag:base-node` access to + `tag:rain-infra` on TCP 10428. VictoriaTraces already runs on the Rain + observability node. Keep this port private to the tailnet. +2. Build and publish a solver image containing this change through the existing + release process. +3. On `base-node`, update the environment used to create `base-solver`: + + ```dotenv + OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://rain-management-observability.taile5cf8a.ts.net:10428/insert/opentelemetry/v1/traces + TRACER_SERVICE_NAME=base-bot + ``` + + Remove the previous telemetry API key from the runtime environment. Preserve + the signer credentials, `CONFIG=./config.yml`, and the config bind mount from + `/root/solver/config.yml` to `/rain-solver/config.yml`. +4. Recreate the container with the new image and environment through its existing + deployment process. A restart alone does not update Docker environment values. +5. Verify DNS resolution and TCP 10428 reachability from inside the container, + not just from the host. Check Docker logs for exporter errors. +6. Open [Rain Grafana](https://rain-management-observability.taile5cf8a.ts.net), + choose Explore and the `victoriatraces` Jaeger datasource, then search for + service `base-bot` over the last 15 minutes. Confirm fresh round and order + spans arrive, including attributes, events, exceptions, and child spans. + +After verification, remove the obsolete telemetry ingestion secrets from GitHub +and the deployment secret store. GitHub previews now print spans in the Actions +run logs, linked from the preview deployment. + +## Endpoint configuration + +`OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` is the full trace ingestion URL and takes +precedence over `OTEL_EXPORTER_OTLP_ENDPOINT`. The generic setting is a base URL; +the SDK appends `/v1/traces`. The exporter sends gzip-compressed OTLP HTTP JSON. +Standard OTLP header environment settings remain available for other receivers. +No API key is needed for the private VictoriaTraces endpoint. + +If both endpoint settings are absent, spans print to the console. For a temporary +fallback, remove both settings and recreate the container. Inspect its Docker +logs until trace ingestion is restored. diff --git a/example.env b/example.env index b4f85975..054ef3aa 100644 --- a/example.env +++ b/example.env @@ -4,10 +4,10 @@ CONFIG= # Git branch to track for docker compose DOCKER_CHANNEL=master -# api key for heyperDx platfomr to send spans to, if not set will send traces to localhost -HYPERDX_API_KEY="" +# Full OTLP HTTP trace URL. Leave empty to print spans to the console. +OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="" -# trace/spans service name, defaults to "arb-bot" if not set +# trace/spans service name, defaults to "rain-solver" if not set TRACER_SERVICE_NAME="" # test rpcs vars diff --git a/src/logger/index.ts b/src/logger/index.ts index babbe1c1..41e08885 100644 --- a/src/logger/index.ts +++ b/src/logger/index.ts @@ -1,5 +1,4 @@ -import { sleep } from "../common"; -import { OrderSpanEvents } from "../core/types"; +import type { OrderSpanEvents } from "../core/types"; import { Resource } from "@opentelemetry/resources"; import { CompressionAlgorithm } from "@opentelemetry/otlp-exporter-base"; import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http"; @@ -46,13 +45,14 @@ export type SpanWithContext = { * * @remarks * - Uses OpenTelemetry SDK for Node.js. - * - Supports exporting traces to HyperDX if the API key is provided. + * - Exports traces to the configured OTLP HTTP endpoint. * - Automatically configures the service name from the `TRACER_SERVICE_NAME` * environment variable or defaults to "rain-solver". */ export class RainSolverLogger { tracer: Tracer; exporter: OTLPTraceExporter | ConsoleSpanExporter; + private readonly provider: BasicTracerProvider; constructor() { // enable diag @@ -64,12 +64,13 @@ export class RainSolverLogger { }), }); - if (process.env.HYPERDX_API_KEY) { + this.provider = provider; + + if ( + process.env.OTEL_EXPORTER_OTLP_TRACES_ENDPOINT || + process.env.OTEL_EXPORTER_OTLP_ENDPOINT + ) { const exporter = new OTLPTraceExporter({ - url: "https://in-otel.hyperdx.io/v1/traces", - headers: { - authorization: process?.env?.HYPERDX_API_KEY, - }, compression: CompressionAlgorithm.GZIP, }); this.exporter = exporter; @@ -163,8 +164,7 @@ export class RainSolverLogger { */ async shutdown() { // flush and close the connection - await this.exporter.shutdown(); - await sleep(3000); + await this.provider.shutdown(); } } diff --git a/src/logger/otlp.test.ts b/src/logger/otlp.test.ts new file mode 100644 index 00000000..948bac17 --- /dev/null +++ b/src/logger/otlp.test.ts @@ -0,0 +1,91 @@ +import { createServer } from "node:http"; +import { gunzipSync } from "node:zlib"; +import { context, trace, propagation, SpanStatusCode } from "@opentelemetry/api"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { PreAssembledSpan, RainSolverLogger } from "."; + +describe("OTLP trace export", () => { + beforeEach(() => { + trace.disable(); + context.disable(); + propagation.disable(); + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", ""); + vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", ""); + vi.stubEnv("OTEL_EXPORTER_OTLP_HEADERS", ""); + vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", ""); + vi.stubEnv("TRACER_SERVICE_NAME", "base-bot"); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + trace.disable(); + context.disable(); + propagation.disable(); + }); + + it.each(["trace-specific", "generic"])( + "exports queued diagnostics on shutdown using the %s endpoint", + async (setting) => { + const requests: { url?: string; encoding?: string; body: any }[] = []; + const server = createServer(async (req, res) => { + const chunks: Buffer[] = []; + for await (const chunk of req) chunks.push(Buffer.from(chunk)); + requests.push({ + url: req.url, + encoding: req.headers["content-encoding"], + body: JSON.parse(gunzipSync(Buffer.concat(chunks)).toString()), + }); + res.writeHead(200, { "Content-Type": "application/json" }); + res.end("{}"); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("Missing server port"); + const base = `http://127.0.0.1:${address.port}/insert/opentelemetry`; + if (setting === "trace-specific") { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://127.0.0.1:1/unused"); + vi.stubEnv("OTEL_EXPORTER_OTLP_TRACES_ENDPOINT", `${base}/v1/traces`); + } else { + vi.stubEnv("OTEL_EXPORTER_OTLP_ENDPOINT", base); + } + const logger = new RainSolverLogger(); + try { + const child = new PreAssembledSpan("order") + .setAttr("order.id", "test-order") + .addEvent("quote", { duration: 12 }) + .recordException("quote failed") + .setStatus({ code: SpanStatusCode.ERROR, message: "quote failed" }) + .end(); + logger.exportPreAssembledSpan(new PreAssembledSpan("round").addChild(child).end()); + await logger.shutdown(); + expect(requests).toHaveLength(1); + expect(requests[0].url).toBe("/insert/opentelemetry/v1/traces"); + expect(requests[0].encoding).toBe("gzip"); + const resource = requests[0].body.resourceSpans[0]; + expect(resource.resource.attributes).toContainEqual({ + key: "service.name", + value: { stringValue: "base-bot" }, + }); + const spans = resource.scopeSpans[0].spans; + const order = spans.find((span: any) => span.name === "order"); + const round = spans.find((span: any) => span.name === "round"); + expect(order.traceId).toBe(round.traceId); + expect(order.parentSpanId).toBe(round.spanId); + expect(order.attributes).toContainEqual({ + key: "order.id", + value: { stringValue: "test-order" }, + }); + expect(order.events.map((event: any) => event.name)).toEqual([ + "quote", + "exception", + ]); + expect(order.status.code).toBe(SpanStatusCode.ERROR); + } finally { + await logger.shutdown(); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + } + }, + ); +});