From a4dfb421ee20d8f0a02b452910f51f1d7d12da95 Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Fri, 11 Sep 2026 11:51:38 +0800 Subject: [PATCH 1/5] feat: gRPC/HTTP collector TLS and mTLS Add SW_AGENT_FORCE_TLS and SSL CA/cert/key paths; share tls helpers across gRPC and HTTP; convert PKCS#1 keys to PKCS#8 for HTTP stacks that reject PKCS#1. Follow symlinks so Kubernetes secret mounts work. Never raise TLS misconfig into the host process: warn and degrade (plaintext / system trust / one-way TLS), including OSError from SSLContext load races and path expanduser/resolve failures. Validate CA and client PEMs when building material so bad content does not defer failure to connect time; FORCE_TLS fallback never attaches client certs, and credential build failure may warn and stay plaintext rather than abort start. Keep HTTP mTLS temp PEMs fork-safe via register_at_fork rebind. Drop test-only ssl_target_name_override; TLS peer-name follows grpc.default_authority. Generate e2e PEMs via shared gen-e2e-tls-certs.sh in digest-pinned alpine/openssl (no apk; PEMs not committed). mTLS e2e healthchecks the sharing-server port. --- .gitattributes | 7 + .github/workflows/CI.yaml | 4 + .gitignore | 7 +- .licenserc.yaml | 1 + docs/en/setup/Configuration.md | 5 +- docs/en/setup/Intrusive.md | 25 +- skywalking/client/http.py | 10 +- skywalking/client/http_aio.py | 20 +- skywalking/config.py | 23 +- skywalking/utils/grpc_channel.py | 1359 +++++++-------- skywalking/utils/tls.py | 678 ++++++++ .../expected/dependency-services-consumer.yml | 52 + .../expected/dependency-services-provider.yml | 39 + tests/e2e/case/expected/layer.yml | 18 + .../expected/service-endpoint-consumer.yml | 19 + .../expected/service-instance-consumer.yml | 39 + .../grpc/mtls/docker-compose.grpc.mtls.yaml | 98 ++ tests/e2e/case/grpc/mtls/e2e.yaml | 54 + .../grpc/ssl/docker-compose.grpc.ssl.yaml | 86 + tests/e2e/case/grpc/ssl/e2e.yaml | 54 + tests/e2e/case/tls-depth-cases.yaml | 61 + tests/e2e/script/prepare/gen-e2e-tls-certs.sh | 120 ++ .../plugin/http/sw_httpx/services/provider.py | 64 +- tests/unit/test_grpc_channel.py | 1465 +++++++++-------- tests/unit/test_grpc_ready_gate.py | 334 ++-- tests/unit/test_shutdown_queue.py | 754 ++++----- tests/unit/test_tls.py | 602 +++++++ 27 files changed, 3997 insertions(+), 2001 deletions(-) create mode 100644 .gitattributes create mode 100644 skywalking/utils/tls.py create mode 100644 tests/e2e/case/expected/dependency-services-consumer.yml create mode 100644 tests/e2e/case/expected/dependency-services-provider.yml create mode 100644 tests/e2e/case/expected/layer.yml create mode 100644 tests/e2e/case/expected/service-endpoint-consumer.yml create mode 100644 tests/e2e/case/expected/service-instance-consumer.yml create mode 100644 tests/e2e/case/grpc/mtls/docker-compose.grpc.mtls.yaml create mode 100644 tests/e2e/case/grpc/mtls/e2e.yaml create mode 100644 tests/e2e/case/grpc/ssl/docker-compose.grpc.ssl.yaml create mode 100644 tests/e2e/case/grpc/ssl/e2e.yaml create mode 100644 tests/e2e/case/tls-depth-cases.yaml create mode 100644 tests/e2e/script/prepare/gen-e2e-tls-certs.sh create mode 100644 tests/unit/test_tls.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..384f64f14 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,7 @@ +# Force LF for shell scripts so bind-mounted scripts in Linux containers +# are not broken by CRLF (e.g. ``set -eu\r``). +*.sh text eol=lf + +# Keep Python sources LF so `git diff --check` does not treat CR as trailing +# whitespace on Windows checkouts (core.autocrlf). +*.py text eol=lf diff --git a/.github/workflows/CI.yaml b/.github/workflows/CI.yaml index 2f2e95699..5bcd070c0 100644 --- a/.github/workflows/CI.yaml +++ b/.github/workflows/CI.yaml @@ -254,6 +254,10 @@ jobs: case: - name: gRPC-single-process path: tests/e2e/case/grpc/single/e2e.yaml + - name: gRPC-ssl + path: tests/e2e/case/grpc/ssl/e2e.yaml + - name: gRPC-mtls + path: tests/e2e/case/grpc/mtls/e2e.yaml - name: gRPC-gunicorn path: tests/e2e/case/grpc/gunicorn/e2e.yaml # - name: gRPC-uwsgi diff --git a/.gitignore b/.gitignore index a2483bc4c..7a9833872 100644 --- a/.gitignore +++ b/.gitignore @@ -31,4 +31,9 @@ dist/ **/.venv/ tests/**/requirements.txt .vscode/ -skywalking/protocol \ No newline at end of file +skywalking/protocol +# Ephemeral e2e TLS material (generated by compose gen-certs / gen-e2e-tls-certs.sh) +tests/e2e/case/grpc/ssl/ca/ +tests/e2e/case/grpc/ssl/certs/ +tests/e2e/case/grpc/mtls/client/ +tests/e2e/case/grpc/mtls/server/ \ No newline at end of file diff --git a/.licenserc.yaml b/.licenserc.yaml index 4c2d37470..4920e1669 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -29,6 +29,7 @@ header: - 'NOTICE' - '.github/PULL_REQUEST_TEMPLATE' - '.gitignore' + - '.gitattributes' - '**/*.json' - '.venv' - 'poetry.lock' diff --git a/docs/en/setup/Configuration.md b/docs/en/setup/Configuration.md index 7d521e4a2..d938bdaaa 100644 --- a/docs/en/setup/Configuration.md +++ b/docs/en/setup/Configuration.md @@ -29,7 +29,10 @@ export SW_AGENT_YourConfiguration=YourValue | kafka_topic_log | SW_KAFKA_TOPIC_LOG | | skywalking-logs | Specifying Kafka topic name for Log data, this should be in sync with OAP | | kafka_topic_meter | SW_KAFKA_TOPIC_METER | | skywalking-meters | Specifying Kafka topic name for Meter data, this should be in sync with OAP | | kafka_reporter_custom_configurations | SW_KAFKA_REPORTER_CUSTOM_CONFIGURATIONS | | | The configs to init KafkaProducer, supports the basic arguments (whose type is either `str`, `bool`, or `int`) listed [here](https://kafka-python.readthedocs.io/en/master/apidoc/KafkaProducer.html#kafka.KafkaProducer) This config only works from env variables, each one should be passed in `SW_KAFKA_REPORTER_CONFIG_` | -| agent_force_tls | SW_AGENT_FORCE_TLS | | False | Use TLS for gRPC/HTTP with the OAP (no client cert required). For gRPC, ensure the server certificate SAN matches the first usable backend in agent_collector_backend_services (used as grpc.default_authority). | +| agent_force_tls | SW_AGENT_FORCE_TLS | | False | Use TLS for gRPC/HTTP with the OAP. One-way TLS uses the process trust store unless agent_ssl_trusted_ca_path points to a readable, parseable CA PEM. For gRPC, :authority and TLS peer-name checks use grpc.default_authority = the first configured endpoint (host after SplitHostPort; port is never checked). The dial target may differ after multi-backend shuffle/DNS expand. If even process-trust credentials cannot be built, reporters warn and stay plaintext (FORCE_TLS never aborts agent start). | +| agent_ssl_trusted_ca_path | SW_AGENT_SSL_TRUSTED_CA_PATH | | | PEM of the CA that signed the OAP server certificate. If this file exists and parses as a CA PEM, the agent uses TLS even when agent_force_tls is false. Missing/unreadable/invalid PEM without FORCE_TLS stays plaintext. Absolute path or relative to the process working directory; symlinks to regular files are followed. Empty means no custom CA. PEM files larger than 256KiB are rejected. | +| agent_ssl_cert_chain_path | SW_AGENT_SSL_CERT_CHAIN_PATH | | | PEM of the agent certificate chain for mTLS. Used only when a usable CA was loaded and agent_ssl_key_path is also set. Missing files log a warning and keep one-way TLS. Symlinks to regular files are followed; 256KiB size cap. | +| agent_ssl_key_path | SW_AGENT_SSL_KEY_PATH | | | PEM of the agent private key for mTLS (unencrypted). PKCS#1 is converted to PKCS#8. Used with agent_ssl_cert_chain_path. Symlinks to regular files are followed; 256KiB size cap. | | agent_authentication | SW_AGENT_AUTHENTICATION | | | The authentication token to verify that the agent is trusted by the backend OAP, as for how to configure the backend, refer to [the yaml](https://github.com/apache/skywalking/blob/4f0f39ffccdc9b41049903cc540b8904f7c9728e/oap-server/server-bootstrap/src/main/resources/application.yml#L155-L158). | | agent_logging_level | SW_AGENT_LOGGING_LEVEL | | INFO | The level of agent self-logs, could be one of `CRITICAL`, `FATAL`, `ERROR`, `WARN`(`WARNING`), `INFO`, `DEBUG`. Please turn on debug if an issue is encountered to find out what's going on | ### Agent Core Danger Zone diff --git a/docs/en/setup/Intrusive.md b/docs/en/setup/Intrusive.md index d00688999..7c074f8ac 100644 --- a/docs/en/setup/Intrusive.md +++ b/docs/en/setup/Intrusive.md @@ -29,7 +29,7 @@ agent.start() #### gRPC multi-address (failover) -Pass a comma-separated list. The agent opens **one** gRPC channel and lets C-core `pick_first` fail over (same idea as Node `sw-static`). Each process shuffles the preferred backend at channel build; `:authority` / TLS SNI still use the **first configured** endpoint. +Pass a comma-separated list. The agent opens **one** gRPC channel and lets C-core `pick_first` fail over (same idea as Node `sw-static`). Each process shuffles the preferred backend at channel build; `:authority` and TLS peer-name checks use the **first configured** endpoint via `grpc.default_authority` (not the shuffled channel target). ```python config.init( @@ -43,7 +43,28 @@ Implementation notes (maintainers / operators): - Mixed IPv4/IPv6 stays in one list; IPv4 is encoded as IPv4-mapped IPv6 for grpcio so `pick_first` can try both families. - Multi-hostname lists are DNS-expanded once at channel build (about 5s lookup budget per name); there is no periodic re-resolve — prefer a single address or stable IPs when DNS changes. -- Channel `:authority` / TLS SAN uses `grpc.default_authority` = the first configured endpoint (before shuffle). With `agent_force_tls`, every backend cert must cover that authority. +- Channel `:authority` uses `grpc.default_authority` = the first configured endpoint (before shuffle). With that arg set, gRPC C-core also uses it for TLS peer-name / SNI checks (host only after `SplitHostPort`; port is never checked) — not the encoded multi-address channel target used for dialing. Prefer a first endpoint whose CN/SAN matches the server cert; multi-backend IP expansion still needs certs that cover those dial targets if you rely on target-derived names without `default_authority`. + +#### gRPC / HTTP TLS and mTLS + +Collector transport security (Kafka protocol does not use these options): + +- `agent_force_tls=true` alone → TLS with the process trust store (no client cert). With a usable CA, custom CA (and optional mTLS) still apply. +- If `agent_ssl_trusted_ca_path` is a readable **and parseable** CA PEM → TLS using that CA, even when `agent_force_tls` is false. +- A **non-empty** `agent_ssl_trusted_ca_path` that is missing/unreadable/**invalid PEM** logs a warning. Without `agent_force_tls`, the collector stays **plaintext** (it does not silently fall back to the system trust store). With `agent_force_tls`, custom CA is ignored and mTLS is disabled (system-trust one-way TLS only). +- mTLS is on only when a usable custom CA was loaded **and** both `agent_ssl_cert_chain_path` and `agent_ssl_key_path` are readable, parseable PEMs. Missing cert/key, only one of the two set, invalid client material, or load/temp-file failure logs a warning and stays one-way TLS (does not abort start). Client cert/key **without** a usable CA also logs a warning and does not enable mTLS (`FORCE_TLS` alone remains one-way TLS via the process trust store). With a usable CA file present, `agent_force_tls` is optional. +- Invalid PEM content / path resolution errors must not crash the host process: PEMs and paths are handled when material is built (not deferred to the first collector connect). Degrade table — bad CA + `FORCE_TLS` → system trust, no client cert; bad CA without `FORCE_TLS` → plaintext; bad client material → one-way TLS; if even process-trust credential / `SSLContext` construction fails → **warn and stay plaintext** (including when `FORCE_TLS` is set — encryption is best-effort so start is never aborted). +- Paths are absolute or relative to the process working directory (symlinks to regular files are followed, e.g. Kubernetes secret mounts). Each PEM is capped at 256KiB. PKCS#1 (`BEGIN RSA PRIVATE KEY`) is converted to PKCS#8 for stacks that reject PKCS#1 (HTTP and gRPC share the normalized key). Passphrase-encrypted private keys (PKCS#8 encrypted or legacy `Proc-Type: 4,ENCRYPTED`) are not supported. HTTP reporters write normalized key material to process-lifetime temp files that are deleted on exit; after `os.fork()` the child does not unlink the parent's temp PEMs. +- Real-OAP e2e covers **gRPC** one-way TLS and mTLS: `tests/e2e/case/grpc/ssl/` and `tests/e2e/case/grpc/mtls/`. PEMs are generated by a one-shot `gen-certs` service that runs digest-pinned `alpine/openssl` with the shared script `tests/e2e/script/prepare/gen-e2e-tls-certs.sh` (not committed; no `apk`). OAP HTTP/REST TLS is server-side only (no client-cert verification), so there is no real-OAP **HTTP mTLS** e2e; the HTTP reporter mTLS path is covered by unit tests and local verification against a TLS terminator that requires a client certificate. + +```python +config.init( + agent_collector_backend_services='oap.example:11800', + agent_ssl_trusted_ca_path='/etc/skywalking/ca.crt', + agent_ssl_cert_chain_path='/etc/skywalking/client.crt', + agent_ssl_key_path='/etc/skywalking/client.pem', +) +``` - Reporters wait until the channel is READY; non-READY skips the RPC rather than failing fast into a black hole. - After a silent backend switch that stays READY, instance properties are re-reported on the normal properties period so the new OAP learns the instance. - Reconnect backoff caps at 30s. diff --git a/skywalking/client/http.py b/skywalking/client/http.py index d880cd2f2..c529fe96a 100644 --- a/skywalking/client/http.py +++ b/skywalking/client/http.py @@ -21,6 +21,7 @@ from skywalking import config from skywalking.client import ServiceManagementClient, TraceSegmentReportService, LogDataReportService from skywalking.loggings import logger, logger_debug_enabled +from skywalking.utils.tls import collector_http_scheme, configure_requests_session class HttpServiceManagementClient(ServiceManagementClient): @@ -28,10 +29,11 @@ def __init__(self): super().__init__() self.instance_properties = self.get_instance_properties() - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_instance_props = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/reportProperties" self.url_heart_beat = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/keepAlive" self.session = requests.Session() + configure_requests_session(self.session) def send_instance_props(self): res = self.session.post(self.url_instance_props, json={ @@ -61,9 +63,10 @@ def send_heart_beat(self): class HttpTraceSegmentReportService(TraceSegmentReportService): def __init__(self): - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/segment" self.session = requests.Session() + configure_requests_session(self.session) def report(self, generator): for segment in generator: @@ -113,9 +116,10 @@ def report(self, generator): class HttpLogDataReportService(LogDataReportService): def __init__(self): - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/logs" self.session = requests.Session() + configure_requests_session(self.session) def report(self, generator): log_batch = [json.loads(json_format.MessageToJson(log_data)) for log_data in generator] diff --git a/skywalking/client/http_aio.py b/skywalking/client/http_aio.py index 899ecb0a2..038a4e4b8 100644 --- a/skywalking/client/http_aio.py +++ b/skywalking/client/http_aio.py @@ -21,6 +21,14 @@ from skywalking import config from skywalking.client import ServiceManagementClientAsync, TraceSegmentReportServiceAsync, LogDataReportServiceAsync from skywalking.loggings import logger, logger_debug_enabled +from skywalking.utils.tls import collector_http_scheme, ssl_context_for_collector + + +def _aiohttp_session(): + ssl_ctx = ssl_context_for_collector() + if ssl_ctx is None: + return aiohttp.ClientSession() + return aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_ctx)) class HttpServiceManagementClientAsync(ServiceManagementClientAsync): @@ -28,11 +36,11 @@ def __init__(self): super().__init__() self.instance_properties = self.get_instance_properties() - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_instance_props = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/reportProperties" self.url_heart_beat = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/keepAlive" # self.client = httpx.AsyncClient() - self.client = aiohttp.ClientSession() + self.client = _aiohttp_session() async def send_instance_props(self): @@ -65,10 +73,10 @@ async def send_heart_beat(self): class HttpTraceSegmentReportServiceAsync(TraceSegmentReportServiceAsync): def __init__(self): - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/segment" # self.client = httpx.AsyncClient() - self.client = aiohttp.ClientSession() + self.client = _aiohttp_session() async def report(self, generator): async for segment in generator: @@ -119,10 +127,10 @@ async def report(self, generator): class HttpLogDataReportServiceAsync(LogDataReportServiceAsync): def __init__(self): - proto = 'https://' if config.agent_force_tls else 'http://' + proto = collector_http_scheme() self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/logs" # self.client = httpx.AsyncClient() - self.client = aiohttp.ClientSession() + self.client = _aiohttp_session() async def report(self, generator): log_batch = [json.loads(json_format.MessageToJson(log_data)) async for log_data in generator] diff --git a/skywalking/config.py b/skywalking/config.py index 9756e3024..fa0db343e 100644 --- a/skywalking/config.py +++ b/skywalking/config.py @@ -77,10 +77,27 @@ # [here](https://kafka-python.readthedocs.io/en/master/apidoc/KafkaProducer.html#kafka.KafkaProducer) # This config only works from env variables, each one should be passed in `SW_KAFKA_REPORTER_CONFIG_` kafka_reporter_custom_configurations: str = os.getenv('SW_KAFKA_REPORTER_CUSTOM_CONFIGURATIONS', '') -# Use TLS for gRPC/HTTP with the OAP (no client cert required). For gRPC, ensure the server -# certificate SAN matches the first usable backend in agent_collector_backend_services -# (used as grpc.default_authority). +# Use TLS for gRPC/HTTP with the OAP. One-way TLS uses the process trust store unless +# agent_ssl_trusted_ca_path points to a readable, parseable CA PEM. For gRPC, +# :authority and TLS peer-name checks use grpc.default_authority = the first +# configured endpoint (host after SplitHostPort; port is never checked). The +# dial target may differ after multi-backend shuffle/DNS expand. If even +# process-trust credentials cannot be built, reporters warn and stay plaintext +# (FORCE_TLS never aborts agent start). agent_force_tls: bool = os.getenv('SW_AGENT_FORCE_TLS', '').lower() == 'true' +# PEM of the CA that signed the OAP server certificate. If this file exists and +# parses as a CA PEM, the agent uses TLS even when agent_force_tls is false. +# Missing/unreadable/invalid PEM without FORCE_TLS stays plaintext. Absolute +# path or relative to the process working directory; symlinks to regular files +# are followed. Empty means no custom CA. PEM files larger than 256KiB are rejected. +agent_ssl_trusted_ca_path: str = os.getenv('SW_AGENT_SSL_TRUSTED_CA_PATH', '') +# PEM of the agent certificate chain for mTLS. Used only when a usable CA was +# loaded and agent_ssl_key_path is also set. Missing files log a warning and +# keep one-way TLS. Symlinks to regular files are followed; 256KiB size cap. +agent_ssl_cert_chain_path: str = os.getenv('SW_AGENT_SSL_CERT_CHAIN_PATH', '') +# PEM of the agent private key for mTLS (unencrypted). PKCS#1 is converted to PKCS#8. +# Used with agent_ssl_cert_chain_path. Symlinks to regular files are followed; 256KiB size cap. +agent_ssl_key_path: str = os.getenv('SW_AGENT_SSL_KEY_PATH', '') # The authentication token to verify that the agent is trusted by the backend OAP, as for how to configure the # backend, refer to [the yaml](https://github.com/apache/skywalking/blob/4f0f39ffccdc9b41049903cc540b8904f7c9728e/ # oap-server/server-bootstrap/src/main/resources/application.yml#L155-L158). diff --git a/skywalking/utils/grpc_channel.py b/skywalking/utils/grpc_channel.py index 6d68a9d3f..71948b817 100644 --- a/skywalking/utils/grpc_channel.py +++ b/skywalking/utils/grpc_channel.py @@ -1,677 +1,682 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -""" -Shared gRPC channel target / options helpers for sync and aio reporters. - -Multi-backend design (aligned with skywalking-nodejs native failover): -- One channel for the process lifetime; no hand-rolled poll/reselect manager. -- Single address → plain host:port (DNS for hostnames, with re-resolve). -- Multiple addresses are assembled like Node sw-static endpoints - ({host, port} list, IPv4 / IPv6 / hostname can coexist). -- grpcio cannot register a custom scheme; the endpoint list is encoded for - C-core: homogeneous ipv4:/ipv6:, mixed families via ipv6: + IPv4-mapped - (::ffff:a.b.c.d) so pick_first can try both families. -- Hostnames in a multi list are resolved once at channel build (grpcio cannot - keep a literal hostname in ipv4:/ipv6:). No periodic DNS re-resolve for multi. -- pick_first shuffleAddressList is on (per-process random preferred backend). - Channel target / grpc.default_authority still follow config order (TLS SAN). -- Invalid entries are logged and dropped; never silently ignored without a log. -- HTTP proxy disabled; keepalive channel options intentionally omitted (OAP conflict). -- Unary and sync streaming RPCs use a deadline (Node 10s floor, always > - agent_queue_timeout + margin so sync collect is not cut off by the batching window). - Aio client-streaming collect/collectBatch/collectSnapshot omit timeout= - (generators await empty queues). DEADLINE_EXCEEDED / RESOURCE_EXHAUSTED on a - READY backend do not rotate or rebuild; failover is for unreachable backends, - not a slow but connected one. -- READY gate (application-level): skip report RPCs unless channel connectivity is READY; - nudge IDLE via get_state(True) so gating does not starve reconnect (Node watch parity). -- Reconnect backoff max 30s (Node multi-backend parity). -- service_config retries only ManagementService.reportInstanceProperties on UNAVAILABLE - (max 3); never retry client-streaming collect. -""" - -from __future__ import annotations - -import ipaddress -import json -import socket -import threading -import time -from contextlib import contextmanager -from dataclasses import dataclass -from enum import Enum -from typing import Dict, List, Optional, Sequence, Tuple - -import grpc - -from skywalking.loggings import logger - -# Retry only unary idempotent reportInstanceProperties (Node service_config parity). -# Client-streaming collect must NOT be retried — replay would duplicate segments. -# keepAlive relies on the next heartbeat tick instead. -_PROPERTIES_RETRY_SERVICE_CONFIG = json.dumps({ - # Shuffle is LB-layer (Node parity): target string stays config-order so - # grpc.default_authority / TLS SNI remain the first configured endpoint. - 'loadBalancingConfig': [{'pick_first': {'shuffleAddressList': True}}], - 'methodConfig': [{ - 'name': [{ - 'service': 'skywalking.v3.ManagementService', - 'method': 'reportInstanceProperties', - }], - 'retryPolicy': { - 'maxAttempts': 3, - 'initialBackoff': '1s', - 'maxBackoff': '10s', - 'backoffMultiplier': 2, - 'retryableStatusCodes': ['UNAVAILABLE'], - }, - }], -}) - -# Channel options shared by sync + aio. Do NOT add keepalive_* options here. -# pick_first + shuffle lives in grpc.service_config (not grpc.lb_policy_name). -GRPC_CHANNEL_OPTIONS: Tuple[Tuple[str, int | str], ...] = ( - ('grpc.enable_http_proxy', 0), - ('grpc.enable_retries', 1), - ('grpc.service_config', _PROPERTIES_RETRY_SERVICE_CONFIG), - ('grpc.initial_reconnect_backoff_ms', 1000), - ('grpc.min_reconnect_backoff_ms', 1000), - # Cap aligns with Node multi-backend (~30s); shorter caps reconnect too aggressively. - ('grpc.max_reconnect_backoff_ms', 30000), -) - -# Node default RPC deadline is 10s. Sync streaming collect must outlive the queue -# batch window with room for protobuf encode + RTT + server handling. -# Sync generators may spend nearly the full queue window on the final queue.get -# (absolute batch deadline); margin keeps healthy sends off DEADLINE_EXCEEDED. -# Do not apply this to aio client-streaming: those generators await queue.get() forever. -_GRPC_RPC_TIMEOUT_FLOOR_SEC = 10.0 -_GRPC_RPC_TIMEOUT_MARGIN_SEC = 5.0 - - -def grpc_call_timeout() -> float: - """Seconds for unary / sync-streaming stub timeout=. Always > agent_queue_timeout + margin.""" - from skywalking import config - - return max( - _GRPC_RPC_TIMEOUT_FLOOR_SEC, - float(config.agent_queue_timeout) + _GRPC_RPC_TIMEOUT_MARGIN_SEC, - ) - - -_AUTH_LOG_INTERVAL_SEC = 60.0 -_last_auth_log_at = 0.0 -_CONNECTIVITY_LOG_INTERVAL_SEC = 30.0 -_last_connectivity_log_at: Dict[str, float] = {} -_DNS_LOOKUP_TIMEOUT_SEC = 5.0 - -# Thread-local: set while create_*_channel builds the agent→OAP channel so sw_grpc -# does not attach client interceptors (multi-address targets no longer match config). -_building_agent_collector = threading.local() -_SW_AGENT_COLLECTOR_ATTR = '_sw_agent_collector_channel' - - -@contextmanager -def agent_collector_channel_scope(): - _building_agent_collector.active = True - try: - yield - finally: - _building_agent_collector.active = False - - -def is_building_agent_collector_channel() -> bool: - return bool(getattr(_building_agent_collector, 'active', False)) - - -def mark_agent_collector_channel(channel): - try: - setattr(channel, _SW_AGENT_COLLECTOR_ATTR, True) - except Exception: # noqa: BLE001 - exotic channel wrappers - pass - return channel - - -def is_agent_collector_channel(channel) -> bool: - return bool(getattr(channel, _SW_AGENT_COLLECTOR_ATTR, False)) - - -class AddressKind(Enum): - IPV4 = 'ipv4' - IPV6 = 'ipv6' - HOSTNAME = 'hostname' - - -@dataclass(frozen=True) -class BackendAddress: - host: str - port: int - kind: AddressKind - - def endpoint(self) -> str: - if self.kind == AddressKind.IPV6: - return f'[{self.host}]:{self.port}' - return f'{self.host}:{self.port}' - - -def _classify_host(host: str) -> Optional[AddressKind]: - if not host or any(c.isspace() or ord(c) < 32 for c in host) or '/' in host: - return None - # Zone indices (fe80::1%eth0) are not usable in static ipv6: targets. - if '%' in host: - return None - try: - ip = ipaddress.ip_address(host) - except ValueError: - return AddressKind.HOSTNAME - if isinstance(ip, ipaddress.IPv4Address): - return AddressKind.IPV4 - return AddressKind.IPV6 - - -def parse_backend_address(raw: str) -> Optional[BackendAddress]: - """Parse a single host:port (IPv6 requires [host]:port). Returns None if invalid.""" - text = (raw or '').strip() - if not text: - return None - - host: str - port_str: str - if text.startswith('['): - # [ipv6]:port - closing = text.find(']') - if closing <= 1 or closing + 1 >= len(text) or text[closing + 1] != ':': - return None - host = text[1:closing] - port_str = text[closing + 2:] - else: - if text.count(':') != 1: - # Ambiguous IPv6 without brackets, or missing port. - return None - host, port_str = text.rsplit(':', 1) - - host = host.strip() - port_str = port_str.strip() - if not host or not port_str: - return None - try: - port = int(port_str) - except ValueError: - return None - if port < 1 or port > 65535: - return None - - kind = _classify_host(host) - if kind is None: - return None - return BackendAddress(host=host, port=port, kind=kind) - - -def parse_backend_addresses(services: str) -> List[BackendAddress]: - """ - Split SW_AGENT_COLLECTOR_BACKEND_SERVICES on commas. - Invalid entries are skipped with an error log (never silent). - """ - parts = [p.strip() for p in (services or '').split(',') if p.strip()] - addresses: List[BackendAddress] = [] - for part in parts: - addr = parse_backend_address(part) - if addr is None: - logger.error( - 'Invalid collector backend address %r in SW_AGENT_COLLECTOR_BACKEND_SERVICES; ' - 'expected host:port or [ipv6]:port', - part, - ) - continue - addresses.append(addr) - return addresses - - -def sw_static_endpoints(addresses: Sequence[BackendAddress]) -> List[Dict]: - """ - Node sw-static resolver output shape: a list of endpoints, each with - addresses: [{host, port}]. IPv4, IPv6, and hostnames can share one list. - """ - return [ - {'addresses': [{'host': addr.host, 'port': addr.port}]} - for addr in addresses - ] - - -def _lookup_hostname(host: str, port: int) -> List[BackendAddress]: - """ - Resolve hostname to BackendAddress IPs (order preserved, duplicates dropped). - - Multi-address targets require literal IPs for ipv4:/ipv6:. A hung DNS lookup - must not block agent startup or process exit — bound the wait and run the - lookup on a daemon thread (ThreadPoolExecutor workers are non-daemon and - would keep the process alive after timeout). - """ - import threading - from concurrent.futures import Future, TimeoutError as FuturesTimeout - - fut: Future = Future() - - def _run() -> None: - try: - fut.set_result(socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)) - except Exception as exc: # noqa: BLE001 - forward any lookup failure to waiter - if not fut.done(): - fut.set_exception(exc) - - threading.Thread(target=_run, name=f'sw-dns-{host}', daemon=True).start() - try: - infos = fut.result(timeout=_DNS_LOOKUP_TIMEOUT_SEC) - except FuturesTimeout: - logger.error( - 'Timed out resolving collector hostname %r:%s after %.1fs; skipping this backend', - host, - port, - _DNS_LOOKUP_TIMEOUT_SEC, - ) - return [] - except socket.gaierror as exc: - logger.error( - 'Failed to resolve collector hostname %r:%s (%s); skipping this backend', - host, - port, - exc, - ) - return [] - except Exception as exc: # noqa: BLE001 - never crash agent init on DNS oddities - logger.error( - 'Unexpected error resolving collector hostname %r:%s (%s); skipping this backend', - host, - port, - exc, - ) - return [] - - resolved: List[BackendAddress] = [] - seen = set() - for family, _type, _proto, _canon, sockaddr in infos: - if family == socket.AF_INET: - ip = sockaddr[0] - kind = AddressKind.IPV4 - elif family == socket.AF_INET6: - ip = sockaddr[0] - if '%' in ip: - ip = ip.split('%', 1)[0] - # Skip IPv4-mapped IPv6; the AF_INET result already covers that backend. - try: - packed = ipaddress.ip_address(ip) - except ValueError: - continue - if packed.ipv4_mapped is not None: - continue - kind = AddressKind.IPV6 - else: - continue - key = (kind, ip, port) - if key in seen: - continue - seen.add(key) - resolved.append(BackendAddress(host=ip, port=port, kind=kind)) - if not resolved: - logger.error( - 'Collector hostname %r:%s resolved to no usable IPv4/IPv6 address; skipping', - host, - port, - ) - return resolved - - -def expand_backend_addresses(addresses: Sequence[BackendAddress]) -> List[BackendAddress]: - """ - Expand hostnames to literal IPs for C-core static targets. - Literal IP entries are kept as-is. Failed hostname lookups are skipped with error logs. - """ - expanded: List[BackendAddress] = [] - seen = set() - for addr in addresses: - candidates: Sequence[BackendAddress] - if addr.kind == AddressKind.HOSTNAME: - candidates = _lookup_hostname(addr.host, addr.port) - else: - candidates = (addr,) - for item in candidates: - key = (item.kind, item.host, item.port) - if key in seen: - continue - seen.add(key) - expanded.append(item) - return expanded - - -def _ipv4_mapped_v6(ipv4: str) -> str: - return f'::ffff:{ipv4}' - - -def encode_sw_static_for_c_core(addresses: Sequence[BackendAddress]) -> str: - """ - Encode a Node-style mixed endpoint list for grpcio/C-core. - - Homogeneous lists use ipv4:/ipv6:. Mixed IPv4+IPv6 uses the ipv6 resolver - with IPv4-mapped addresses so pick_first can try both families. - """ - if not addresses: - raise ValueError( - 'No valid collector backend address in SW_AGENT_COLLECTOR_BACKEND_SERVICES' - ) - if any(a.kind == AddressKind.HOSTNAME for a in addresses): - raise ValueError('encode_sw_static_for_c_core requires literal IP endpoints') - - kinds = {a.kind for a in addresses} - if kinds == {AddressKind.IPV4}: - return 'ipv4:' + ','.join(f'{a.host}:{a.port}' for a in addresses) - if kinds == {AddressKind.IPV6}: - return 'ipv6:' + ','.join(f'[{a.host}]:{a.port}' for a in addresses) - - parts = [] - for addr in addresses: - if addr.kind == AddressKind.IPV4: - parts.append(f'[{_ipv4_mapped_v6(addr.host)}]:{addr.port}') - else: - parts.append(f'[{addr.host}]:{addr.port}') - logger.info( - 'Encoding mixed-family sw-static endpoints for grpcio via ipv6 IPv4-mapped list: %s', - parts, - ) - return 'ipv6:' + ','.join(parts) - - -def prepare_grpc_channel_endpoints( - addresses: Sequence[BackendAddress], -) -> Tuple[str, str]: - """ - Build (channel_target, default_authority) from parsed backends. - - Authority prefers the first *usable* original entry's host:port (hostname kept - for TLS SAN when that name resolved). Never points at a hostname that DNS skipped. - """ - if not addresses: - raise ValueError( - 'No valid collector backend address in SW_AGENT_COLLECTOR_BACKEND_SERVICES' - ) - - if len(addresses) == 1: - ep = addresses[0].endpoint() - return ep, ep - - to_encode: List[BackendAddress] = [] - seen = set() - authority: Optional[str] = None - - for orig in addresses: - if orig.kind == AddressKind.HOSTNAME: - candidates = _lookup_hostname(orig.host, orig.port) - if not candidates: - continue - if authority is None: - # Prefer original hostname for :authority / SNI (Node sw-static style). - authority = orig.endpoint() - else: - candidates = (orig,) - if authority is None: - authority = orig.endpoint() - for item in candidates: - key = (item.kind, item.host, item.port) - if key in seen: - continue - seen.add(key) - to_encode.append(item) - - if not to_encode: - raise ValueError( - 'No usable collector backend address after DNS expansion of ' - 'SW_AGENT_COLLECTOR_BACKEND_SERVICES' - ) - if authority is None: - authority = to_encode[0].endpoint() - - if any(a.kind == AddressKind.HOSTNAME for a in addresses): - logger.info( - 'Expanded multi-backend collector addresses %s -> %s (authority=%s)', - [a.endpoint() for a in addresses], - [a.endpoint() for a in to_encode], - authority, - ) - return encode_sw_static_for_c_core(to_encode), authority - - -def _resolve_channel_target_and_authority() -> Tuple[str, str]: - """ - Never raise into the host app. prepare_grpc_channel_endpoints stays strict; - factories degrade so the agent can idle behind the READY gate. - """ - from skywalking import config - - raw = config.agent_collector_backend_services - addresses = parse_backend_addresses(raw) - try: - return prepare_grpc_channel_endpoints(addresses) - except ValueError: - if addresses: - target = addresses[0].endpoint() - logger.error( - 'No usable collector backend after DNS expansion of %r; ' - 'falling back to plain target %s so grpcio can re-resolve', - raw, - target, - ) - return target, target - fallback = (raw or '').strip() or 'localhost:1' - logger.error( - 'No valid collector backend address in %r; opening a channel to %s ' - '(agent stays up; READY gate skips reports)', - raw, - fallback, - ) - return fallback, fallback - - -def build_grpc_target(addresses: Sequence[BackendAddress]) -> str: - """Build a gRPC channel target string (see prepare_grpc_channel_endpoints).""" - target, _authority = prepare_grpc_channel_endpoints(addresses) - return target - - -def resolve_grpc_target(services: Optional[str] = None) -> str: - from skywalking import config - - raw = config.agent_collector_backend_services if services is None else services - return build_grpc_target(parse_backend_addresses(raw)) - - -def _channel_options(default_authority: str) -> Tuple[Tuple[str, int | str], ...]: - options = list(GRPC_CHANNEL_OPTIONS) - # Align with Node sw-static getDefaultAuthority (first usable backend). - options.append(('grpc.default_authority', default_authority)) - return tuple(options) - - -def create_sync_channel(): - """Create one sync gRPC channel (caller may wrap with auth interceptor).""" - from skywalking import config - - target, authority = _resolve_channel_target_and_authority() - options = _channel_options(authority) - logger.info('Creating gRPC channel to collector target %s (authority=%s)', target, authority) - with agent_collector_channel_scope(): - try: - if config.agent_force_tls: - channel = grpc.secure_channel(target, grpc.ssl_channel_credentials(), options=options) - else: - channel = grpc.insecure_channel(target, options=options) - except Exception: # noqa: BLE001 - never fail host process start - logger.exception( - 'Failed to create gRPC channel to %s; using localhost:1 placeholder', - target, - ) - channel = grpc.insecure_channel('localhost:1', options=options) - return mark_agent_collector_channel(channel) - - -def create_aio_channel(interceptors=None): - """Create one aio gRPC channel with optional interceptors.""" - from skywalking import config - - target, authority = _resolve_channel_target_and_authority() - options = _channel_options(authority) - logger.info('Creating aio gRPC channel to collector target %s (authority=%s)', target, authority) - with agent_collector_channel_scope(): - try: - if config.agent_force_tls: - channel = grpc.aio.secure_channel( - target, - grpc.ssl_channel_credentials(), - options=options, - interceptors=interceptors, - ) - else: - channel = grpc.aio.insecure_channel(target, options=options, interceptors=interceptors) - except Exception: # noqa: BLE001 - never fail host process start - logger.exception( - 'Failed to create aio gRPC channel to %s; using localhost:1 placeholder', - target, - ) - channel = grpc.aio.insecure_channel('localhost:1', options=options, interceptors=interceptors) - return mark_agent_collector_channel(channel) - - -def _unwrap_connectivity_state(channel, try_to_connect: bool): - """ - Read channel connectivity for sync, aio, and intercept_channel wrappers. - - grpc.aio.Channel exposes get_state(). Sync grpc._channel.Channel does not - (subscribe only); use C-core check_connectivity_state on the cython channel. - Intercepted channels nest the real Channel on ``_channel``. - """ - get_state = getattr(channel, 'get_state', None) - if callable(get_state): - return get_state(try_to_connect) - - candidate = channel - for _ in range(4): - inner = getattr(candidate, '_channel', None) - if inner is None: - break - check = getattr(inner, 'check_connectivity_state', None) - if callable(check): - code = check(try_to_connect) - for state in grpc.ChannelConnectivity: - if state.value[0] == code: - return state - return None - candidate = inner - return None - - -def is_channel_ready(channel) -> bool: - """ - Application-level READY gate (gRPC only exposes connectivity state). - - Returns True only when connectivity is READY. Uses try_to_connect=True so an - IDLE channel is nudged into CONNECTING — otherwise skipping all RPCs would - leave the channel idle forever (same role as Node watchConnectivityState - with requestConnection=true). CONNECTING / TRANSIENT_FAILURE still return - False so reporters skip until READY. - - If connectivity cannot be read, returns True (fail-open) so a missing API - cannot permanently silence reporting. - """ - try: - state = _unwrap_connectivity_state(channel, True) - except Exception: # noqa: BLE001 - defensive for closed / exotic channels - return True - if state is None: - return True - return state == grpc.ChannelConnectivity.READY - - -def is_auth_rpc_error(error: BaseException) -> bool: - code = getattr(error, 'code', None) - if not callable(code): - return False - try: - status = code() - except Exception: # noqa: BLE001 - defensive for non-grpc exceptions - return False - return status in (grpc.StatusCode.UNAUTHENTICATED, grpc.StatusCode.PERMISSION_DENIED) - - -def log_auth_failure_throttled(error: BaseException) -> None: - """Auth failures must not rotate backends; same cluster shares one token.""" - global _last_auth_log_at - now = time.monotonic() - if now - _last_auth_log_at < _AUTH_LOG_INTERVAL_SEC: - return - _last_auth_log_at = now - logger.error( - 'Collector rejected authentication (%s). Check SW_AGENT_AUTHENTICATION; ' - 'the agent will not rotate backends for auth failures.', - error, - ) - - -def log_connectivity_event(kind: str, message: str, *args) -> None: - """Throttle disconnect warnings; recovery stays informative but rate-limited.""" - now = time.monotonic() - last = _last_connectivity_log_at.get(kind, 0.0) - if now - last < _CONNECTIVITY_LOG_INTERVAL_SEC: - return - _last_connectivity_log_at[kind] = now - if kind == 'recovered': - logger.info(message, *args) - else: - logger.warning(message, *args) - - -def apply_connectivity_transition(prev, state) -> None: - """Shared INFO/WARN side-effects for sync subscribe and aio watch.""" - if state == grpc.ChannelConnectivity.TRANSIENT_FAILURE: - log_connectivity_event( - 'transient_failure', - 'gRPC collector channel disconnected (TRANSIENT_FAILURE)', - ) - elif state == grpc.ChannelConnectivity.IDLE and prev == grpc.ChannelConnectivity.READY: - log_connectivity_event('idle', 'gRPC collector channel disconnected (IDLE)') - elif state == grpc.ChannelConnectivity.READY and prev in ( - grpc.ChannelConnectivity.TRANSIENT_FAILURE, - grpc.ChannelConnectivity.IDLE, - grpc.ChannelConnectivity.CONNECTING, - None, - ): - log_connectivity_event('recovered', 'gRPC collector channel recovered (READY)') - - -def handle_rpc_error(error: BaseException, on_connectivity_error) -> None: - """ - Shared RpcError side-effects for sync/aio reporters. - Auth: throttle log only (no channel rebuild / backend rotate). - Other: invoke connectivity recovery hook (resubscribe / debug), never rebuild channel. - - Failover is for an unreachable backend, not functional health of a connected one. - DEADLINE_EXCEEDED and RESOURCE_EXHAUSTED on a READY channel are intentionally - left to the call site: pick_first will not move off a slow-but-READY backend, - and this agent does not rotate or rebuild the channel for those codes. - """ - if is_auth_rpc_error(error): - log_auth_failure_throttled(error) - return - on_connectivity_error() +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Shared gRPC channel target / options helpers for sync and aio reporters. + +Multi-backend design (aligned with skywalking-nodejs native failover): +- One channel for the process lifetime; no hand-rolled poll/reselect manager. +- Single address → plain host:port (DNS for hostnames, with re-resolve). +- Multiple addresses are assembled like Node sw-static endpoints + ({host, port} list, IPv4 / IPv6 / hostname can coexist). +- grpcio cannot register a custom scheme; the endpoint list is encoded for + C-core: homogeneous ipv4:/ipv6:, mixed families via ipv6: + IPv4-mapped + (::ffff:a.b.c.d) so pick_first can try both families. +- Hostnames in a multi list are resolved once at channel build (grpcio cannot + keep a literal hostname in ipv4:/ipv6:). No periodic DNS re-resolve for multi. +- pick_first shuffleAddressList is on (per-process random preferred backend). + Dial uses the encoded channel target; grpc.default_authority (and thus TLS + peer-name checks when that arg is set) still follow the first configured endpoint. +- Invalid entries are logged and dropped; never silently ignored without a log. +- HTTP proxy disabled; keepalive channel options intentionally omitted (OAP conflict). +- Unary and sync streaming RPCs use a deadline (Node 10s floor, always > + agent_queue_timeout + margin so sync collect is not cut off by the batching window). + Aio client-streaming collect/collectBatch/collectSnapshot omit timeout= + (generators await empty queues). DEADLINE_EXCEEDED / RESOURCE_EXHAUSTED on a + READY backend do not rotate or rebuild; failover is for unreachable backends, + not a slow but connected one. +- READY gate (application-level): skip report RPCs unless channel connectivity is READY; + nudge IDLE via get_state(True) so gating does not starve reconnect (Node watch parity). +- Reconnect backoff max 30s (Node multi-backend parity). +- service_config retries only ManagementService.reportInstanceProperties on UNAVAILABLE + (max 3); never retry client-streaming collect. +""" + +from __future__ import annotations + +import ipaddress +import json +import socket +import threading +import time +from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum +from typing import Dict, List, Optional, Sequence, Tuple + +import grpc + +from skywalking.loggings import logger +from skywalking.utils.tls import grpc_ssl_credentials + +# Retry only unary idempotent reportInstanceProperties (Node service_config parity). +# Client-streaming collect must NOT be retried — replay would duplicate segments. +# keepAlive relies on the next heartbeat tick instead. +_PROPERTIES_RETRY_SERVICE_CONFIG = json.dumps({ + # Shuffle is LB-layer (Node parity): target string stays config-order so + # grpc.default_authority / TLS SNI remain the first configured endpoint. + 'loadBalancingConfig': [{'pick_first': {'shuffleAddressList': True}}], + 'methodConfig': [{ + 'name': [{ + 'service': 'skywalking.v3.ManagementService', + 'method': 'reportInstanceProperties', + }], + 'retryPolicy': { + 'maxAttempts': 3, + 'initialBackoff': '1s', + 'maxBackoff': '10s', + 'backoffMultiplier': 2, + 'retryableStatusCodes': ['UNAVAILABLE'], + }, + }], +}) + +# Channel options shared by sync + aio. Do NOT add keepalive_* options here. +# pick_first + shuffle lives in grpc.service_config (not grpc.lb_policy_name). +GRPC_CHANNEL_OPTIONS: Tuple[Tuple[str, int | str], ...] = ( + ('grpc.enable_http_proxy', 0), + ('grpc.enable_retries', 1), + ('grpc.service_config', _PROPERTIES_RETRY_SERVICE_CONFIG), + ('grpc.initial_reconnect_backoff_ms', 1000), + ('grpc.min_reconnect_backoff_ms', 1000), + # Cap aligns with Node multi-backend (~30s); shorter caps reconnect too aggressively. + ('grpc.max_reconnect_backoff_ms', 30000), +) + +# Node default RPC deadline is 10s. Sync streaming collect must outlive the queue +# batch window with room for protobuf encode + RTT + server handling. +# Sync generators may spend nearly the full queue window on the final queue.get +# (absolute batch deadline); margin keeps healthy sends off DEADLINE_EXCEEDED. +# Do not apply this to aio client-streaming: those generators await queue.get() forever. +_GRPC_RPC_TIMEOUT_FLOOR_SEC = 10.0 +_GRPC_RPC_TIMEOUT_MARGIN_SEC = 5.0 + + +def grpc_call_timeout() -> float: + """Seconds for unary / sync-streaming stub timeout=. Always > agent_queue_timeout + margin.""" + from skywalking import config + + return max( + _GRPC_RPC_TIMEOUT_FLOOR_SEC, + float(config.agent_queue_timeout) + _GRPC_RPC_TIMEOUT_MARGIN_SEC, + ) + + +_AUTH_LOG_INTERVAL_SEC = 60.0 +_last_auth_log_at = 0.0 +_CONNECTIVITY_LOG_INTERVAL_SEC = 30.0 +_last_connectivity_log_at: Dict[str, float] = {} +_DNS_LOOKUP_TIMEOUT_SEC = 5.0 + +# Thread-local: set while create_*_channel builds the agent→OAP channel so sw_grpc +# does not attach client interceptors (multi-address targets no longer match config). +_building_agent_collector = threading.local() +_SW_AGENT_COLLECTOR_ATTR = '_sw_agent_collector_channel' + + +@contextmanager +def agent_collector_channel_scope(): + _building_agent_collector.active = True + try: + yield + finally: + _building_agent_collector.active = False + + +def is_building_agent_collector_channel() -> bool: + return bool(getattr(_building_agent_collector, 'active', False)) + + +def mark_agent_collector_channel(channel): + try: + setattr(channel, _SW_AGENT_COLLECTOR_ATTR, True) + except Exception: # noqa: BLE001 - exotic channel wrappers + pass + return channel + + +def is_agent_collector_channel(channel) -> bool: + return bool(getattr(channel, _SW_AGENT_COLLECTOR_ATTR, False)) + + +class AddressKind(Enum): + IPV4 = 'ipv4' + IPV6 = 'ipv6' + HOSTNAME = 'hostname' + + +@dataclass(frozen=True) +class BackendAddress: + host: str + port: int + kind: AddressKind + + def endpoint(self) -> str: + if self.kind == AddressKind.IPV6: + return f'[{self.host}]:{self.port}' + return f'{self.host}:{self.port}' + + +def _classify_host(host: str) -> Optional[AddressKind]: + if not host or any(c.isspace() or ord(c) < 32 for c in host) or '/' in host: + return None + # Zone indices (fe80::1%eth0) are not usable in static ipv6: targets. + if '%' in host: + return None + try: + ip = ipaddress.ip_address(host) + except ValueError: + return AddressKind.HOSTNAME + if isinstance(ip, ipaddress.IPv4Address): + return AddressKind.IPV4 + return AddressKind.IPV6 + + +def parse_backend_address(raw: str) -> Optional[BackendAddress]: + """Parse a single host:port (IPv6 requires [host]:port). Returns None if invalid.""" + text = (raw or '').strip() + if not text: + return None + + host: str + port_str: str + if text.startswith('['): + # [ipv6]:port + closing = text.find(']') + if closing <= 1 or closing + 1 >= len(text) or text[closing + 1] != ':': + return None + host = text[1:closing] + port_str = text[closing + 2:] + else: + if text.count(':') != 1: + # Ambiguous IPv6 without brackets, or missing port. + return None + host, port_str = text.rsplit(':', 1) + + host = host.strip() + port_str = port_str.strip() + if not host or not port_str: + return None + try: + port = int(port_str) + except ValueError: + return None + if port < 1 or port > 65535: + return None + + kind = _classify_host(host) + if kind is None: + return None + return BackendAddress(host=host, port=port, kind=kind) + + +def parse_backend_addresses(services: str) -> List[BackendAddress]: + """ + Split SW_AGENT_COLLECTOR_BACKEND_SERVICES on commas. + Invalid entries are skipped with an error log (never silent). + """ + parts = [p.strip() for p in (services or '').split(',') if p.strip()] + addresses: List[BackendAddress] = [] + for part in parts: + addr = parse_backend_address(part) + if addr is None: + logger.error( + 'Invalid collector backend address %r in SW_AGENT_COLLECTOR_BACKEND_SERVICES; ' + 'expected host:port or [ipv6]:port', + part, + ) + continue + addresses.append(addr) + return addresses + + +def sw_static_endpoints(addresses: Sequence[BackendAddress]) -> List[Dict]: + """ + Node sw-static resolver output shape: a list of endpoints, each with + addresses: [{host, port}]. IPv4, IPv6, and hostnames can share one list. + """ + return [ + {'addresses': [{'host': addr.host, 'port': addr.port}]} + for addr in addresses + ] + + +def _lookup_hostname(host: str, port: int) -> List[BackendAddress]: + """ + Resolve hostname to BackendAddress IPs (order preserved, duplicates dropped). + + Multi-address targets require literal IPs for ipv4:/ipv6:. A hung DNS lookup + must not block agent startup or process exit — bound the wait and run the + lookup on a daemon thread (ThreadPoolExecutor workers are non-daemon and + would keep the process alive after timeout). + """ + import threading + from concurrent.futures import Future, TimeoutError as FuturesTimeout + + fut: Future = Future() + + def _run() -> None: + try: + fut.set_result(socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM)) + except Exception as exc: # noqa: BLE001 - forward any lookup failure to waiter + if not fut.done(): + fut.set_exception(exc) + + threading.Thread(target=_run, name=f'sw-dns-{host}', daemon=True).start() + try: + infos = fut.result(timeout=_DNS_LOOKUP_TIMEOUT_SEC) + except FuturesTimeout: + logger.error( + 'Timed out resolving collector hostname %r:%s after %.1fs; skipping this backend', + host, + port, + _DNS_LOOKUP_TIMEOUT_SEC, + ) + return [] + except socket.gaierror as exc: + logger.error( + 'Failed to resolve collector hostname %r:%s (%s); skipping this backend', + host, + port, + exc, + ) + return [] + except Exception as exc: # noqa: BLE001 - never crash agent init on DNS oddities + logger.error( + 'Unexpected error resolving collector hostname %r:%s (%s); skipping this backend', + host, + port, + exc, + ) + return [] + + resolved: List[BackendAddress] = [] + seen = set() + for family, _type, _proto, _canon, sockaddr in infos: + if family == socket.AF_INET: + ip = sockaddr[0] + kind = AddressKind.IPV4 + elif family == socket.AF_INET6: + ip = sockaddr[0] + if '%' in ip: + ip = ip.split('%', 1)[0] + # Skip IPv4-mapped IPv6; the AF_INET result already covers that backend. + try: + packed = ipaddress.ip_address(ip) + except ValueError: + continue + if packed.ipv4_mapped is not None: + continue + kind = AddressKind.IPV6 + else: + continue + key = (kind, ip, port) + if key in seen: + continue + seen.add(key) + resolved.append(BackendAddress(host=ip, port=port, kind=kind)) + if not resolved: + logger.error( + 'Collector hostname %r:%s resolved to no usable IPv4/IPv6 address; skipping', + host, + port, + ) + return resolved + + +def expand_backend_addresses(addresses: Sequence[BackendAddress]) -> List[BackendAddress]: + """ + Expand hostnames to literal IPs for C-core static targets. + Literal IP entries are kept as-is. Failed hostname lookups are skipped with error logs. + """ + expanded: List[BackendAddress] = [] + seen = set() + for addr in addresses: + candidates: Sequence[BackendAddress] + if addr.kind == AddressKind.HOSTNAME: + candidates = _lookup_hostname(addr.host, addr.port) + else: + candidates = (addr,) + for item in candidates: + key = (item.kind, item.host, item.port) + if key in seen: + continue + seen.add(key) + expanded.append(item) + return expanded + + +def _ipv4_mapped_v6(ipv4: str) -> str: + return f'::ffff:{ipv4}' + + +def encode_sw_static_for_c_core(addresses: Sequence[BackendAddress]) -> str: + """ + Encode a Node-style mixed endpoint list for grpcio/C-core. + + Homogeneous lists use ipv4:/ipv6:. Mixed IPv4+IPv6 uses the ipv6 resolver + with IPv4-mapped addresses so pick_first can try both families. + """ + if not addresses: + raise ValueError( + 'No valid collector backend address in SW_AGENT_COLLECTOR_BACKEND_SERVICES' + ) + if any(a.kind == AddressKind.HOSTNAME for a in addresses): + raise ValueError('encode_sw_static_for_c_core requires literal IP endpoints') + + kinds = {a.kind for a in addresses} + if kinds == {AddressKind.IPV4}: + return 'ipv4:' + ','.join(f'{a.host}:{a.port}' for a in addresses) + if kinds == {AddressKind.IPV6}: + return 'ipv6:' + ','.join(f'[{a.host}]:{a.port}' for a in addresses) + + parts = [] + for addr in addresses: + if addr.kind == AddressKind.IPV4: + parts.append(f'[{_ipv4_mapped_v6(addr.host)}]:{addr.port}') + else: + parts.append(f'[{addr.host}]:{addr.port}') + logger.info( + 'Encoding mixed-family sw-static endpoints for grpcio via ipv6 IPv4-mapped list: %s', + parts, + ) + return 'ipv6:' + ','.join(parts) + + +def prepare_grpc_channel_endpoints( + addresses: Sequence[BackendAddress], +) -> Tuple[str, str]: + """ + Build (channel_target, default_authority) from parsed backends. + + Authority prefers the first *usable* original entry's host:port (hostname kept + for :authority / TLS peer-name when that name resolved). Never points at a + hostname that DNS skipped. + """ + if not addresses: + raise ValueError( + 'No valid collector backend address in SW_AGENT_COLLECTOR_BACKEND_SERVICES' + ) + + if len(addresses) == 1: + ep = addresses[0].endpoint() + return ep, ep + + to_encode: List[BackendAddress] = [] + seen = set() + authority: Optional[str] = None + + for orig in addresses: + if orig.kind == AddressKind.HOSTNAME: + candidates = _lookup_hostname(orig.host, orig.port) + if not candidates: + continue + if authority is None: + # Prefer original hostname for :authority / SNI (Node sw-static style). + authority = orig.endpoint() + else: + candidates = (orig,) + if authority is None: + authority = orig.endpoint() + for item in candidates: + key = (item.kind, item.host, item.port) + if key in seen: + continue + seen.add(key) + to_encode.append(item) + + if not to_encode: + raise ValueError( + 'No usable collector backend address after DNS expansion of ' + 'SW_AGENT_COLLECTOR_BACKEND_SERVICES' + ) + if authority is None: + authority = to_encode[0].endpoint() + + if any(a.kind == AddressKind.HOSTNAME for a in addresses): + logger.info( + 'Expanded multi-backend collector addresses %s -> %s (authority=%s)', + [a.endpoint() for a in addresses], + [a.endpoint() for a in to_encode], + authority, + ) + return encode_sw_static_for_c_core(to_encode), authority + + +def _resolve_channel_target_and_authority() -> Tuple[str, str]: + """ + Never raise into the host app. prepare_grpc_channel_endpoints stays strict; + factories degrade so the agent can idle behind the READY gate. + """ + from skywalking import config + + raw = config.agent_collector_backend_services + addresses = parse_backend_addresses(raw) + try: + return prepare_grpc_channel_endpoints(addresses) + except ValueError: + if addresses: + target = addresses[0].endpoint() + logger.error( + 'No usable collector backend after DNS expansion of %r; ' + 'falling back to plain target %s so grpcio can re-resolve', + raw, + target, + ) + return target, target + fallback = (raw or '').strip() or 'localhost:1' + logger.error( + 'No valid collector backend address in %r; opening a channel to %s ' + '(agent stays up; READY gate skips reports)', + raw, + fallback, + ) + return fallback, fallback + + +def build_grpc_target(addresses: Sequence[BackendAddress]) -> str: + """Build a gRPC channel target string (see prepare_grpc_channel_endpoints).""" + target, _authority = prepare_grpc_channel_endpoints(addresses) + return target + + +def resolve_grpc_target(services: Optional[str] = None) -> str: + from skywalking import config + + raw = config.agent_collector_backend_services if services is None else services + return build_grpc_target(parse_backend_addresses(raw)) + + +def _channel_options(default_authority: str) -> Tuple[Tuple[str, int | str], ...]: + options = list(GRPC_CHANNEL_OPTIONS) + # Align with Node sw-static getDefaultAuthority (first usable backend). + # :authority and TLS peer-name (when this arg is set) use host:port here — + # do not set grpc.ssl_target_name_override (test-only / special dial cases). + options.append(('grpc.default_authority', default_authority)) + return tuple(options) + + +def create_sync_channel(): + """Create one sync gRPC channel (caller may wrap with auth interceptor).""" + target, authority = _resolve_channel_target_and_authority() + with agent_collector_channel_scope(): + try: + credentials = grpc_ssl_credentials() + options = _channel_options(authority) + logger.info('Creating gRPC channel to collector target %s (authority=%s)', target, authority) + if credentials is not None: + channel = grpc.secure_channel(target, credentials, options=options) + else: + channel = grpc.insecure_channel(target, options=options) + except Exception: # noqa: BLE001 - never fail host process start + logger.exception( + 'Failed to create gRPC channel to %s; using localhost:1 placeholder', + target, + ) + channel = grpc.insecure_channel('localhost:1', options=_channel_options(authority)) + return mark_agent_collector_channel(channel) + + +def create_aio_channel(interceptors=None): + """Create one aio gRPC channel with optional interceptors.""" + target, authority = _resolve_channel_target_and_authority() + with agent_collector_channel_scope(): + try: + credentials = grpc_ssl_credentials() + options = _channel_options(authority) + logger.info('Creating aio gRPC channel to collector target %s (authority=%s)', target, authority) + if credentials is not None: + channel = grpc.aio.secure_channel( + target, + credentials, + options=options, + interceptors=interceptors, + ) + else: + channel = grpc.aio.insecure_channel(target, options=options, interceptors=interceptors) + except Exception: # noqa: BLE001 - never fail host process start + logger.exception( + 'Failed to create aio gRPC channel to %s; using localhost:1 placeholder', + target, + ) + channel = grpc.aio.insecure_channel( + 'localhost:1', options=_channel_options(authority), interceptors=interceptors, + ) + return mark_agent_collector_channel(channel) + + +def _unwrap_connectivity_state(channel, try_to_connect: bool): + """ + Read channel connectivity for sync, aio, and intercept_channel wrappers. + + grpc.aio.Channel exposes get_state(). Sync grpc._channel.Channel does not + (subscribe only); use C-core check_connectivity_state on the cython channel. + Intercepted channels nest the real Channel on ``_channel``. + """ + get_state = getattr(channel, 'get_state', None) + if callable(get_state): + return get_state(try_to_connect) + + candidate = channel + for _ in range(4): + inner = getattr(candidate, '_channel', None) + if inner is None: + break + check = getattr(inner, 'check_connectivity_state', None) + if callable(check): + code = check(try_to_connect) + for state in grpc.ChannelConnectivity: + if state.value[0] == code: + return state + return None + candidate = inner + return None + + +def is_channel_ready(channel) -> bool: + """ + Application-level READY gate (gRPC only exposes connectivity state). + + Returns True only when connectivity is READY. Uses try_to_connect=True so an + IDLE channel is nudged into CONNECTING — otherwise skipping all RPCs would + leave the channel idle forever (same role as Node watchConnectivityState + with requestConnection=true). CONNECTING / TRANSIENT_FAILURE still return + False so reporters skip until READY. + + If connectivity cannot be read, returns True (fail-open) so a missing API + cannot permanently silence reporting. + """ + try: + state = _unwrap_connectivity_state(channel, True) + except Exception: # noqa: BLE001 - defensive for closed / exotic channels + return True + if state is None: + return True + return state == grpc.ChannelConnectivity.READY + + +def is_auth_rpc_error(error: BaseException) -> bool: + code = getattr(error, 'code', None) + if not callable(code): + return False + try: + status = code() + except Exception: # noqa: BLE001 - defensive for non-grpc exceptions + return False + return status in (grpc.StatusCode.UNAUTHENTICATED, grpc.StatusCode.PERMISSION_DENIED) + + +def log_auth_failure_throttled(error: BaseException) -> None: + """Auth failures must not rotate backends; same cluster shares one token.""" + global _last_auth_log_at + now = time.monotonic() + if now - _last_auth_log_at < _AUTH_LOG_INTERVAL_SEC: + return + _last_auth_log_at = now + logger.error( + 'Collector rejected authentication (%s). Check SW_AGENT_AUTHENTICATION; ' + 'the agent will not rotate backends for auth failures.', + error, + ) + + +def log_connectivity_event(kind: str, message: str, *args) -> None: + """Throttle disconnect warnings; recovery stays informative but rate-limited.""" + now = time.monotonic() + last = _last_connectivity_log_at.get(kind, 0.0) + if now - last < _CONNECTIVITY_LOG_INTERVAL_SEC: + return + _last_connectivity_log_at[kind] = now + if kind == 'recovered': + logger.info(message, *args) + else: + logger.warning(message, *args) + + +def apply_connectivity_transition(prev, state) -> None: + """Shared INFO/WARN side-effects for sync subscribe and aio watch.""" + if state == grpc.ChannelConnectivity.TRANSIENT_FAILURE: + log_connectivity_event( + 'transient_failure', + 'gRPC collector channel disconnected (TRANSIENT_FAILURE)', + ) + elif state == grpc.ChannelConnectivity.IDLE and prev == grpc.ChannelConnectivity.READY: + log_connectivity_event('idle', 'gRPC collector channel disconnected (IDLE)') + elif state == grpc.ChannelConnectivity.READY and prev in ( + grpc.ChannelConnectivity.TRANSIENT_FAILURE, + grpc.ChannelConnectivity.IDLE, + grpc.ChannelConnectivity.CONNECTING, + None, + ): + log_connectivity_event('recovered', 'gRPC collector channel recovered (READY)') + + +def handle_rpc_error(error: BaseException, on_connectivity_error) -> None: + """ + Shared RpcError side-effects for sync/aio reporters. + Auth: throttle log only (no channel rebuild / backend rotate). + Other: invoke connectivity recovery hook (resubscribe / debug), never rebuild channel. + + Failover is for an unreachable backend, not functional health of a connected one. + DEADLINE_EXCEEDED and RESOURCE_EXHAUSTED on a READY channel are intentionally + left to the call site: pick_first will not move off a slow-but-READY backend, + and this agent does not rotate or rebuild the channel for those codes. + """ + if is_auth_rpc_error(error): + log_auth_failure_throttled(error) + return + on_connectivity_error() diff --git a/skywalking/utils/tls.py b/skywalking/utils/tls.py new file mode 100644 index 000000000..27b14f3fb --- /dev/null +++ b/skywalking/utils/tls.py @@ -0,0 +1,678 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +""" +Collector TLS / mTLS helpers for gRPC and HTTP reporters. + +Rules: +- TLS when FORCE_TLS is set or the trusted CA file exists. +- Custom CA is used only when that file exists *and* parses as a CA PEM + (otherwise system trust with FORCE_TLS, or plaintext without it). +- mTLS (client cert + key) is enabled only when a usable custom CA was loaded + *and* both cert-chain and key files exist and parse. Missing / invalid + cert/key logs a warning and stays one-way TLS — it does not fail agent start. +- A non-empty CA / cert / key path that cannot be resolved logs a warning + (misconfig must not look like "TLS off by design"). +- Invalid PEM content, SSLContext build failures, and temp-file I/O errors + never raise into the host process: warn and degrade (see degrade table in + ``ssl_context_for_collector`` / ``requests_tls_settings``). +- HTTP mTLS temp PEMs are process-local: after ``os.fork()`` the child rebinds + its temp-file list so exiting the child cannot unlink the parent's files. +""" + +from __future__ import annotations + +import atexit +import base64 +import os +import ssl +import tempfile +from pathlib import Path +from typing import List, Optional, Tuple + +from skywalking.loggings import logger + +# Align with Node TlsMaterialCache: refuse oversized PEMs (DoS / misconfig). +_MAX_PEM_BYTES = 256 * 1024 +# Avoid repeating the same misconfig warning on every reporter call. +_warned_keys: set = set() + + +def _warn_once(key: str, message: str, *args) -> None: + if key in _warned_keys: + return + _warned_keys.add(key) + logger.warning(message, *args) + + +def _configured_path(value: str) -> str: + return (value or '').strip() + + +def ssl_file_path(value: str) -> Optional[Path]: + """ + Return a readable regular file path, or None. + + Paths are cwd-relative or absolute. Symlinks are followed (Kubernetes + secret mounts are typically ``ca.crt -> ..data/ca.crt``); the resolved + target must be a regular file. + + Never raises: ``expanduser`` / ``resolve`` failures (including symlink + loops and unresolvable ``~user`` → ``RuntimeError``) degrade to None so + TLS misconfig cannot abort agent start. + """ + text = _configured_path(value) + if not text: + return None + try: + path = Path(text).expanduser() + resolved = path.resolve(strict=True) + if resolved.is_file(): + return resolved + except (OSError, ValueError, RuntimeError): + # OSError: missing path / I/O. ValueError: embedded NUL. RuntimeError: + # expanduser ~user miss or symlink-loop resolve on some platforms. + return None + return None + + +def collector_uses_tls() -> bool: + from skywalking import config + + force = bool(config.agent_force_tls) + ca_cfg = _configured_path(config.agent_ssl_trusted_ca_path) + ca_path = ssl_file_path(ca_cfg) if ca_cfg else None + if ca_cfg and ca_path is None: + if force: + _warn_once( + f'ca-missing:{ca_cfg}', + 'SW_AGENT_SSL_TRUSTED_CA_PATH is set (%r) but is not a readable regular ' + 'file; custom CA is ignored. FORCE_TLS remains on with the process trust store.', + ca_cfg, + ) + else: + _warn_once( + f'ca-missing:{ca_cfg}', + 'SW_AGENT_SSL_TRUSTED_CA_PATH is set (%r) but is not a readable regular ' + 'file; TLS via custom CA is disabled and the collector stays plaintext ' + '(set SW_AGENT_FORCE_TLS to use the process trust store).', + ca_cfg, + ) + return force or ca_path is not None + + +def _read_bytes(path: Path) -> bytes: + size = os.path.getsize(path) + if size > _MAX_PEM_BYTES: + raise ValueError( + f'SSL PEM file exceeds {_MAX_PEM_BYTES} bytes: {path} ({size} bytes)' + ) + return path.read_bytes() + + +def _load_trusted_ca(path: Path) -> bytes: + """ + Read and parse a trusted CA PEM. + + Raises OSError / ValueError when the file is unreadable, oversized, or not + a CA bundle OpenSSL can load — callers degrade instead of handing garbage + to gRPC / requests (which may only fail at connect time). + """ + data = _read_bytes(path) + try: + ssl.create_default_context(cafile=str(path)) + except OSError as exc: + # ssl.SSLError subclasses OSError on CPython. + raise ValueError(f'Invalid trusted CA PEM {path}: {exc}') from exc + return data + + +def _validate_client_cert_key(cert_pem: bytes, key_pem: bytes) -> None: + """Raise OSError / ValueError when cert+key cannot load as a client identity.""" + cert_path = key_path = None + try: + fd, cert_path = tempfile.mkstemp(suffix='.crt') + try: + os.write(fd, cert_pem) + finally: + os.close(fd) + fd, key_path = tempfile.mkstemp(suffix='.pem') + try: + os.write(fd, key_pem) + finally: + os.close(fd) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_cert_chain(cert_path, key_path) + finally: + for path in (cert_path, key_path): + if path is None: + continue + try: + os.unlink(path) + except OSError: + pass + + +# OpenSSL PKCS#1 PEM → PKCS#8 PEM for stacks that only accept PKCS#8. +# grpcio/BoringSSL and CPython ssl both accept PKCS#1 RSA PEMs directly; this +# wrap exists for HTTP stacks / older OpenSSL bindings that reject +# ``BEGIN RSA PRIVATE KEY`` and only load PKCS#8 ``BEGIN PRIVATE KEY``. +_PKCS1_PEM_HEADER = '-----BEGIN RSA PRIVATE KEY-----' +_PKCS1_PEM_FOOTER = '-----END RSA PRIVATE KEY-----' +_PKCS8_PEM_HEADER = '-----BEGIN PRIVATE KEY-----' +_PKCS8_PEM_FOOTER = '-----END PRIVATE KEY-----' +_ENCRYPTED_PEM_HEADER = '-----BEGIN ENCRYPTED PRIVATE KEY-----' + + +def normalize_private_key_pem(key_pem: bytes) -> bytes: + """ + Return PKCS#8 PEM bytes. PKCS#1 (`BEGIN RSA PRIVATE KEY`) is wrapped into + an unencrypted PKCS#8 PEM; other PEM/DER forms pass through. + + Passphrase-encrypted PEMs (PKCS#8 encrypted or legacy OpenSSL Proc-Type) + are rejected with a clear error (not supported). + """ + text = key_pem.decode('utf-8', errors='ignore') + if _ENCRYPTED_PEM_HEADER in text: + raise ValueError( + 'Passphrase-encrypted private keys are not supported; ' + 'use an unencrypted PEM (PKCS#8 or PKCS#1 RSA)' + ) + # Legacy OpenSSL ``openssl genrsa -aes256`` / ``rsa -des3`` output. + if 'Proc-Type:' in text and 'ENCRYPTED' in text.upper(): + raise ValueError( + 'Passphrase-encrypted private keys are not supported; ' + 'use an unencrypted PEM (PKCS#8 or PKCS#1 RSA)' + ) + if _PKCS1_PEM_HEADER not in text: + return key_pem + + body = text.replace(_PKCS1_PEM_HEADER, '').replace(_PKCS1_PEM_FOOTER, '') + body = body.replace('\r', '').replace('\n', '').replace(' ', '') + try: + pkcs1 = base64.b64decode(body) + except ValueError as exc: + raise ValueError(f'Invalid PKCS#1 private key PEM: {exc}') from exc + + pkcs1_length = len(pkcs1) + total_length = pkcs1_length + 22 + # Fixed 2-byte length form (keys up to ~64KiB DER). + if total_length > 0xFFFF or pkcs1_length > 0xFFFF: + raise ValueError('PKCS#1 private key too large for PKCS#8 wrap') + + # Hand-rolled PKCS#8 PrivateKeyInfo wrapping RSA PKCS#1 (AlgorithmIdentifier + # rsaEncryption). Verified byte-identical to cryptography for 2048/4096 RSA. + pkcs8_header = bytes([ + 0x30, 0x82, (total_length >> 8) & 0xFF, total_length & 0xFF, + 0x02, 0x01, 0x00, + 0x30, 0x0D, 0x06, 0x09, 0x2A, 0x86, 0x48, 0x86, 0xF7, 0x0D, 0x01, 0x01, 0x01, 0x05, 0x00, + 0x04, 0x82, (pkcs1_length >> 8) & 0xFF, pkcs1_length & 0xFF, + ]) + encoded = base64.b64encode(pkcs8_header + pkcs1).decode('ascii') + # 64-char lines, matching typical PEM layout. + lines = [encoded[i:i + 64] for i in range(0, len(encoded), 64)] + pem = _PKCS8_PEM_HEADER + '\n' + '\n'.join(lines) + '\n' + _PKCS8_PEM_FOOTER + '\n' + return pem.encode('ascii') + + +def _mtls_material(*, ca_usable: bool) -> Tuple[Optional[bytes], Optional[bytes]]: + """ + Client certificate_chain and private_key PEM bytes, or (None, None). + + mTLS is considered only when a custom CA was successfully loaded and parsed + (``ca_usable``). A CA path that exists but failed to load must not unlock + client certs for FORCE_TLS / system-trust fallback. + Private keys are normalized to PKCS#8 when the file is PKCS#1. + """ + from skywalking import config + + cert_cfg = _configured_path(config.agent_ssl_cert_chain_path) + key_cfg = _configured_path(config.agent_ssl_key_path) + + if not ca_usable: + # Client material without a usable CA cannot enable mTLS (by design). + if cert_cfg or key_cfg: + _warn_once( + f'mtls-without-ca:{cert_cfg}|{key_cfg}', + 'Client SSL cert/key is configured but SW_AGENT_SSL_TRUSTED_CA_PATH ' + 'is missing, unreadable, or not a valid CA PEM; mTLS is disabled. ' + 'Provide a readable CA PEM to enable mTLS (FORCE_TLS alone uses ' + 'system trust for one-way TLS).', + ) + return None, None + + if bool(cert_cfg) ^ bool(key_cfg): + _warn_once( + f'mtls-partial:{cert_cfg}|{key_cfg}', + 'Only one of SW_AGENT_SSL_CERT_CHAIN_PATH / SW_AGENT_SSL_KEY_PATH is set; ' + 'mTLS requires both. Staying on one-way TLS.', + ) + return None, None + + if not cert_cfg or not key_cfg: + return None, None + + cert_path = ssl_file_path(cert_cfg) + key_path = ssl_file_path(key_cfg) + if cert_path is None or key_path is None: + logger.warning('Failed to enable mTLS caused by cert or key cannot be found.') + return None, None + + try: + cert_pem = _read_bytes(cert_path) + key_pem = normalize_private_key_pem(_read_bytes(key_path)) + _validate_client_cert_key(cert_pem, key_pem) + return cert_pem, key_pem + except (OSError, ValueError) as exc: + logger.warning('Failed to enable mTLS caused by cert or key read error: %s', exc) + return None, None + + +# Keep mTLS temp PEM paths alive for the process (requests/ssl need file paths). +_mtls_temp_files: List[str] = [] +_mtls_file_cache_key: Optional[Tuple[bytes, bytes]] = None +_mtls_file_cache: Optional[Tuple[str, str]] = None +_atexit_registered = False + + +def _cleanup_mtls_temp_files() -> None: + for path in list(_mtls_temp_files): + try: + os.unlink(path) + except OSError: + pass + _mtls_temp_files.clear() + + +def _after_fork_in_child() -> None: + """ + Drop inherited temp-path bookkeeping without unlinking. + + Parent-created PEM temps must stay on disk for the parent reporter. Clearing + the list in-place would also empty the parent's list (same object); rebind + instead so the child's atexit handler cannot delete the parent's files. + """ + global _mtls_temp_files, _mtls_file_cache, _mtls_file_cache_key + _mtls_temp_files = [] + _mtls_file_cache = None + _mtls_file_cache_key = None + + +if hasattr(os, 'register_at_fork'): + os.register_at_fork(after_in_child=_after_fork_in_child) + + +def _ensure_atexit() -> None: + global _atexit_registered + if not _atexit_registered: + atexit.register(_cleanup_mtls_temp_files) + _atexit_registered = True + + +def _pem_bytes_to_temp_file(data: bytes, suffix: str) -> str: + _ensure_atexit() + handle, path = tempfile.mkstemp(suffix=suffix) + try: + view = memoryview(data) + offset = 0 + while offset < len(view): + written = os.write(handle, view[offset:]) + if written <= 0: + raise OSError(f'Failed to write SSL PEM temp file {path}') + offset += written + except Exception: + os.close(handle) + try: + os.unlink(path) + except OSError: + pass + raise + else: + os.close(handle) + _mtls_temp_files.append(path) + return path + + +def _mtls_cert_key_files( + certificate_chain: bytes, + private_key: bytes, +) -> Optional[Tuple[str, str]]: + """ + Write normalized PEM bytes to temp files (cached per material). + + Returns None when temp files cannot be created (caller stays one-way TLS). + """ + global _mtls_file_cache_key, _mtls_file_cache + + cache_key = (certificate_chain, private_key) + if _mtls_file_cache is not None and _mtls_file_cache_key == cache_key: + return _mtls_file_cache + + try: + paths = ( + _pem_bytes_to_temp_file(certificate_chain, '.crt'), + _pem_bytes_to_temp_file(private_key, '.pem'), + ) + except OSError as exc: + _warn_once( + f'mtls-temp:{exc}', + 'Failed to write mTLS cert/key temp files (%s); staying on one-way TLS.', + exc, + ) + return None + + _mtls_file_cache_key = cache_key + _mtls_file_cache = paths + return paths + + +def tls_pem_material() -> Optional[Tuple[Optional[bytes], Optional[bytes], Optional[bytes]]]: + """ + PEM bytes for (root_certificates, private_key, certificate_chain). + + None means plaintext (no TLS). Tuple members may still be None when using + FORCE_TLS with the process trust store and/or one-way TLS. + + CA / client PEMs are parsed before return so invalid content degrades here + (plaintext or system-trust one-way TLS) instead of only failing at connect. + + Enable decision uses the *current* CA load result (not a prior existence + check alone): if the CA path races away between resolve and read without + FORCE_TLS, returns None (plaintext) rather than a system-trust HTTPS tuple. + """ + from skywalking import config + + # Emit configured-but-unusable CA path warnings (warn_once dedupes). + collector_uses_tls() + + force = bool(config.agent_force_tls) + ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) + root_certificates = None + ca_usable = False + if ca_path is not None: + try: + root_certificates = _load_trusted_ca(ca_path) + ca_usable = True + except (OSError, ValueError) as exc: + if force: + logger.warning( + 'Failed to load trusted CA file (%s); continuing with FORCE_TLS ' + 'and the process trust store (custom CA ignored, mTLS disabled).', + exc, + ) + else: + logger.warning( + 'Failed to load trusted CA file (%s); collector stays plaintext ' + '(set SW_AGENT_FORCE_TLS to use the process trust store instead).', + exc, + ) + # Still surface mTLS-without-CA diagnostics when client PEMs are set. + _mtls_material(ca_usable=False) + return None + + if not force and not ca_usable: + # No TLS without FORCE or a usable CA — including CA that vanished after + # collector_uses_tls() saw it (TOCTOU). Warn if client cert/key are set. + _mtls_material(ca_usable=False) + return None + + certificate_chain, private_key = _mtls_material(ca_usable=ca_usable) + return root_certificates, private_key, certificate_chain + + +def grpc_ssl_credentials(): + """ + grpc.ChannelCredentials for TLS/mTLS, or None for plaintext. + + FORCE_TLS without a CA file uses the process trust store + (grpc.ssl_channel_credentials() with no roots). + + If credential construction ultimately fails (including process-trust + FORCE_TLS fallback), returns None so the channel stays plaintext with a + warning — never raises into agent start. + """ + import grpc + + material = tls_pem_material() + if material is None: + return None + + root_certificates, private_key, certificate_chain = material + try: + return grpc.ssl_channel_credentials( + root_certificates=root_certificates, + private_key=private_key, + certificate_chain=certificate_chain, + ) + except Exception as exc: # noqa: BLE001 - never fail host process start + from skywalking import config + + if private_key is not None or certificate_chain is not None: + _warn_once( + f'grpc-creds-mtls:{exc}', + 'Failed to build gRPC mTLS credentials (%s); retrying one-way TLS.', + exc, + ) + try: + return grpc.ssl_channel_credentials(root_certificates=root_certificates) + except Exception as one_way_exc: # noqa: BLE001 + _warn_once( + f'grpc-creds-oneway:{one_way_exc}', + 'Failed to build gRPC TLS credentials (%s); collector stays plaintext.', + one_way_exc, + ) + return None + if config.agent_force_tls: + _warn_once( + f'grpc-creds-force:{exc}', + 'Failed to build gRPC TLS credentials with custom CA (%s); ' + 'continuing with FORCE_TLS and the process trust store.', + exc, + ) + try: + return grpc.ssl_channel_credentials() + except Exception as force_exc: # noqa: BLE001 + _warn_once( + f'grpc-creds-force-fail:{force_exc}', + 'Failed to build gRPC TLS credentials (%s); collector stays plaintext.', + force_exc, + ) + return None + _warn_once( + f'grpc-creds:{exc}', + 'Failed to build gRPC TLS credentials (%s); collector stays plaintext.', + exc, + ) + return None + + +def collector_http_scheme() -> str: + """ + ``https://`` when TLS material is enabled, else ``http://``. + + Never raises into reporter ``__init__`` (HTTP clients call this before + other TLS helpers that may wrap failures). + """ + try: + return 'https://' if tls_pem_material() is not None else 'http://' + except Exception as exc: # noqa: BLE001 - never fail host process start + from skywalking import config + + _warn_once( + f'http-scheme:{exc}', + 'Failed to decide collector HTTP scheme (%s); falling back.', + exc, + ) + return 'https://' if config.agent_force_tls else 'http://' + + +def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: + """ + (verify, cert) for requests.Session. + + verify is True (system CAs), a CA file path, or unused for plaintext callers. + cert is (cert_path, key_path) when mTLS files are present. + + Keeps the same enable/disable decision as grpc_ssl_credentials / tls_pem_material + so an unreadable or oversized CA cannot leave HTTP on https:// with a bad verify path. + + When custom CA bytes were loaded but the CA path is no longer readable (symlink + race), verify uses a process-local temp PEM of those bytes instead of silently + falling back to the system trust store. + + Temp-file failures drop client certs only (one-way TLS), with a warning. + """ + from skywalking import config + + material = tls_pem_material() + if material is None: + return True, None + + root_certificates, private_key, certificate_chain = material + ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) + if root_certificates is not None: + if ca_path is not None: + verify: object = str(ca_path) + else: + # Prefer in-memory CA over system trust when the path raced away. + try: + verify = _pem_bytes_to_temp_file(root_certificates, '.crt') + except OSError as exc: + _warn_once( + f'requests-ca-temp:{exc}', + 'Failed to persist trusted CA temp file (%s); ' + 'using process trust store for HTTP verify.', + exc, + ) + verify = True + else: + verify = True + + if private_key is None or certificate_chain is None: + return verify, None + + pair = _mtls_cert_key_files(certificate_chain, private_key) + return verify, pair + + +def configure_requests_session(session) -> None: + try: + verify, cert = requests_tls_settings() + except Exception as exc: # noqa: BLE001 - never fail host process start + from skywalking import config + + _warn_once( + f'requests-tls:{exc}', + 'Failed to configure HTTP reporter TLS (%s); falling back.', + exc, + ) + if config.agent_force_tls or ssl_file_path(config.agent_ssl_trusted_ca_path): + # Prefer system-trust https over aborting agent start. + session.verify = True + session.cert = None + return + session.verify = verify + if cert is not None: + session.cert = cert + + +def ssl_context_for_collector() -> Optional[ssl.SSLContext]: + """ + stdlib SSLContext for aiohttp, or None when the collector stays plaintext. + + Degrade on SSLError / OSError (never raise into agent bootstrap): + - bad / unparsable custom CA + FORCE_TLS → process trust store, no client cert + - bad / unparsable custom CA without FORCE_TLS → plaintext (None) + - bad client cert/key or temp-file failure → one-way TLS (CA or system trust) + + Custom CA is loaded from in-memory PEM bytes (``cadata``) when present so a + path race after ``tls_pem_material`` cannot silently switch to system trust. + """ + from skywalking import config + + material = tls_pem_material() + if material is None: + return None + + root_certificates, private_key, certificate_chain = material + + def _system_trust_context() -> ssl.SSLContext: + return ssl.create_default_context() + + ctx: Optional[ssl.SSLContext] + if root_certificates is not None: + try: + # cadata avoids re-reading the CA path (TOCTOU with K8s secret mounts). + ctx = ssl.create_default_context( + cadata=root_certificates.decode('ascii', errors='strict'), + ) + except (OSError, ValueError) as exc: + # ssl.SSLError subclasses OSError; ValueError: non-ASCII PEM bytes. + if config.agent_force_tls: + _warn_once( + f'ssl-ctx-ca:{exc}', + 'Failed to load trusted CA into SSLContext (%s); continuing with ' + 'FORCE_TLS and the process trust store (mTLS disabled).', + exc, + ) + try: + ctx = _system_trust_context() + except OSError as sys_exc: + _warn_once( + f'ssl-ctx-system:{sys_exc}', + 'Failed to create SSLContext from process trust store (%s); ' + 'collector stays plaintext.', + sys_exc, + ) + return None + private_key = None + certificate_chain = None + else: + _warn_once( + f'ssl-ctx-ca-plain:{exc}', + 'Failed to load trusted CA into SSLContext (%s); collector stays ' + 'plaintext (set SW_AGENT_FORCE_TLS to use the process trust store).', + exc, + ) + return None + else: + # FORCE_TLS without usable CA → process trust store. + try: + ctx = _system_trust_context() + except OSError as exc: + _warn_once( + f'ssl-ctx-system:{exc}', + 'Failed to create SSLContext from process trust store (%s); ' + 'collector stays plaintext.', + exc, + ) + return None + + if private_key is not None and certificate_chain is not None: + pair = _mtls_cert_key_files(certificate_chain, private_key) + if pair is None: + return ctx + cert_file, key_file = pair + try: + ctx.load_cert_chain(cert_file, key_file) + except OSError as exc: + _warn_once( + f'ssl-ctx-mtls:{exc}', + 'Failed to load client cert/key into SSLContext (%s); staying on one-way TLS.', + exc, + ) + return ctx diff --git a/tests/e2e/case/expected/dependency-services-consumer.yml b/tests/e2e/case/expected/dependency-services-consumer.yml new file mode 100644 index 000000000..626e20f42 --- /dev/null +++ b/tests/e2e/case/expected/dependency-services-consumer.yml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pinned SW_CTL returns empty component lists; include [] so map compare matches. +nodes: +{{- contains .nodes }} +- id: {{ b64enc "User"}}.0 + name: User + type: USER + isreal: false +- id: {{ b64enc "e2e-service-consumer|namespace"}}.1 + name: "e2e-service-consumer|namespace" + type: {{ notEmpty .type }} + isreal: true +- id: {{ b64enc "e2e-service-provider"}}.1 + name: e2e-service-provider + type: {{ notEmpty .type }} + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer|namespace"}}.1 + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider"}}.1 + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer|namespace"}}.1-{{ b64enc "e2e-service-provider"}}.1 + detectpoints: + {{- contains .detectpoints }} + - {{ notEmpty . }} + {{- end }} +- source: {{ b64enc "User" }}.0 + sourcecomponents: [] + target: {{ b64enc "e2e-service-consumer|namespace"}}.1 + targetcomponents: [] + id: {{ b64enc "User" }}.0-{{ b64enc "e2e-service-consumer|namespace"}}.1 + detectpoints: + {{- contains .detectpoints }} + - {{ notEmpty . }} + {{- end }} +{{- end }} diff --git a/tests/e2e/case/expected/dependency-services-provider.yml b/tests/e2e/case/expected/dependency-services-provider.yml new file mode 100644 index 000000000..9885acabc --- /dev/null +++ b/tests/e2e/case/expected/dependency-services-provider.yml @@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Pinned SW_CTL returns empty component lists; include [] so map compare matches. +nodes: +{{- contains .nodes }} +- id: {{ b64enc "e2e-service-provider"}}.1 + name: e2e-service-provider + type: {{ notEmpty .type }} + isreal: true +- id: {{ b64enc "e2e-service-consumer|namespace"}}.1 + name: "e2e-service-consumer|namespace" + type: {{ notEmpty .type }} + isreal: true +{{- end }} +calls: +{{- contains .calls }} +- source: {{ b64enc "e2e-service-consumer|namespace"}}.1 + sourcecomponents: [] + target: {{ b64enc "e2e-service-provider"}}.1 + targetcomponents: [] + id: {{ b64enc "e2e-service-consumer|namespace"}}.1-{{ b64enc "e2e-service-provider"}}.1 + detectpoints: + {{- contains .detectpoints }} + - {{ notEmpty . }} + {{- end }} +{{- end }} diff --git a/tests/e2e/case/expected/layer.yml b/tests/e2e/case/expected/layer.yml new file mode 100644 index 000000000..be0bda350 --- /dev/null +++ b/tests/e2e/case/expected/layer.yml @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- contains . }} +- GENERAL +{{- end }} diff --git a/tests/e2e/case/expected/service-endpoint-consumer.yml b/tests/e2e/case/expected/service-endpoint-consumer.yml new file mode 100644 index 000000000..8da9d0643 --- /dev/null +++ b/tests/e2e/case/expected/service-endpoint-consumer.yml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + {{- contains . }} +- id: {{ b64enc "e2e-service-consumer|namespace" }}.1_{{ b64enc "/artist-consumer" }} + name: /artist-consumer + {{- end }} diff --git a/tests/e2e/case/expected/service-instance-consumer.yml b/tests/e2e/case/expected/service-instance-consumer.yml new file mode 100644 index 000000000..0bf021c6b --- /dev/null +++ b/tests/e2e/case/expected/service-instance-consumer.yml @@ -0,0 +1,39 @@ +# Licensed to Apache Software Foundation (ASF) under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Apache Software Foundation (ASF) licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +{{- contains . }} +# id is base64 encoded version of e2e-service-consumer|namespace=._instance_id +- id: {{ regexp .id "ZTJlLXNlcnZpY2UtY29uc3VtZXJ8bmFtZXNwYWNl.*" }} + name: {{ notEmpty .name }} + attributes: + {{- contains .attributes }} + - name: OS Name + value: "posix" + - name: Process No. + value: {{ notEmpty .value }} + - name: hostname + value: {{ notEmpty .value }} + - name: python_implementation + value: "CPython" + - name: python_version + value: {{ notEmpty .value }} + - name: ipv4s + value: {{ notEmpty .value }} + {{- end }} + language: PYTHON + instanceuuid: {{ regexp .instanceuuid "ZTJlLXNlcnZpY2UtY29uc3VtZXJ8bmFtZXNwYWNl.*" }} +{{- end}} diff --git a/tests/e2e/case/grpc/mtls/docker-compose.grpc.mtls.yaml b/tests/e2e/case/grpc/mtls/docker-compose.grpc.mtls.yaml new file mode 100644 index 000000000..bf32149b8 --- /dev/null +++ b/tests/e2e/case/grpc/mtls/docker-compose.grpc.mtls.yaml @@ -0,0 +1,98 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +services: + + # Ephemeral PEMs — not committed. Script: tests/e2e/script/prepare/gen-e2e-tls-certs.sh + # Image: alpine/openssl (openssl preinstalled; digest-pinned; no apk). + gen-certs: + image: docker.io/alpine/openssl@sha256:80b347d7b4d58e28aae515cf808ba3ce1c4a2a69a323b2cb2cb2d6d278729684 + networks: + - e2e + volumes: + - ../../../script/prepare/gen-e2e-tls-certs.sh:/scripts/gen-e2e-tls-certs.sh:ro + - ./server:/out/server + - ./client:/out/client + entrypoint: ["/bin/sh"] + command: ["/scripts/gen-e2e-tls-certs.sh", "mtls"] + + oap: + extends: + file: ../../../base/docker-compose.base.yml + service: oap + volumes: + - ./server:/skywalking/certs + environment: + # mTLS requires receiver-sharing-server. + SW_RECEIVER_GRPC_PORT: 11811 + SW_RECEIVER_GRPC_SSL_ENABLED: "true" + SW_RECEIVER_GRPC_SSL_KEY_PATH: /skywalking/certs/server.pem + SW_RECEIVER_GRPC_SSL_CERT_CHAIN_PATH: /skywalking/certs/server.crt + SW_RECEIVER_GRPC_SSL_TRUSTED_CAS_PATH: /skywalking/certs/ca.crt + expose: + - 11811 + depends_on: + gen-certs: + condition: service_completed_successfully + # Probe the sharing-server mTLS port (not core 11800 from the base service). + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/11811" ] + interval: 5s + timeout: 60s + retries: 120 + ports: + - "12800" + + provider: + extends: + file: ../../../base/docker-compose.base.yml + service: fastapi-provider + volumes: + - ./client:/certs + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11811 + SW_AGENT_SSL_TRUSTED_CA_PATH: /certs/ca.crt + SW_AGENT_SSL_KEY_PATH: /certs/client.pem + SW_AGENT_SSL_CERT_CHAIN_PATH: /certs/client.crt + depends_on: + gen-certs: + condition: service_completed_successfully + oap: + condition: service_healthy + ports: + - "9090" + + consumer: + extends: + file: ../../../base/docker-compose.base.yml + service: flask-consumer + volumes: + - ./client:/certs + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11811 + SW_AGENT_SSL_TRUSTED_CA_PATH: /certs/ca.crt + SW_AGENT_SSL_KEY_PATH: /certs/client.pem + SW_AGENT_SSL_CERT_CHAIN_PATH: /certs/client.crt + depends_on: + provider: + condition: service_healthy + ports: + - "9090" + +networks: + e2e: diff --git a/tests/e2e/case/grpc/mtls/e2e.yaml b/tests/e2e/case/grpc/mtls/e2e.yaml new file mode 100644 index 000000000..17ad54baf --- /dev/null +++ b/tests/e2e/case/grpc/mtls/e2e.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gRPC mTLS against real OAP sharing server. + +setup: + env: compose + file: docker-compose.grpc.mtls.yaml + timeout: 20m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/usr/local/bin:/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash tests/e2e/script/prepare/install.sh yq + - name: install swctl + command: bash tests/e2e/script/prepare/install.sh swctl + +trigger: + action: http + # Continuous traffic so metrics have ongoing load. + interval: 3s + times: -1 + url: http://${consumer_host}:${consumer_9090}/artist-consumer + method: POST + headers: + "Content-Type": "application/json" + body: '{"song": "Despacito"}' + +verify: + retry: + count: 20 + interval: 10s + cases: + # Full signal packs (grpc/single) plus tls-depth extras. + - includes: + - ../../logging-cases.yaml + - ../../metadata-cases.yaml + - ../../tls-depth-cases.yaml + - ../../metrics-cases.yaml + - ../../pvm-cases.yaml + - ../../tracing-cases.yaml diff --git a/tests/e2e/case/grpc/ssl/docker-compose.grpc.ssl.yaml b/tests/e2e/case/grpc/ssl/docker-compose.grpc.ssl.yaml new file mode 100644 index 000000000..47ef88086 --- /dev/null +++ b/tests/e2e/case/grpc/ssl/docker-compose.grpc.ssl.yaml @@ -0,0 +1,86 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +services: + + # Ephemeral PEMs — not committed. Script: tests/e2e/script/prepare/gen-e2e-tls-certs.sh + # Image: alpine/openssl (openssl preinstalled; digest-pinned; no apk). + gen-certs: + image: docker.io/alpine/openssl@sha256:80b347d7b4d58e28aae515cf808ba3ce1c4a2a69a323b2cb2cb2d6d278729684 + networks: + - e2e + volumes: + - ../../../script/prepare/gen-e2e-tls-certs.sh:/scripts/gen-e2e-tls-certs.sh:ro + - ./certs:/out/certs + - ./ca:/out/ca + entrypoint: ["/bin/sh"] + command: ["/scripts/gen-e2e-tls-certs.sh", "ssl"] + + oap: + extends: + file: ../../../base/docker-compose.base.yml + service: oap + volumes: + - ./certs:/skywalking/certs + environment: + SW_CORE_GRPC_SSL_ENABLED: "true" + SW_CORE_GRPC_SSL_KEY_PATH: /skywalking/certs/server-key.pem + SW_CORE_GRPC_SSL_CERT_CHAIN_PATH: /skywalking/certs/server.crt + # Required whenever core gRPC SSL is on: OAP still builds a cluster-client + # SSL context from this path (empty path crashes bootstrap). Not agent mTLS. + SW_CORE_GRPC_SSL_TRUSTED_CA_PATH: /skywalking/certs/ca.crt + depends_on: + gen-certs: + condition: service_completed_successfully + ports: + - "12800" + + provider: + extends: + file: ../../../base/docker-compose.base.yml + service: fastapi-provider + volumes: + - ./ca:/certs + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_AGENT_SSL_TRUSTED_CA_PATH: /certs/ca.crt + depends_on: + gen-certs: + condition: service_completed_successfully + oap: + condition: service_healthy + ports: + - "9090" + + consumer: + extends: + file: ../../../base/docker-compose.base.yml + service: flask-consumer + volumes: + - ./ca:/certs + environment: + SW_AGENT_COLLECTOR_BACKEND_SERVICES: oap:11800 + SW_AGENT_SSL_TRUSTED_CA_PATH: /certs/ca.crt + depends_on: + provider: + condition: service_healthy + ports: + - "9090" + +networks: + e2e: diff --git a/tests/e2e/case/grpc/ssl/e2e.yaml b/tests/e2e/case/grpc/ssl/e2e.yaml new file mode 100644 index 000000000..0eb43a7ef --- /dev/null +++ b/tests/e2e/case/grpc/ssl/e2e.yaml @@ -0,0 +1,54 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# gRPC one-way TLS against real OAP. + +setup: + env: compose + file: docker-compose.grpc.ssl.yaml + timeout: 20m + init-system-environment: ../../../script/env + steps: + - name: set PATH + command: export PATH=/usr/local/bin:/tmp/skywalking-infra-e2e/bin:$PATH + - name: install yq + command: bash tests/e2e/script/prepare/install.sh yq + - name: install swctl + command: bash tests/e2e/script/prepare/install.sh swctl + +trigger: + action: http + # Continuous traffic so metrics have ongoing load. + interval: 3s + times: -1 + url: http://${consumer_host}:${consumer_9090}/artist-consumer + method: POST + headers: + "Content-Type": "application/json" + body: '{"song": "Despacito"}' + +verify: + retry: + count: 20 + interval: 10s + cases: + # Full signal packs (grpc/single) plus tls-depth extras. + - includes: + - ../../logging-cases.yaml + - ../../metadata-cases.yaml + - ../../tls-depth-cases.yaml + - ../../metrics-cases.yaml + - ../../pvm-cases.yaml + - ../../tracing-cases.yaml diff --git a/tests/e2e/case/tls-depth-cases.yaml b/tests/e2e/case/tls-depth-cases.yaml new file mode 100644 index 000000000..e86f77549 --- /dev/null +++ b/tests/e2e/case/tls-depth-cases.yaml @@ -0,0 +1,61 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Extra verify depth for collector TLS/mTLS e2e beyond the grpc/single packs: +# layer, consumer metadata, service dependency, and broader metrics. + + cases: + # layer list + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql layer ls + expected: expected/layer.yml + + # consumer metadata (provider is covered by metadata-cases) + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql endpoint list --service-name="e2e-service-consumer|namespace" + expected: expected/service-endpoint-consumer.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql instance list --service-name="e2e-service-consumer|namespace" + expected: expected/service-instance-consumer.yml + + # service dependency topology — before metrics so topology still verifies + # when metrics GraphQL and swctl are skewed. + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name=e2e-service-provider + expected: expected/dependency-services-provider.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql dependency service --service-name="e2e-service-consumer|namespace" + expected: expected/dependency-services-consumer.yml + + # broader service metrics on both sides + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_cpm --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_resp_time --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_sla --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_cpm --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name service_resp_time --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + + # endpoint metrics on both sides + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_resp_time --endpoint-name=/artist-provider --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_sla --endpoint-name=/artist-provider --service-name=e2e-service-provider |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_cpm --endpoint-name=/artist-consumer --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_resp_time --endpoint-name=/artist-consumer --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml + - query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics linear --name endpoint_sla --endpoint-name=/artist-consumer --service-name="e2e-service-consumer|namespace" |yq e 'to_entries' - + expected: expected/metrics-has-value.yml diff --git a/tests/e2e/script/prepare/gen-e2e-tls-certs.sh b/tests/e2e/script/prepare/gen-e2e-tls-certs.sh new file mode 100644 index 000000000..03821d1f8 --- /dev/null +++ b/tests/e2e/script/prepare/gen-e2e-tls-certs.sh @@ -0,0 +1,120 @@ +#!/bin/sh + +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# Single source of truth for e2e TLS material (SAN + EKU, distinct CA/leaf DNs). +# +# Usage: +# gen-e2e-tls-certs.sh # local: write under tests/e2e/case/grpc/{ssl,mtls} +# gen-e2e-tls-certs.sh all # same as above +# gen-e2e-tls-certs.sh ssl # compose: write to /out/certs and /out/ca +# gen-e2e-tls-certs.sh mtls # compose: write to /out/server and /out/client +# +# Compose mounts this script into alpine/openssl (digest-pinned; openssl preinstalled — +# no apk). PEMs are not committed. +# +# Requires: openssl, POSIX sh (works on alpine/openssl and host Git Bash / Linux). + +set -eu + +MODE=${1:-all} + +# Resolve repo paths before cd into WORKDIR. With a relative $0, doing this after +# cd would fold ``tests/.../../../../..`` against the temp dir and silently +# install under WORKDIR (then the EXIT trap deletes it). +# Script lives at tests/e2e/script/prepare/ → repo root is ../../../.. +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +ROOT="$(CDPATH= cd -- "$SCRIPT_DIR/../../../.." && pwd)" +SSL_DIR="$ROOT/tests/e2e/case/grpc/ssl" +MTLS_DIR="$ROOT/tests/e2e/case/grpc/mtls" + +WORKDIR="${TMPDIR:-/tmp}/skywalking-e2e-tls-$$" +mkdir -p "$WORKDIR" +# shellcheck disable=SC2064 +trap 'rm -rf "$WORKDIR"' EXIT +cd "$WORKDIR" + +# CA (distinct CN from leaf services) +openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -out ca.crt -days 36500 \ + -subj "/CN=skywalking-e2e-ca" + +# Server leaf (CN/SAN must match docker service name ``oap``) +printf '%s\n' \ + 'subjectAltName=DNS:oap,DNS:localhost,IP:127.0.0.1' \ + 'extendedKeyUsage=serverAuth' \ + 'basicConstraints=CA:FALSE' \ + 'keyUsage=digitalSignature,keyEncipherment' > server.ext +openssl req -newkey rsa:2048 -nodes -keyout server.key -out server.csr -subj "/CN=oap" +openssl x509 -req -in server.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ + -out server.crt -days 36500 -extfile server.ext +openssl pkcs8 -topk8 -nocrypt -in server.key -out server.pem +cp server.pem server-key.pem + +# Client leaf for mTLS +printf '%s\n' \ + 'subjectAltName=DNS:python-agent' \ + 'extendedKeyUsage=clientAuth' \ + 'basicConstraints=CA:FALSE' \ + 'keyUsage=digitalSignature,keyEncipherment' > client.ext +openssl req -newkey rsa:2048 -nodes -keyout client.key -out client.csr -subj "/CN=python-agent" +openssl x509 -req -in client.csr -CA ca.crt -CAkey ca.key -CAcreateserial \ + -out client.crt -days 36500 -extfile client.ext +openssl pkcs8 -topk8 -nocrypt -in client.key -out client.pem + +install_ssl_layout() { + certs_dir=$1 + ca_dir=$2 + mkdir -p "$certs_dir" "$ca_dir" + cp ca.crt "$ca_dir/ca.crt" + cp ca.crt server-key.pem server.crt "$certs_dir/" + cp ca.crt "$certs_dir/ca.crt" +} + +install_mtls_layout() { + server_dir=$1 + client_dir=$2 + mkdir -p "$server_dir" "$client_dir" + cp ca.crt server.crt server.pem "$server_dir/" + cp ca.crt client.crt client.pem "$client_dir/" +} + +case "$MODE" in + ssl) + install_ssl_layout /out/certs /out/ca + echo "Generated SSL certs under /out/{certs,ca}" + ;; + mtls) + install_mtls_layout /out/server /out/client + echo "Generated mTLS certs under /out/{server,client}" + ;; + all) + install_ssl_layout "$SSL_DIR/certs" "$SSL_DIR/ca" + install_mtls_layout "$MTLS_DIR/server" "$MTLS_DIR/client" + echo "Generated SSL certs under $SSL_DIR" + echo "Generated mTLS certs under $MTLS_DIR" + openssl x509 -in "$SSL_DIR/certs/server.crt" -noout -subject -issuer + openssl x509 -in "$MTLS_DIR/client/client.crt" -noout -subject -issuer + openssl x509 -in "$SSL_DIR/ca/ca.crt" -noout -subject -issuer + ;; + *) + echo "usage: $0 [all|ssl|mtls]" >&2 + exit 2 + ;; +esac diff --git a/tests/plugin/http/sw_httpx/services/provider.py b/tests/plugin/http/sw_httpx/services/provider.py index 8b0839944..57e03bef4 100644 --- a/tests/plugin/http/sw_httpx/services/provider.py +++ b/tests/plugin/http/sw_httpx/services/provider.py @@ -1,32 +1,32 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the 'License'); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an 'AS IS' BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -import uvicorn -from fastapi import FastAPI - -app = FastAPI() - - -@app.post('/users') -async def application(): - try: - return {'song': 'Despacito', 'artist': 'Luis Fonsi'} - except Exception: # noqa - return {'message': 'Error'} - - -if __name__ == '__main__': - uvicorn.run(app, host='0.0.0.0', port=9091) +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the 'License'); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import uvicorn +from fastapi import FastAPI + +app = FastAPI() + + +@app.post('/users') +async def application(): + try: + return {'song': 'Despacito', 'artist': 'Luis Fonsi'} + except Exception: # noqa + return {'message': 'Error'} + + +if __name__ == '__main__': + uvicorn.run(app, host='0.0.0.0', port=9091) diff --git a/tests/unit/test_grpc_channel.py b/tests/unit/test_grpc_channel.py index bd73fc5c3..75cb336be 100644 --- a/tests/unit/test_grpc_channel.py +++ b/tests/unit/test_grpc_channel.py @@ -1,732 +1,733 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import socket -import json -import unittest -import asyncio -from queue import Queue -from time import monotonic -from unittest.mock import MagicMock, patch - -import grpc - -from skywalking.utils.grpc_channel import ( - GRPC_CHANNEL_OPTIONS, - _GRPC_RPC_TIMEOUT_MARGIN_SEC, - AddressKind, - BackendAddress, - build_grpc_target, - encode_sw_static_for_c_core, - expand_backend_addresses, - grpc_call_timeout, - handle_rpc_error, - is_auth_rpc_error, - is_channel_ready, - parse_backend_address, - parse_backend_addresses, - prepare_grpc_channel_endpoints, - resolve_grpc_target, - sw_static_endpoints, -) - - -class TestGrpcBackendAddress(unittest.TestCase): - - def test_parse_ipv4_and_hostname(self): - v4 = parse_backend_address('127.0.0.1:11800') - self.assertEqual(v4.host, '127.0.0.1') - self.assertEqual(v4.port, 11800) - self.assertEqual(v4.kind.value, 'ipv4') - - host = parse_backend_address('oap.example.com:11800') - self.assertEqual(host.host, 'oap.example.com') - self.assertEqual(host.kind.value, 'hostname') - - def test_parse_ipv6_requires_brackets(self): - v6 = parse_backend_address('[::1]:11800') - self.assertEqual(v6.host, '::1') - self.assertEqual(v6.port, 11800) - self.assertEqual(v6.kind.value, 'ipv6') - self.assertIsNone(parse_backend_address('::1:11800')) - - def test_parse_invalid_logged_and_skipped(self): - with self.assertLogs('skywalking', level='ERROR') as cm: - addrs = parse_backend_addresses('127.0.0.1:11800,bad-entry,10.0.0.2:11800') - self.assertEqual(len(addrs), 2) - self.assertTrue(any('bad-entry' in line for line in cm.output)) - - def test_single_target_plain(self): - self.assertEqual( - build_grpc_target(parse_backend_addresses('oap.svc:11800')), - 'oap.svc:11800', - ) - self.assertEqual( - build_grpc_target(parse_backend_addresses('127.0.0.1:11800')), - '127.0.0.1:11800', - ) - - def test_multi_ipv4_static_target(self): - target = build_grpc_target(parse_backend_addresses('10.0.0.1:11800,10.0.0.2:11800')) - self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.2:11800') - - def test_multi_ipv6_static_target(self): - target = build_grpc_target(parse_backend_addresses('[::1]:11800,[::2]:11800')) - self.assertEqual(target, 'ipv6:[::1]:11800,[::2]:11800') - - def test_multi_hostname_expands_to_ipv4_static(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - mapping = { - 'oap-a': [('10.0.0.1', port)], - 'oap-b': [('10.0.0.2', port)], - } - return [ - (socket.AF_INET, socket.SOCK_STREAM, 6, '', (ip, p)) - for ip, p in mapping[host] - ] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - target = build_grpc_target(parse_backend_addresses('oap-a:11800,oap-b:11800')) - self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.2:11800') - - def test_mixed_hostname_and_ip_expands(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.9', port))] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - target = build_grpc_target( - parse_backend_addresses('10.0.0.1:11800,oap-b:11800') - ) - self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.9:11800') - - def test_mixed_families_encoded_as_ipv4_mapped(self): - addrs = [ - BackendAddress('10.0.0.1', 11800, AddressKind.IPV4), - BackendAddress('::1', 11800, AddressKind.IPV6), - ] - target = build_grpc_target(addrs) - self.assertEqual(target, 'ipv6:[::ffff:10.0.0.1]:11800,[::1]:11800') - self.assertEqual( - sw_static_endpoints(addrs), - [ - {'addresses': [{'host': '10.0.0.1', 'port': 11800}]}, - {'addresses': [{'host': '::1', 'port': 11800}]}, - ], - ) - - def test_hostname_dual_stack_keeps_both_families(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - if host == 'oap-a': - return [ - (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.1', port)), - (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('2001:db8::1', port)), - ] - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.2', port))] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - target = build_grpc_target(parse_backend_addresses('oap-a:11800,oap-b:11800')) - self.assertEqual( - target, - 'ipv6:[::ffff:10.0.0.1]:11800,[2001:db8::1]:11800,[::ffff:10.0.0.2]:11800', - ) - - def test_encode_rejects_hostname(self): - with self.assertRaises(ValueError): - encode_sw_static_for_c_core([ - BackendAddress('oap.svc', 11800, AddressKind.HOSTNAME), - ]) - - def test_expand_skips_failed_hostname(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - if host == 'bad.host': - raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.3', port))] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - with self.assertLogs('skywalking', level='ERROR'): - expanded = expand_backend_addresses( - parse_backend_addresses('bad.host:11800,ok.host:11800') - ) - self.assertEqual([a.endpoint() for a in expanded], ['10.0.0.3:11800']) - - def test_authority_skips_failed_first_hostname(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - if host == 'bad.host': - raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.2', port))] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - with self.assertLogs('skywalking', level='ERROR'): - target, authority = prepare_grpc_channel_endpoints( - parse_backend_addresses('bad.host:11800,good.host:11800') - ) - self.assertEqual(target, 'ipv4:10.0.0.2:11800') - self.assertEqual(authority, 'good.host:11800') - - def test_rejects_ipv6_zone_and_control_chars(self): - self.assertIsNone(parse_backend_address('[fe80::1%eth0]:11800')) - self.assertIsNone(parse_backend_address('bad\nhost:11800')) - self.assertIsNone(parse_backend_address('has space:11800')) - - def test_dns_timeout_returns_quickly_without_joining_worker(self): - import threading - import time - from skywalking.utils import grpc_channel as mod - - def hang_getaddrinfo(*_args, **_kwargs): - time.sleep(30) - return [] - - previous = mod._DNS_LOOKUP_TIMEOUT_SEC - try: - mod._DNS_LOOKUP_TIMEOUT_SEC = 0.3 - before = {t.ident for t in threading.enumerate()} - t0 = time.monotonic() - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=hang_getaddrinfo): - with self.assertLogs('skywalking', level='ERROR'): - result = mod._lookup_hostname('slow.host', 11800) - elapsed = time.monotonic() - t0 - leftover = [ - t for t in threading.enumerate() - if t.ident not in before and t.is_alive() - ] - finally: - mod._DNS_LOOKUP_TIMEOUT_SEC = previous - - self.assertEqual(result, []) - self.assertLess(elapsed, 2.0) - # Hung lookup may still be running, but must be daemon so exit is not blocked. - for t in leftover: - self.assertTrue(t.daemon, msg=f'non-daemon leftover thread: {t.name}') - - def test_all_hostname_resolve_fail_raises(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - with self.assertLogs('skywalking', level='ERROR'): - with self.assertRaises(ValueError): - build_grpc_target(parse_backend_addresses('a.host:11800,b.host:11800')) - - def test_empty_raises(self): - with self.assertRaises(ValueError): - build_grpc_target([]) - - def test_channel_options_disable_proxy_no_keepalive(self): - keys = {k for k, _ in GRPC_CHANNEL_OPTIONS} - self.assertIn('grpc.enable_http_proxy', keys) - self.assertNotIn('grpc.lb_policy_name', keys) - self.assertEqual(dict(GRPC_CHANNEL_OPTIONS)['grpc.enable_http_proxy'], 0) - self.assertEqual(dict(GRPC_CHANNEL_OPTIONS)['grpc.max_reconnect_backoff_ms'], 30000) - self.assertFalse(any('keepalive' in k for k in keys)) - - def test_channel_options_properties_retry_service_config(self): - opts = dict(GRPC_CHANNEL_OPTIONS) - self.assertEqual(opts['grpc.enable_retries'], 1) - cfg = json.loads(opts['grpc.service_config']) - methods = cfg['methodConfig'] - self.assertEqual(len(methods), 1) - names = methods[0]['name'] - self.assertEqual(names, [{ - 'service': 'skywalking.v3.ManagementService', - 'method': 'reportInstanceProperties', - }]) - policy = methods[0]['retryPolicy'] - self.assertEqual(policy['maxAttempts'], 3) - self.assertEqual(policy['retryableStatusCodes'], ['UNAVAILABLE']) - # Streaming collect must not appear — retries would duplicate segments. - blob = opts['grpc.service_config'] - self.assertNotIn('collect', blob) - self.assertNotIn('keepAlive', blob) - lb = cfg['loadBalancingConfig'] - self.assertEqual(lb, [{'pick_first': {'shuffleAddressList': True}}]) - - def test_resolve_uses_config(self): - from skywalking import config - - previous = config.agent_collector_backend_services - try: - config.agent_collector_backend_services = '1.1.1.1:11800,1.1.1.2:11800' - self.assertEqual(resolve_grpc_target(), 'ipv4:1.1.1.1:11800,1.1.1.2:11800') - finally: - config.agent_collector_backend_services = previous - - def test_create_sync_channel_tls_passes_authority(self): - from skywalking import config - from skywalking.utils.grpc_channel import create_sync_channel - - previous = config.agent_collector_backend_services - previous_tls = config.agent_force_tls - try: - config.agent_collector_backend_services = 'oap.example:11800,10.0.0.2:11800' - config.agent_force_tls = True - - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.1', port))] - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo), \ - patch('skywalking.utils.grpc_channel.grpc.secure_channel') as secure, \ - patch('skywalking.utils.grpc_channel.grpc.ssl_channel_credentials', return_value='creds'): - create_sync_channel() - args, kwargs = secure.call_args - self.assertEqual(args[0], 'ipv4:10.0.0.1:11800,10.0.0.2:11800') - opts = dict(kwargs['options']) - self.assertEqual(opts['grpc.default_authority'], 'oap.example:11800') - finally: - config.agent_collector_backend_services = previous - config.agent_force_tls = previous_tls - - -class TestAuthRpcHandling(unittest.TestCase): - - def _rpc_error(self, status): - err = MagicMock() - err.code = MagicMock(return_value=status) - return err - - def test_auth_errors_detected(self): - self.assertTrue(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.UNAUTHENTICATED))) - self.assertTrue(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.PERMISSION_DENIED))) - self.assertFalse(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.UNAVAILABLE))) - - def test_auth_does_not_invoke_connectivity_hook(self): - hook = MagicMock() - with patch('skywalking.utils.grpc_channel._last_auth_log_at', 0): - handle_rpc_error(self._rpc_error(grpc.StatusCode.UNAUTHENTICATED), hook) - hook.assert_not_called() - - def test_unavailable_invokes_connectivity_hook(self): - hook = MagicMock() - handle_rpc_error(self._rpc_error(grpc.StatusCode.UNAVAILABLE), hook) - hook.assert_called_once() - - -class TestReadyGate(unittest.TestCase): - - def test_ready_true_only_for_ready_state(self): - channel = MagicMock() - channel.get_state.return_value = grpc.ChannelConnectivity.READY - # Ensure unwrap prefers public get_state (aio path). - channel._channel = MagicMock() - self.assertTrue(is_channel_ready(channel)) - channel.get_state.assert_called_with(True) - - def test_non_ready_states_skip(self): - channel = MagicMock() - for state in ( - grpc.ChannelConnectivity.IDLE, - grpc.ChannelConnectivity.CONNECTING, - grpc.ChannelConnectivity.TRANSIENT_FAILURE, - grpc.ChannelConnectivity.SHUTDOWN, - ): - channel.get_state.return_value = state - self.assertFalse(is_channel_ready(channel), msg=str(state)) - - def test_sync_channel_without_get_state_uses_cython_check(self): - # grpcio sync Channel has subscribe but no get_state — must not fail-closed forever. - class SyncLikeChannel: - pass - - channel = SyncLikeChannel() - cython = MagicMock() - cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.READY.value[0] - channel._channel = cython - self.assertTrue(is_channel_ready(channel)) - cython.check_connectivity_state.assert_called_with(True) - - cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.IDLE.value[0] - self.assertFalse(is_channel_ready(channel)) - - def test_intercept_channel_unwraps_to_cython_check(self): - class InterceptLike: - pass - - class SyncLike: - pass - - intercept = InterceptLike() - sync = SyncLike() - cython = MagicMock() - cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.READY.value[0] - sync._channel = cython - intercept._channel = sync - self.assertTrue(is_channel_ready(intercept)) - cython.check_connectivity_state.assert_called_with(True) - - def test_unknown_channel_fail_open(self): - # Cannot read connectivity → do not permanently silence reporters. - self.assertTrue(is_channel_ready(object())) - - -class TestLogThrottle(unittest.TestCase): - - def test_reporter_exception_throttled(self): - from skywalking.utils import reporter_log as mod - - # patch replaces the module dict for this test only (auto-restored); - # do not .clear() the shared throttle state — that leaks across tests. - with patch.object(mod, '_last_reporter_log_at', {}): - with self.assertLogs('skywalking', level='ERROR') as cm: - try: - raise RuntimeError('boom') - except RuntimeError: - mod.log_reporter_exception_throttled('segment', 1) - mod.log_reporter_exception_throttled('segment', 2) - self.assertEqual(len(cm.records), 1) - - def test_connectivity_event_throttled(self): - from skywalking.utils import grpc_channel as mod - - with patch.object(mod, '_last_connectivity_log_at', {}): - with self.assertLogs('skywalking', level='WARNING') as cm: - mod.log_connectivity_event('transient_failure', 'down1') - mod.log_connectivity_event('transient_failure', 'down2') - self.assertEqual(len(cm.records), 1) - - def test_dropped_throttled_includes_delta_and_total(self): - from skywalking.utils import reporter_log as mod - - with patch.object(mod, '_last_drop_log_at', {}), \ - patch.object(mod, '_drop_totals', {}), \ - patch.object(mod, '_drop_logged_totals', {}): - with self.assertLogs('skywalking', level='WARNING') as cm: - mod.log_dropped_throttled('segment', 2) - mod.log_dropped_throttled('segment', 3) - self.assertEqual(len(cm.records), 1) - self.assertIn('+2 since last log', cm.records[0].getMessage()) - self.assertIn('2 total', cm.records[0].getMessage()) - - -class TestCreateChannelDoesNotRaise(unittest.TestCase): - - def _assert_factory_degrades(self, services: str): - from skywalking import config - from skywalking.utils.grpc_channel import create_sync_channel - - previous = config.agent_collector_backend_services - channel = MagicMock() - channel.get_state.return_value = grpc.ChannelConnectivity.IDLE - try: - config.agent_collector_backend_services = services - with patch('skywalking.utils.grpc_channel.grpc.insecure_channel', return_value=channel) as insecure: - with self.assertLogs('skywalking', level='ERROR'): - got = create_sync_channel() - self.assertIs(got, channel) - insecure.assert_called() - self.assertNotEqual(got.get_state(), grpc.ChannelConnectivity.READY) - finally: - config.agent_collector_backend_services = previous - - def test_empty_config_does_not_raise(self): - self._assert_factory_degrades('') - - def test_garbage_config_does_not_raise(self): - self._assert_factory_degrades('not-an-address,also bad') - - def test_unresolvable_hostnames_do_not_raise(self): - def fake_getaddrinfo(host, port, type=0, *args, **kwargs): - raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') - - with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): - self._assert_factory_degrades('no.such.host.invalid:11800,also.invalid:11800') - - -class TestProfilingSnapshotNonBlocking(unittest.TestCase): - - def test_full_snapshot_queue_does_not_block(self): - from queue import Queue - from threading import Event, Thread - - from skywalking.agent import SkyWalkingAgent - - agent = SkyWalkingAgent.__new__(SkyWalkingAgent) - agent._SkyWalkingAgent__reporting = True - q = Queue(maxsize=1) - q.put('full') - agent._SkyWalkingAgent__snapshot_queue = q - - done = Event() - - def _put(): - agent.add_profiling_snapshot('next') - done.set() - - Thread(target=_put, daemon=True).start() - self.assertTrue(done.wait(1.0), 'add_profiling_snapshot blocked on a full queue') - self.assertEqual(q.qsize(), 1) - - -class TestGrpcCallTimeoutAndKeepAlive(unittest.TestCase): - - def test_rpc_timeout_exceeds_queue_window(self): - from skywalking import config - - prev = config.agent_queue_timeout - try: - config.agent_queue_timeout = 1 - self.assertEqual(grpc_call_timeout(), 10.0) - config.agent_queue_timeout = 20 - self.assertEqual(grpc_call_timeout(), 25.0) - finally: - config.agent_queue_timeout = prev - - def test_sync_collect_passes_timeout(self): - from skywalking.client.grpc import GrpcTraceSegmentReportService - - stub = MagicMock() - svc = GrpcTraceSegmentReportService.__new__(GrpcTraceSegmentReportService) - svc.report_stub = stub - svc.report(iter(())) - self.assertEqual(stub.collect.call_args.kwargs.get('timeout'), grpc_call_timeout()) - - def test_keep_alive_after_properties_refresh_failure(self): - from skywalking.client.grpc import GrpcServiceManagementClient - - class FakeRpcError(grpc.RpcError): - def code(self): - return grpc.StatusCode.UNAVAILABLE - - def details(self): - return 'props failed' - - client = GrpcServiceManagementClient.__new__(GrpcServiceManagementClient) - client.service_stub = MagicMock() - client.refresh_instance_props = MagicMock(side_effect=FakeRpcError()) - client.send_heart_beat() - client.service_stub.keepAlive.assert_called_once() - self.assertEqual( - client.service_stub.keepAlive.call_args.kwargs.get('timeout'), - grpc_call_timeout(), - ) - - -class TestAioStreamingOmitsDeadline(unittest.IsolatedAsyncioTestCase): - - async def test_aio_collect_omits_timeout(self): - from unittest.mock import AsyncMock - - from skywalking.client.grpc_aio import ( - GrpcLogReportServiceAsync, - GrpcMeterReportServiceAsync, - GrpcProfileTaskChannelServiceAsync, - GrpcTraceSegmentReportServiceAsync, - ) - - traces = MagicMock() - traces.collect = AsyncMock() - svc = GrpcTraceSegmentReportServiceAsync.__new__(GrpcTraceSegmentReportServiceAsync) - svc.report_stub = traces - await svc.report(object()) - self.assertNotIn('timeout', traces.collect.call_args.kwargs) - - meters = MagicMock() - meters.collect = AsyncMock() - meters.collectBatch = AsyncMock() - meter_svc = GrpcMeterReportServiceAsync.__new__(GrpcMeterReportServiceAsync) - meter_svc.report_stub = meters - await meter_svc.report(object()) - await meter_svc.report_batch(object()) - self.assertNotIn('timeout', meters.collect.call_args.kwargs) - self.assertNotIn('timeout', meters.collectBatch.call_args.kwargs) - - logs = MagicMock() - logs.collect = AsyncMock() - log_svc = GrpcLogReportServiceAsync.__new__(GrpcLogReportServiceAsync) - log_svc.report_stub = logs - await log_svc.report(object()) - self.assertNotIn('timeout', logs.collect.call_args.kwargs) - - profile = MagicMock() - profile.collectSnapshot = AsyncMock() - profile_svc = GrpcProfileTaskChannelServiceAsync.__new__(GrpcProfileTaskChannelServiceAsync) - profile_svc.profile_stub = profile - await profile_svc.report(object()) - self.assertNotIn('timeout', profile.collectSnapshot.call_args.kwargs) - - async def test_aio_unary_keeps_timeout(self): - from unittest.mock import AsyncMock - - from skywalking.client.grpc_aio import GrpcServiceManagementClientAsync - - client = GrpcServiceManagementClientAsync.__new__(GrpcServiceManagementClientAsync) - client.service_stub = MagicMock() - client.service_stub.keepAlive = AsyncMock() - client.refresh_instance_props = AsyncMock() - await client.send_heart_beat() - self.assertEqual( - client.service_stub.keepAlive.call_args.kwargs.get('timeout'), - grpc_call_timeout(), - ) - - -class TestClosePreviousProtocol(unittest.IsolatedAsyncioTestCase): - - def test_sync_close_never_blocks_on_aclose(self): - from skywalking.agent import _close_previous_protocol - - proto = MagicMock() - proto.close = MagicMock() - proto.aclose = MagicMock() - _close_previous_protocol(proto) - proto.close.assert_called_once() - proto.aclose.assert_not_called() - _close_previous_protocol(None) - - async def test_aclose_awaited_on_running_loop(self): - from skywalking.agent import _aclose_previous_protocol - - called = [] - loop = asyncio.get_running_loop() - - class _Proto: - async def aclose(self): - called.append(loop) - - def close(self): - called.append('close') - - await _aclose_previous_protocol(_Proto()) - self.assertEqual(called, [loop]) - await _aclose_previous_protocol(None) - - async def test_aclose_falls_back_to_close(self): - from skywalking.agent import _aclose_previous_protocol - - proto = MagicMock() - proto.aclose = None - proto.close = MagicMock() - await _aclose_previous_protocol(proto) - proto.close.assert_called_once() - - - -class TestRpcTimeoutVsQueueWindow(unittest.TestCase): - - def test_timeout_has_margin_over_worst_case_batch(self): - """RPC timeout must exceed absolute batch window + encode/RTT margin.""" - from skywalking import config - - prev = config.agent_queue_timeout - try: - config.agent_queue_timeout = 20 - timeout = grpc_call_timeout() - self.assertEqual(timeout, 20 + _GRPC_RPC_TIMEOUT_MARGIN_SEC) - self.assertGreater(timeout, float(config.agent_queue_timeout) + 1.0) - finally: - config.agent_queue_timeout = prev - - def test_sync_report_uses_timeout_with_margin(self): - from skywalking import config - from skywalking.client.grpc import GrpcTraceSegmentReportService - - prev = config.agent_queue_timeout - try: - config.agent_queue_timeout = 20 - stub = MagicMock() - svc = GrpcTraceSegmentReportService.__new__(GrpcTraceSegmentReportService) - svc.report_stub = stub - svc.report(iter(())) - self.assertEqual( - stub.collect.call_args.kwargs.get('timeout'), - 20 + _GRPC_RPC_TIMEOUT_MARGIN_SEC, - ) - finally: - config.agent_queue_timeout = prev - - -class TestQueueGetWithinBatch(unittest.TestCase): - - def test_queue_timeout_zero_drains_immediately_available_item(self): - from skywalking.agent.protocol.grpc import _queue_get_within_batch - - q = Queue() - q.put('segment') - batch_deadline = monotonic() - item = _queue_get_within_batch(q, True, batch_deadline, allow_immediate=True) - self.assertEqual(item, 'segment') - self.assertTrue(q.empty()) - - def test_queue_timeout_zero_skips_when_empty(self): - from skywalking.agent.protocol.grpc import _queue_get_within_batch - - q = Queue() - batch_deadline = monotonic() - self.assertIsNone( - _queue_get_within_batch(q, True, batch_deadline, allow_immediate=True), - ) - - -class TestCollectorChannelNotInstrumented(unittest.TestCase): - - def test_multi_address_collector_channel_skips_sw_interceptor(self): - """Regression: ipv4: multi targets must not get sw_grpc client interceptors.""" - import grpc - - from skywalking import config - from skywalking.plugins import sw_grpc - from skywalking.utils.grpc_channel import ( - create_sync_channel, - is_agent_collector_channel, - ) - - prev = config.agent_collector_backend_services - sw_grpc.install_sync() - try: - config.agent_collector_backend_services = '10.0.0.1:11800,10.0.0.2:11800' - with patch('grpc.intercept_channel') as intercept: - channel = create_sync_channel() - intercept.assert_not_called() - self.assertTrue(is_agent_collector_channel(channel)) - with patch('grpc.intercept_channel', - side_effect=lambda c, *a, **k: c) as intercept: - grpc.insecure_channel('business.example:50051') - intercept.assert_called() - finally: - config.agent_collector_backend_services = prev - - def test_aio_multi_address_uses_collector_scope(self): - from skywalking import config - from skywalking.plugins import sw_grpc - from skywalking.utils.grpc_channel import ( - create_aio_channel, - is_agent_collector_channel, - is_building_agent_collector_channel, - ) - - prev = config.agent_collector_backend_services - sw_grpc.install_async() - seen_building = [] - - class _Probe: - def __init__(self, *args, **kwargs): - seen_building.append(is_building_agent_collector_channel()) - # Minimal stand-in; create_aio_channel only needs a return object. - self._sw_agent_collector_channel = False - - try: - config.agent_collector_backend_services = '10.0.0.1:11800,10.0.0.2:11800' - with patch('skywalking.utils.grpc_channel.grpc.aio.insecure_channel', side_effect=_Probe): - channel = create_aio_channel() - self.assertEqual(seen_building, [True]) - self.assertTrue(is_agent_collector_channel(channel)) - self.assertFalse(is_building_agent_collector_channel()) - finally: - config.agent_collector_backend_services = prev - - -if __name__ == '__main__': - unittest.main() +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import socket +import json +import unittest +import asyncio +from queue import Queue +from time import monotonic +from unittest.mock import MagicMock, patch + +import grpc + +from skywalking.utils.grpc_channel import ( + GRPC_CHANNEL_OPTIONS, + _GRPC_RPC_TIMEOUT_MARGIN_SEC, + AddressKind, + BackendAddress, + build_grpc_target, + encode_sw_static_for_c_core, + expand_backend_addresses, + grpc_call_timeout, + handle_rpc_error, + is_auth_rpc_error, + is_channel_ready, + parse_backend_address, + parse_backend_addresses, + prepare_grpc_channel_endpoints, + resolve_grpc_target, + sw_static_endpoints, +) + + +class TestGrpcBackendAddress(unittest.TestCase): + + def test_parse_ipv4_and_hostname(self): + v4 = parse_backend_address('127.0.0.1:11800') + self.assertEqual(v4.host, '127.0.0.1') + self.assertEqual(v4.port, 11800) + self.assertEqual(v4.kind.value, 'ipv4') + + host = parse_backend_address('oap.example.com:11800') + self.assertEqual(host.host, 'oap.example.com') + self.assertEqual(host.kind.value, 'hostname') + + def test_parse_ipv6_requires_brackets(self): + v6 = parse_backend_address('[::1]:11800') + self.assertEqual(v6.host, '::1') + self.assertEqual(v6.port, 11800) + self.assertEqual(v6.kind.value, 'ipv6') + self.assertIsNone(parse_backend_address('::1:11800')) + + def test_parse_invalid_logged_and_skipped(self): + with self.assertLogs('skywalking', level='ERROR') as cm: + addrs = parse_backend_addresses('127.0.0.1:11800,bad-entry,10.0.0.2:11800') + self.assertEqual(len(addrs), 2) + self.assertTrue(any('bad-entry' in line for line in cm.output)) + + def test_single_target_plain(self): + self.assertEqual( + build_grpc_target(parse_backend_addresses('oap.svc:11800')), + 'oap.svc:11800', + ) + self.assertEqual( + build_grpc_target(parse_backend_addresses('127.0.0.1:11800')), + '127.0.0.1:11800', + ) + + def test_multi_ipv4_static_target(self): + target = build_grpc_target(parse_backend_addresses('10.0.0.1:11800,10.0.0.2:11800')) + self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.2:11800') + + def test_multi_ipv6_static_target(self): + target = build_grpc_target(parse_backend_addresses('[::1]:11800,[::2]:11800')) + self.assertEqual(target, 'ipv6:[::1]:11800,[::2]:11800') + + def test_multi_hostname_expands_to_ipv4_static(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + mapping = { + 'oap-a': [('10.0.0.1', port)], + 'oap-b': [('10.0.0.2', port)], + } + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', (ip, p)) + for ip, p in mapping[host] + ] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + target = build_grpc_target(parse_backend_addresses('oap-a:11800,oap-b:11800')) + self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.2:11800') + + def test_mixed_hostname_and_ip_expands(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.9', port))] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + target = build_grpc_target( + parse_backend_addresses('10.0.0.1:11800,oap-b:11800') + ) + self.assertEqual(target, 'ipv4:10.0.0.1:11800,10.0.0.9:11800') + + def test_mixed_families_encoded_as_ipv4_mapped(self): + addrs = [ + BackendAddress('10.0.0.1', 11800, AddressKind.IPV4), + BackendAddress('::1', 11800, AddressKind.IPV6), + ] + target = build_grpc_target(addrs) + self.assertEqual(target, 'ipv6:[::ffff:10.0.0.1]:11800,[::1]:11800') + self.assertEqual( + sw_static_endpoints(addrs), + [ + {'addresses': [{'host': '10.0.0.1', 'port': 11800}]}, + {'addresses': [{'host': '::1', 'port': 11800}]}, + ], + ) + + def test_hostname_dual_stack_keeps_both_families(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + if host == 'oap-a': + return [ + (socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.1', port)), + (socket.AF_INET6, socket.SOCK_STREAM, 6, '', ('2001:db8::1', port)), + ] + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.2', port))] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + target = build_grpc_target(parse_backend_addresses('oap-a:11800,oap-b:11800')) + self.assertEqual( + target, + 'ipv6:[::ffff:10.0.0.1]:11800,[2001:db8::1]:11800,[::ffff:10.0.0.2]:11800', + ) + + def test_encode_rejects_hostname(self): + with self.assertRaises(ValueError): + encode_sw_static_for_c_core([ + BackendAddress('oap.svc', 11800, AddressKind.HOSTNAME), + ]) + + def test_expand_skips_failed_hostname(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + if host == 'bad.host': + raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.3', port))] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + with self.assertLogs('skywalking', level='ERROR'): + expanded = expand_backend_addresses( + parse_backend_addresses('bad.host:11800,ok.host:11800') + ) + self.assertEqual([a.endpoint() for a in expanded], ['10.0.0.3:11800']) + + def test_authority_skips_failed_first_hostname(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + if host == 'bad.host': + raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.2', port))] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + with self.assertLogs('skywalking', level='ERROR'): + target, authority = prepare_grpc_channel_endpoints( + parse_backend_addresses('bad.host:11800,good.host:11800') + ) + self.assertEqual(target, 'ipv4:10.0.0.2:11800') + self.assertEqual(authority, 'good.host:11800') + + def test_rejects_ipv6_zone_and_control_chars(self): + self.assertIsNone(parse_backend_address('[fe80::1%eth0]:11800')) + self.assertIsNone(parse_backend_address('bad\nhost:11800')) + self.assertIsNone(parse_backend_address('has space:11800')) + + def test_dns_timeout_returns_quickly_without_joining_worker(self): + import threading + import time + from skywalking.utils import grpc_channel as mod + + def hang_getaddrinfo(*_args, **_kwargs): + time.sleep(30) + return [] + + previous = mod._DNS_LOOKUP_TIMEOUT_SEC + try: + mod._DNS_LOOKUP_TIMEOUT_SEC = 0.3 + before = {t.ident for t in threading.enumerate()} + t0 = time.monotonic() + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=hang_getaddrinfo): + with self.assertLogs('skywalking', level='ERROR'): + result = mod._lookup_hostname('slow.host', 11800) + elapsed = time.monotonic() - t0 + leftover = [ + t for t in threading.enumerate() + if t.ident not in before and t.is_alive() + ] + finally: + mod._DNS_LOOKUP_TIMEOUT_SEC = previous + + self.assertEqual(result, []) + self.assertLess(elapsed, 2.0) + # Hung lookup may still be running, but must be daemon so exit is not blocked. + for t in leftover: + self.assertTrue(t.daemon, msg=f'non-daemon leftover thread: {t.name}') + + def test_all_hostname_resolve_fail_raises(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + with self.assertLogs('skywalking', level='ERROR'): + with self.assertRaises(ValueError): + build_grpc_target(parse_backend_addresses('a.host:11800,b.host:11800')) + + def test_empty_raises(self): + with self.assertRaises(ValueError): + build_grpc_target([]) + + def test_channel_options_disable_proxy_no_keepalive(self): + keys = {k for k, _ in GRPC_CHANNEL_OPTIONS} + self.assertIn('grpc.enable_http_proxy', keys) + self.assertNotIn('grpc.lb_policy_name', keys) + self.assertEqual(dict(GRPC_CHANNEL_OPTIONS)['grpc.enable_http_proxy'], 0) + self.assertEqual(dict(GRPC_CHANNEL_OPTIONS)['grpc.max_reconnect_backoff_ms'], 30000) + self.assertFalse(any('keepalive' in k for k in keys)) + + def test_channel_options_properties_retry_service_config(self): + opts = dict(GRPC_CHANNEL_OPTIONS) + self.assertEqual(opts['grpc.enable_retries'], 1) + cfg = json.loads(opts['grpc.service_config']) + methods = cfg['methodConfig'] + self.assertEqual(len(methods), 1) + names = methods[0]['name'] + self.assertEqual(names, [{ + 'service': 'skywalking.v3.ManagementService', + 'method': 'reportInstanceProperties', + }]) + policy = methods[0]['retryPolicy'] + self.assertEqual(policy['maxAttempts'], 3) + self.assertEqual(policy['retryableStatusCodes'], ['UNAVAILABLE']) + # Streaming collect must not appear — retries would duplicate segments. + blob = opts['grpc.service_config'] + self.assertNotIn('collect', blob) + self.assertNotIn('keepAlive', blob) + lb = cfg['loadBalancingConfig'] + self.assertEqual(lb, [{'pick_first': {'shuffleAddressList': True}}]) + + def test_resolve_uses_config(self): + from skywalking import config + + previous = config.agent_collector_backend_services + try: + config.agent_collector_backend_services = '1.1.1.1:11800,1.1.1.2:11800' + self.assertEqual(resolve_grpc_target(), 'ipv4:1.1.1.1:11800,1.1.1.2:11800') + finally: + config.agent_collector_backend_services = previous + + def test_create_sync_channel_tls_passes_authority(self): + from skywalking import config + from skywalking.utils.grpc_channel import create_sync_channel + + previous = config.agent_collector_backend_services + previous_tls = config.agent_force_tls + try: + config.agent_collector_backend_services = 'oap.example:11800,10.0.0.2:11800' + config.agent_force_tls = True + + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', ('10.0.0.1', port))] + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo), \ + patch('skywalking.utils.grpc_channel.grpc.secure_channel') as secure, \ + patch('skywalking.utils.grpc_channel.grpc_ssl_credentials', return_value='creds'): + create_sync_channel() + args, kwargs = secure.call_args + self.assertEqual(args[0], 'ipv4:10.0.0.1:11800,10.0.0.2:11800') + opts = dict(kwargs['options']) + self.assertEqual(opts['grpc.default_authority'], 'oap.example:11800') + self.assertNotIn('grpc.ssl_target_name_override', opts) + finally: + config.agent_collector_backend_services = previous + config.agent_force_tls = previous_tls + + +class TestAuthRpcHandling(unittest.TestCase): + + def _rpc_error(self, status): + err = MagicMock() + err.code = MagicMock(return_value=status) + return err + + def test_auth_errors_detected(self): + self.assertTrue(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.UNAUTHENTICATED))) + self.assertTrue(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.PERMISSION_DENIED))) + self.assertFalse(is_auth_rpc_error(self._rpc_error(grpc.StatusCode.UNAVAILABLE))) + + def test_auth_does_not_invoke_connectivity_hook(self): + hook = MagicMock() + with patch('skywalking.utils.grpc_channel._last_auth_log_at', 0): + handle_rpc_error(self._rpc_error(grpc.StatusCode.UNAUTHENTICATED), hook) + hook.assert_not_called() + + def test_unavailable_invokes_connectivity_hook(self): + hook = MagicMock() + handle_rpc_error(self._rpc_error(grpc.StatusCode.UNAVAILABLE), hook) + hook.assert_called_once() + + +class TestReadyGate(unittest.TestCase): + + def test_ready_true_only_for_ready_state(self): + channel = MagicMock() + channel.get_state.return_value = grpc.ChannelConnectivity.READY + # Ensure unwrap prefers public get_state (aio path). + channel._channel = MagicMock() + self.assertTrue(is_channel_ready(channel)) + channel.get_state.assert_called_with(True) + + def test_non_ready_states_skip(self): + channel = MagicMock() + for state in ( + grpc.ChannelConnectivity.IDLE, + grpc.ChannelConnectivity.CONNECTING, + grpc.ChannelConnectivity.TRANSIENT_FAILURE, + grpc.ChannelConnectivity.SHUTDOWN, + ): + channel.get_state.return_value = state + self.assertFalse(is_channel_ready(channel), msg=str(state)) + + def test_sync_channel_without_get_state_uses_cython_check(self): + # grpcio sync Channel has subscribe but no get_state — must not fail-closed forever. + class SyncLikeChannel: + pass + + channel = SyncLikeChannel() + cython = MagicMock() + cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.READY.value[0] + channel._channel = cython + self.assertTrue(is_channel_ready(channel)) + cython.check_connectivity_state.assert_called_with(True) + + cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.IDLE.value[0] + self.assertFalse(is_channel_ready(channel)) + + def test_intercept_channel_unwraps_to_cython_check(self): + class InterceptLike: + pass + + class SyncLike: + pass + + intercept = InterceptLike() + sync = SyncLike() + cython = MagicMock() + cython.check_connectivity_state.return_value = grpc.ChannelConnectivity.READY.value[0] + sync._channel = cython + intercept._channel = sync + self.assertTrue(is_channel_ready(intercept)) + cython.check_connectivity_state.assert_called_with(True) + + def test_unknown_channel_fail_open(self): + # Cannot read connectivity → do not permanently silence reporters. + self.assertTrue(is_channel_ready(object())) + + +class TestLogThrottle(unittest.TestCase): + + def test_reporter_exception_throttled(self): + from skywalking.utils import reporter_log as mod + + # patch replaces the module dict for this test only (auto-restored); + # do not .clear() the shared throttle state — that leaks across tests. + with patch.object(mod, '_last_reporter_log_at', {}): + with self.assertLogs('skywalking', level='ERROR') as cm: + try: + raise RuntimeError('boom') + except RuntimeError: + mod.log_reporter_exception_throttled('segment', 1) + mod.log_reporter_exception_throttled('segment', 2) + self.assertEqual(len(cm.records), 1) + + def test_connectivity_event_throttled(self): + from skywalking.utils import grpc_channel as mod + + with patch.object(mod, '_last_connectivity_log_at', {}): + with self.assertLogs('skywalking', level='WARNING') as cm: + mod.log_connectivity_event('transient_failure', 'down1') + mod.log_connectivity_event('transient_failure', 'down2') + self.assertEqual(len(cm.records), 1) + + def test_dropped_throttled_includes_delta_and_total(self): + from skywalking.utils import reporter_log as mod + + with patch.object(mod, '_last_drop_log_at', {}), \ + patch.object(mod, '_drop_totals', {}), \ + patch.object(mod, '_drop_logged_totals', {}): + with self.assertLogs('skywalking', level='WARNING') as cm: + mod.log_dropped_throttled('segment', 2) + mod.log_dropped_throttled('segment', 3) + self.assertEqual(len(cm.records), 1) + self.assertIn('+2 since last log', cm.records[0].getMessage()) + self.assertIn('2 total', cm.records[0].getMessage()) + + +class TestCreateChannelDoesNotRaise(unittest.TestCase): + + def _assert_factory_degrades(self, services: str): + from skywalking import config + from skywalking.utils.grpc_channel import create_sync_channel + + previous = config.agent_collector_backend_services + channel = MagicMock() + channel.get_state.return_value = grpc.ChannelConnectivity.IDLE + try: + config.agent_collector_backend_services = services + with patch('skywalking.utils.grpc_channel.grpc.insecure_channel', return_value=channel) as insecure: + with self.assertLogs('skywalking', level='ERROR'): + got = create_sync_channel() + self.assertIs(got, channel) + insecure.assert_called() + self.assertNotEqual(got.get_state(), grpc.ChannelConnectivity.READY) + finally: + config.agent_collector_backend_services = previous + + def test_empty_config_does_not_raise(self): + self._assert_factory_degrades('') + + def test_garbage_config_does_not_raise(self): + self._assert_factory_degrades('not-an-address,also bad') + + def test_unresolvable_hostnames_do_not_raise(self): + def fake_getaddrinfo(host, port, type=0, *args, **kwargs): + raise socket.gaierror(socket.EAI_NONAME, 'Name or service not known') + + with patch('skywalking.utils.grpc_channel.socket.getaddrinfo', side_effect=fake_getaddrinfo): + self._assert_factory_degrades('no.such.host.invalid:11800,also.invalid:11800') + + +class TestProfilingSnapshotNonBlocking(unittest.TestCase): + + def test_full_snapshot_queue_does_not_block(self): + from queue import Queue + from threading import Event, Thread + + from skywalking.agent import SkyWalkingAgent + + agent = SkyWalkingAgent.__new__(SkyWalkingAgent) + agent._SkyWalkingAgent__reporting = True + q = Queue(maxsize=1) + q.put('full') + agent._SkyWalkingAgent__snapshot_queue = q + + done = Event() + + def _put(): + agent.add_profiling_snapshot('next') + done.set() + + Thread(target=_put, daemon=True).start() + self.assertTrue(done.wait(1.0), 'add_profiling_snapshot blocked on a full queue') + self.assertEqual(q.qsize(), 1) + + +class TestGrpcCallTimeoutAndKeepAlive(unittest.TestCase): + + def test_rpc_timeout_exceeds_queue_window(self): + from skywalking import config + + prev = config.agent_queue_timeout + try: + config.agent_queue_timeout = 1 + self.assertEqual(grpc_call_timeout(), 10.0) + config.agent_queue_timeout = 20 + self.assertEqual(grpc_call_timeout(), 25.0) + finally: + config.agent_queue_timeout = prev + + def test_sync_collect_passes_timeout(self): + from skywalking.client.grpc import GrpcTraceSegmentReportService + + stub = MagicMock() + svc = GrpcTraceSegmentReportService.__new__(GrpcTraceSegmentReportService) + svc.report_stub = stub + svc.report(iter(())) + self.assertEqual(stub.collect.call_args.kwargs.get('timeout'), grpc_call_timeout()) + + def test_keep_alive_after_properties_refresh_failure(self): + from skywalking.client.grpc import GrpcServiceManagementClient + + class FakeRpcError(grpc.RpcError): + def code(self): + return grpc.StatusCode.UNAVAILABLE + + def details(self): + return 'props failed' + + client = GrpcServiceManagementClient.__new__(GrpcServiceManagementClient) + client.service_stub = MagicMock() + client.refresh_instance_props = MagicMock(side_effect=FakeRpcError()) + client.send_heart_beat() + client.service_stub.keepAlive.assert_called_once() + self.assertEqual( + client.service_stub.keepAlive.call_args.kwargs.get('timeout'), + grpc_call_timeout(), + ) + + +class TestAioStreamingOmitsDeadline(unittest.IsolatedAsyncioTestCase): + + async def test_aio_collect_omits_timeout(self): + from unittest.mock import AsyncMock + + from skywalking.client.grpc_aio import ( + GrpcLogReportServiceAsync, + GrpcMeterReportServiceAsync, + GrpcProfileTaskChannelServiceAsync, + GrpcTraceSegmentReportServiceAsync, + ) + + traces = MagicMock() + traces.collect = AsyncMock() + svc = GrpcTraceSegmentReportServiceAsync.__new__(GrpcTraceSegmentReportServiceAsync) + svc.report_stub = traces + await svc.report(object()) + self.assertNotIn('timeout', traces.collect.call_args.kwargs) + + meters = MagicMock() + meters.collect = AsyncMock() + meters.collectBatch = AsyncMock() + meter_svc = GrpcMeterReportServiceAsync.__new__(GrpcMeterReportServiceAsync) + meter_svc.report_stub = meters + await meter_svc.report(object()) + await meter_svc.report_batch(object()) + self.assertNotIn('timeout', meters.collect.call_args.kwargs) + self.assertNotIn('timeout', meters.collectBatch.call_args.kwargs) + + logs = MagicMock() + logs.collect = AsyncMock() + log_svc = GrpcLogReportServiceAsync.__new__(GrpcLogReportServiceAsync) + log_svc.report_stub = logs + await log_svc.report(object()) + self.assertNotIn('timeout', logs.collect.call_args.kwargs) + + profile = MagicMock() + profile.collectSnapshot = AsyncMock() + profile_svc = GrpcProfileTaskChannelServiceAsync.__new__(GrpcProfileTaskChannelServiceAsync) + profile_svc.profile_stub = profile + await profile_svc.report(object()) + self.assertNotIn('timeout', profile.collectSnapshot.call_args.kwargs) + + async def test_aio_unary_keeps_timeout(self): + from unittest.mock import AsyncMock + + from skywalking.client.grpc_aio import GrpcServiceManagementClientAsync + + client = GrpcServiceManagementClientAsync.__new__(GrpcServiceManagementClientAsync) + client.service_stub = MagicMock() + client.service_stub.keepAlive = AsyncMock() + client.refresh_instance_props = AsyncMock() + await client.send_heart_beat() + self.assertEqual( + client.service_stub.keepAlive.call_args.kwargs.get('timeout'), + grpc_call_timeout(), + ) + + +class TestClosePreviousProtocol(unittest.IsolatedAsyncioTestCase): + + def test_sync_close_never_blocks_on_aclose(self): + from skywalking.agent import _close_previous_protocol + + proto = MagicMock() + proto.close = MagicMock() + proto.aclose = MagicMock() + _close_previous_protocol(proto) + proto.close.assert_called_once() + proto.aclose.assert_not_called() + _close_previous_protocol(None) + + async def test_aclose_awaited_on_running_loop(self): + from skywalking.agent import _aclose_previous_protocol + + called = [] + loop = asyncio.get_running_loop() + + class _Proto: + async def aclose(self): + called.append(loop) + + def close(self): + called.append('close') + + await _aclose_previous_protocol(_Proto()) + self.assertEqual(called, [loop]) + await _aclose_previous_protocol(None) + + async def test_aclose_falls_back_to_close(self): + from skywalking.agent import _aclose_previous_protocol + + proto = MagicMock() + proto.aclose = None + proto.close = MagicMock() + await _aclose_previous_protocol(proto) + proto.close.assert_called_once() + + + +class TestRpcTimeoutVsQueueWindow(unittest.TestCase): + + def test_timeout_has_margin_over_worst_case_batch(self): + """RPC timeout must exceed absolute batch window + encode/RTT margin.""" + from skywalking import config + + prev = config.agent_queue_timeout + try: + config.agent_queue_timeout = 20 + timeout = grpc_call_timeout() + self.assertEqual(timeout, 20 + _GRPC_RPC_TIMEOUT_MARGIN_SEC) + self.assertGreater(timeout, float(config.agent_queue_timeout) + 1.0) + finally: + config.agent_queue_timeout = prev + + def test_sync_report_uses_timeout_with_margin(self): + from skywalking import config + from skywalking.client.grpc import GrpcTraceSegmentReportService + + prev = config.agent_queue_timeout + try: + config.agent_queue_timeout = 20 + stub = MagicMock() + svc = GrpcTraceSegmentReportService.__new__(GrpcTraceSegmentReportService) + svc.report_stub = stub + svc.report(iter(())) + self.assertEqual( + stub.collect.call_args.kwargs.get('timeout'), + 20 + _GRPC_RPC_TIMEOUT_MARGIN_SEC, + ) + finally: + config.agent_queue_timeout = prev + + +class TestQueueGetWithinBatch(unittest.TestCase): + + def test_queue_timeout_zero_drains_immediately_available_item(self): + from skywalking.agent.protocol.grpc import _queue_get_within_batch + + q = Queue() + q.put('segment') + batch_deadline = monotonic() + item = _queue_get_within_batch(q, True, batch_deadline, allow_immediate=True) + self.assertEqual(item, 'segment') + self.assertTrue(q.empty()) + + def test_queue_timeout_zero_skips_when_empty(self): + from skywalking.agent.protocol.grpc import _queue_get_within_batch + + q = Queue() + batch_deadline = monotonic() + self.assertIsNone( + _queue_get_within_batch(q, True, batch_deadline, allow_immediate=True), + ) + + +class TestCollectorChannelNotInstrumented(unittest.TestCase): + + def test_multi_address_collector_channel_skips_sw_interceptor(self): + """Regression: ipv4: multi targets must not get sw_grpc client interceptors.""" + import grpc + + from skywalking import config + from skywalking.plugins import sw_grpc + from skywalking.utils.grpc_channel import ( + create_sync_channel, + is_agent_collector_channel, + ) + + prev = config.agent_collector_backend_services + sw_grpc.install_sync() + try: + config.agent_collector_backend_services = '10.0.0.1:11800,10.0.0.2:11800' + with patch('grpc.intercept_channel') as intercept: + channel = create_sync_channel() + intercept.assert_not_called() + self.assertTrue(is_agent_collector_channel(channel)) + with patch('grpc.intercept_channel', + side_effect=lambda c, *a, **k: c) as intercept: + grpc.insecure_channel('business.example:50051') + intercept.assert_called() + finally: + config.agent_collector_backend_services = prev + + def test_aio_multi_address_uses_collector_scope(self): + from skywalking import config + from skywalking.plugins import sw_grpc + from skywalking.utils.grpc_channel import ( + create_aio_channel, + is_agent_collector_channel, + is_building_agent_collector_channel, + ) + + prev = config.agent_collector_backend_services + sw_grpc.install_async() + seen_building = [] + + class _Probe: + def __init__(self, *args, **kwargs): + seen_building.append(is_building_agent_collector_channel()) + # Minimal stand-in; create_aio_channel only needs a return object. + self._sw_agent_collector_channel = False + + try: + config.agent_collector_backend_services = '10.0.0.1:11800,10.0.0.2:11800' + with patch('skywalking.utils.grpc_channel.grpc.aio.insecure_channel', side_effect=_Probe): + channel = create_aio_channel() + self.assertEqual(seen_building, [True]) + self.assertTrue(is_agent_collector_channel(channel)) + self.assertFalse(is_building_agent_collector_channel()) + finally: + config.agent_collector_backend_services = prev + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/test_grpc_ready_gate.py b/tests/unit/test_grpc_ready_gate.py index 795574cc2..22627e48d 100644 --- a/tests/unit/test_grpc_ready_gate.py +++ b/tests/unit/test_grpc_ready_gate.py @@ -1,167 +1,167 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import unittest -from unittest.mock import MagicMock, patch - -import grpc - -from skywalking.agent.protocol.grpc import GrpcProtocol - - -class TestSyncGrpcReadyGate(unittest.TestCase): - - def _protocol(self) -> GrpcProtocol: - channel = MagicMock() - with patch('skywalking.agent.protocol.grpc.create_sync_channel', return_value=channel), \ - patch('skywalking.agent.protocol.grpc.GrpcServiceManagementClient'), \ - patch('skywalking.agent.protocol.grpc.GrpcTraceSegmentReportService'), \ - patch('skywalking.agent.protocol.grpc.GrpcProfileTaskChannelService'), \ - patch('skywalking.agent.protocol.grpc.GrpcLogDataReportService'), \ - patch('skywalking.agent.protocol.grpc.GrpcMeterReportService'): - return GrpcProtocol() - - def test_is_ready_follows_subscribe_state(self): - protocol = self._protocol() - self.assertFalse(protocol.is_ready()) - - protocol.state = grpc.ChannelConnectivity.CONNECTING - self.assertFalse(protocol.is_ready()) - - protocol.state = grpc.ChannelConnectivity.READY - self.assertTrue(protocol.is_ready()) - - protocol.properties_sent = True - protocol.service_management.sent_properties_counter = 7 - protocol._cb(grpc.ChannelConnectivity.TRANSIENT_FAILURE) - self.assertFalse(protocol.properties_sent) - self.assertEqual(protocol.service_management.sent_properties_counter, 0) - - protocol.state = grpc.ChannelConnectivity.IDLE - with patch('skywalking.agent.protocol.grpc.is_channel_ready') as nudge: - nudge.return_value = False - self.assertFalse(protocol.is_ready()) - nudge.assert_called_once_with(protocol.channel) - - def test_heartbeat_keep_alive_after_instance_props_failure(self): - class FakeRpcError(grpc.RpcError): - def code(self): - return grpc.StatusCode.UNAVAILABLE - - def details(self): - return 'props failed' - - protocol = self._protocol() - protocol.state = grpc.ChannelConnectivity.READY - protocol.properties_sent = False - protocol.service_management.send_instance_props = MagicMock(side_effect=FakeRpcError()) - protocol.service_management.send_heart_beat = MagicMock() - protocol.heartbeat() - protocol.service_management.send_heart_beat.assert_called_once() - self.assertFalse(protocol.properties_sent) - - def test_failed_segment_batch_counts_drops(self): - from queue import Queue - - class FakeRpcError(grpc.RpcError): - def code(self): - return grpc.StatusCode.UNAVAILABLE - - def details(self): - return 'collect failed' - - protocol = self._protocol() - protocol.state = grpc.ChannelConnectivity.READY - protocol.on_error = MagicMock() - - segment = MagicMock() - segment.related_traces = ['trace'] - segment.segment_id = 'seg' - segment.is_size_limited = False - segment.spans = [] - - queue = Queue() - queue.put(segment) - - def _report(generator): - list(generator) - raise FakeRpcError() - - protocol.traces_reporter.report = _report - with patch('skywalking.agent.protocol.grpc.log_dropped_throttled') as dropped, \ - patch('skywalking.agent.protocol.grpc.SegmentObject', return_value=object()), \ - patch('skywalking.agent.protocol.grpc.handle_rpc_error'): - with self.assertRaises(FakeRpcError): - protocol.report_segment(queue, block=False) - dropped.assert_called_with('segment', 1) - - def test_properties_refresh_every_factor_heartbeats(self): - """Java/Node cadence: reportInstanceProperties every N keepAlive ticks.""" - from skywalking import config - from skywalking.client import ServiceManagementClient - - class _Client(ServiceManagementClient): - def send_instance_props(self) -> None: - pass - - client = _Client() - client.send_instance_props = MagicMock() - - prev = config.agent_collector_properties_report_period_factor - try: - config.agent_collector_properties_report_period_factor = 3 - client.refresh_instance_props() # 1 - client.refresh_instance_props() # 2 - self.assertEqual(client.send_instance_props.call_count, 0) - client.refresh_instance_props() # 3 - self.assertEqual(client.send_instance_props.call_count, 1) - client.refresh_instance_props() # 4 - client.refresh_instance_props() # 5 - client.refresh_instance_props() # 6 - self.assertEqual(client.send_instance_props.call_count, 2) - finally: - config.agent_collector_properties_report_period_factor = prev - - -class TestAsyncGrpcReadyGate(unittest.TestCase): - - def test_aio_is_ready_follows_watched_state(self): - from skywalking.agent.protocol.grpc_aio import GrpcProtocolAsync - - channel = MagicMock() - channel.get_state.return_value = grpc.ChannelConnectivity.CONNECTING - with patch('skywalking.agent.protocol.grpc_aio.create_aio_channel', return_value=channel), \ - patch('skywalking.agent.protocol.grpc_aio.GrpcServiceManagementClientAsync'), \ - patch('skywalking.agent.protocol.grpc_aio.GrpcTraceSegmentReportServiceAsync'), \ - patch('skywalking.agent.protocol.grpc_aio.GrpcProfileTaskChannelServiceAsync'), \ - patch('skywalking.agent.protocol.grpc_aio.GrpcLogReportServiceAsync'), \ - patch('skywalking.agent.protocol.grpc_aio.GrpcMeterReportServiceAsync'): - protocol = GrpcProtocolAsync() - - protocol.state = grpc.ChannelConnectivity.READY - self.assertTrue(protocol.is_ready()) - protocol.properties_sent.set() - protocol.service_management.sent_properties_counter = 4 - protocol._on_connectivity(grpc.ChannelConnectivity.TRANSIENT_FAILURE) - self.assertFalse(protocol.properties_sent.is_set()) - self.assertEqual(protocol.service_management.sent_properties_counter, 0) - protocol.state = grpc.ChannelConnectivity.TRANSIENT_FAILURE - self.assertFalse(protocol.is_ready()) - - -if __name__ == '__main__': - unittest.main() +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import unittest +from unittest.mock import MagicMock, patch + +import grpc + +from skywalking.agent.protocol.grpc import GrpcProtocol + + +class TestSyncGrpcReadyGate(unittest.TestCase): + + def _protocol(self) -> GrpcProtocol: + channel = MagicMock() + with patch('skywalking.agent.protocol.grpc.create_sync_channel', return_value=channel), \ + patch('skywalking.agent.protocol.grpc.GrpcServiceManagementClient'), \ + patch('skywalking.agent.protocol.grpc.GrpcTraceSegmentReportService'), \ + patch('skywalking.agent.protocol.grpc.GrpcProfileTaskChannelService'), \ + patch('skywalking.agent.protocol.grpc.GrpcLogDataReportService'), \ + patch('skywalking.agent.protocol.grpc.GrpcMeterReportService'): + return GrpcProtocol() + + def test_is_ready_follows_subscribe_state(self): + protocol = self._protocol() + self.assertFalse(protocol.is_ready()) + + protocol.state = grpc.ChannelConnectivity.CONNECTING + self.assertFalse(protocol.is_ready()) + + protocol.state = grpc.ChannelConnectivity.READY + self.assertTrue(protocol.is_ready()) + + protocol.properties_sent = True + protocol.service_management.sent_properties_counter = 7 + protocol._cb(grpc.ChannelConnectivity.TRANSIENT_FAILURE) + self.assertFalse(protocol.properties_sent) + self.assertEqual(protocol.service_management.sent_properties_counter, 0) + + protocol.state = grpc.ChannelConnectivity.IDLE + with patch('skywalking.agent.protocol.grpc.is_channel_ready') as nudge: + nudge.return_value = False + self.assertFalse(protocol.is_ready()) + nudge.assert_called_once_with(protocol.channel) + + def test_heartbeat_keep_alive_after_instance_props_failure(self): + class FakeRpcError(grpc.RpcError): + def code(self): + return grpc.StatusCode.UNAVAILABLE + + def details(self): + return 'props failed' + + protocol = self._protocol() + protocol.state = grpc.ChannelConnectivity.READY + protocol.properties_sent = False + protocol.service_management.send_instance_props = MagicMock(side_effect=FakeRpcError()) + protocol.service_management.send_heart_beat = MagicMock() + protocol.heartbeat() + protocol.service_management.send_heart_beat.assert_called_once() + self.assertFalse(protocol.properties_sent) + + def test_failed_segment_batch_counts_drops(self): + from queue import Queue + + class FakeRpcError(grpc.RpcError): + def code(self): + return grpc.StatusCode.UNAVAILABLE + + def details(self): + return 'collect failed' + + protocol = self._protocol() + protocol.state = grpc.ChannelConnectivity.READY + protocol.on_error = MagicMock() + + segment = MagicMock() + segment.related_traces = ['trace'] + segment.segment_id = 'seg' + segment.is_size_limited = False + segment.spans = [] + + queue = Queue() + queue.put(segment) + + def _report(generator): + list(generator) + raise FakeRpcError() + + protocol.traces_reporter.report = _report + with patch('skywalking.agent.protocol.grpc.log_dropped_throttled') as dropped, \ + patch('skywalking.agent.protocol.grpc.SegmentObject', return_value=object()), \ + patch('skywalking.agent.protocol.grpc.handle_rpc_error'): + with self.assertRaises(FakeRpcError): + protocol.report_segment(queue, block=False) + dropped.assert_called_with('segment', 1) + + def test_properties_refresh_every_factor_heartbeats(self): + """Java/Node cadence: reportInstanceProperties every N keepAlive ticks.""" + from skywalking import config + from skywalking.client import ServiceManagementClient + + class _Client(ServiceManagementClient): + def send_instance_props(self) -> None: + pass + + client = _Client() + client.send_instance_props = MagicMock() + + prev = config.agent_collector_properties_report_period_factor + try: + config.agent_collector_properties_report_period_factor = 3 + client.refresh_instance_props() # 1 + client.refresh_instance_props() # 2 + self.assertEqual(client.send_instance_props.call_count, 0) + client.refresh_instance_props() # 3 + self.assertEqual(client.send_instance_props.call_count, 1) + client.refresh_instance_props() # 4 + client.refresh_instance_props() # 5 + client.refresh_instance_props() # 6 + self.assertEqual(client.send_instance_props.call_count, 2) + finally: + config.agent_collector_properties_report_period_factor = prev + + +class TestAsyncGrpcReadyGate(unittest.TestCase): + + def test_aio_is_ready_follows_watched_state(self): + from skywalking.agent.protocol.grpc_aio import GrpcProtocolAsync + + channel = MagicMock() + channel.get_state.return_value = grpc.ChannelConnectivity.CONNECTING + with patch('skywalking.agent.protocol.grpc_aio.create_aio_channel', return_value=channel), \ + patch('skywalking.agent.protocol.grpc_aio.GrpcServiceManagementClientAsync'), \ + patch('skywalking.agent.protocol.grpc_aio.GrpcTraceSegmentReportServiceAsync'), \ + patch('skywalking.agent.protocol.grpc_aio.GrpcProfileTaskChannelServiceAsync'), \ + patch('skywalking.agent.protocol.grpc_aio.GrpcLogReportServiceAsync'), \ + patch('skywalking.agent.protocol.grpc_aio.GrpcMeterReportServiceAsync'): + protocol = GrpcProtocolAsync() + + protocol.state = grpc.ChannelConnectivity.READY + self.assertTrue(protocol.is_ready()) + protocol.properties_sent.set() + protocol.service_management.sent_properties_counter = 4 + protocol._on_connectivity(grpc.ChannelConnectivity.TRANSIENT_FAILURE) + self.assertFalse(protocol.properties_sent.is_set()) + self.assertEqual(protocol.service_management.sent_properties_counter, 0) + protocol.state = grpc.ChannelConnectivity.TRANSIENT_FAILURE + self.assertFalse(protocol.is_ready()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/test_shutdown_queue.py b/tests/unit/test_shutdown_queue.py index 5258fe99a..a69dc23e2 100644 --- a/tests/unit/test_shutdown_queue.py +++ b/tests/unit/test_shutdown_queue.py @@ -1,377 +1,377 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -import asyncio -import logging -import time -import unittest -from queue import Queue -from threading import Event, Thread - -from skywalking.agent import ( - _abandon_async_queue, - _abandon_sync_queue, - _await_shutdown_or_background_failure, - _cancel_pending_tasks, - _join_sync_queue, - _shutdown_async_queue, - _shutdown_sync_queue, -) - - -class TestShutdownQueueHelpers(unittest.TestCase): - - def test_abandon_sync_queue_unblocks_join(self): - q = Queue() - q.put('a') - q.put('b') - self.assertEqual(q.unfinished_tasks, 2) - - hung = Event() - - def _join(): - q.join() - hung.set() - - Thread(target=_join, daemon=True).start() - time.sleep(0.05) - self.assertFalse(hung.is_set()) - - abandoned = _abandon_sync_queue(q) - self.assertEqual(abandoned, 2) - self.assertTrue(hung.wait(1.0)) - self.assertTrue(_join_sync_queue(q, 0.5)) - - def test_shutdown_sync_skips_flush_when_not_ready(self): - q = Queue() - q.put('x') - called = [] - - def report(): - called.append(1) - raise AssertionError('must not flush when may_send is False') - - _shutdown_sync_queue(report, q, 'test', may_send=False) - self.assertEqual(called, []) - self.assertTrue(q.empty()) - self.assertEqual(q.unfinished_tasks, 0) - - def test_shutdown_sync_flush_timeout_then_abandon(self): - q = Queue() - q.put('x') - started = Event() - - def report(): - started.set() - time.sleep(10) # longer than flush budget - - # Temporarily shrink budget via monkeypatch on module constant - import skywalking.agent as agent_mod - - previous = agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC - try: - agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC = 0.2 - t0 = time.monotonic() - _shutdown_sync_queue(report, q, 'test', may_send=True) - elapsed = time.monotonic() - t0 - finally: - agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC = previous - - self.assertTrue(started.wait(1.0)) - self.assertLess(elapsed, 2.0) - self.assertTrue(q.empty()) - self.assertEqual(q.unfinished_tasks, 0) - - def test_abandon_async_queue_unblocks_join(self): - async def _run(): - q = asyncio.Queue() - await q.put('a') - await q.put('b') - join_task = asyncio.create_task(q.join()) - await asyncio.sleep(0.05) - self.assertFalse(join_task.done()) - abandoned = await _abandon_async_queue(q) - self.assertEqual(abandoned, 2) - await asyncio.wait_for(join_task, timeout=1.0) - - asyncio.run(_run()) - - def test_cancel_pending_tasks_excludes_caller(self): - """Regression: gathering the caller's own task made shutdown hang until the outer budget.""" - loop = asyncio.new_event_loop() - thread = Thread(target=loop.run_forever, daemon=True) - thread.start() - - async def _forever(): - while True: - await asyncio.sleep(0.05) - - try: - async def _spawn(): - return [asyncio.create_task(_forever()) for _ in range(2)] - - reporter_tasks = asyncio.run_coroutine_threadsafe(_spawn(), loop).result(timeout=2.0) - time.sleep(0.1) - - t0 = time.monotonic() - future = asyncio.run_coroutine_threadsafe(_cancel_pending_tasks(reporter_tasks), loop) - future.result(timeout=2.0) - self.assertLess(time.monotonic() - t0, 2.0) - for task in reporter_tasks: - self.assertTrue(task.cancelled() or task.done()) - finally: - loop.call_soon_threadsafe(loop.stop) - thread.join(timeout=2.0) - loop.close() - - def test_async_shutdown_cleanup_runs_inside_asyncio_run_root(self): - """ - Production topology: root waits on _finished, then cancels reporters and - awaits a yielding protocol aclose() before asyncio.run returns. - """ - holder = {} - aclose_entered = Event() - aclose_done = Event() - loop_ready = Event() - - async def yielding_aclose(): - aclose_entered.set() - await asyncio.sleep(0.05) - aclose_done.set() - - async def root(): - finished = asyncio.Event() - holder['finished'] = finished - holder['loop'] = asyncio.get_running_loop() - - async def reporter(): - while not finished.is_set(): - await asyncio.sleep(0.02) - - tasks = {asyncio.create_task(reporter()) for _ in range(2)} - loop_ready.set() - await finished.wait() - await _cancel_pending_tasks(tasks) - await yielding_aclose() - holder['root_finished'] = True - - thread = Thread(target=lambda: asyncio.run(root()), daemon=True) - thread.start() - self.assertTrue(loop_ready.wait(3.0)) - holder['loop'].call_soon_threadsafe(holder['finished'].set) - thread.join(timeout=5.0) - self.assertTrue(aclose_entered.wait(2.0)) - self.assertTrue(aclose_done.wait(2.0)) - self.assertTrue(holder.get('root_finished', False)) - - def test_background_task_failure_is_logged_not_silenced(self): - """Regression: failing background tasks must be observed and logged exactly once.""" - holder = {'errors': []} - error_logged = Event() - aclose_done = Event() - - class _Handler(logging.Handler): - def emit(self, record): - msg = record.getMessage() - if 'Error in Python agent asyncio event loop' in msg: - holder['errors'].append(msg) - error_logged.set() - - agent_logger = logging.getLogger('skywalking') - handler = _Handler() - agent_logger.addHandler(handler) - previous_level = agent_logger.level - agent_logger.setLevel(logging.ERROR) - - async def failing_background(): - await asyncio.sleep(0.02) - raise ValueError('command dispatch failed') - - async def yielding_aclose(): - await asyncio.sleep(0.02) - aclose_done.set() - - async def root(): - finished = asyncio.Event() - holder['finished'] = finished - failing_task = asyncio.create_task(failing_background()) - - async def reporter(): - while not finished.is_set(): - await asyncio.sleep(0.05) - - reporter_task = asyncio.create_task(reporter()) - tasks = {failing_task, reporter_task} - await _await_shutdown_or_background_failure(finished, tasks) - await _cancel_pending_tasks(tasks) - await yielding_aclose() - holder['after_wait'] = True - holder['failing_task'] = failing_task - - try: - asyncio.run(root()) - self.assertTrue(holder.get('after_wait')) - self.assertTrue(error_logged.wait(2.0)) - self.assertTrue(aclose_done.wait(2.0)) - self.assertEqual(len(holder['errors']), 1) - self.assertIn('command dispatch failed', holder['errors'][0]) - self.assertTrue(holder['finished'].is_set()) - self.assertTrue(holder['failing_task'].done()) - self.assertIsInstance(holder['failing_task'].exception(), ValueError) - finally: - agent_logger.removeHandler(handler) - agent_logger.setLevel(previous_level) - - def test_clean_shutdown_does_not_log_normal_background_completion(self): - """Clean shutdown: reporters finish after _finished; cleanup must stay silent.""" - holder = {'errors': []} - - class _Handler(logging.Handler): - def emit(self, record): - if 'Error in Python agent asyncio event loop' in record.getMessage(): - holder['errors'].append(record.getMessage()) - - agent_logger = logging.getLogger('skywalking') - handler = _Handler() - agent_logger.addHandler(handler) - previous_level = agent_logger.level - agent_logger.setLevel(logging.ERROR) - - async def root(): - finished = asyncio.Event() - - async def reporter(): - while not finished.is_set(): - await asyncio.sleep(0.01) - - tasks = {asyncio.create_task(reporter()) for _ in range(2)} - finished.set() - await _await_shutdown_or_background_failure(finished, tasks) - # Give reporters a turn to observe _finished and return normally. - await asyncio.sleep(0.05) - await _cancel_pending_tasks(tasks) - holder['all_done'] = all(task.done() for task in tasks) - - try: - asyncio.run(root()) - self.assertTrue(holder.get('all_done')) - self.assertEqual(holder['errors'], []) - finally: - agent_logger.removeHandler(handler) - agent_logger.setLevel(previous_level) - - def test_unexpected_pre_shutdown_cancellation_is_logged(self): - """Cancellation before shutdown must be logged and trigger orderly cleanup.""" - holder = {'errors': []} - error_logged = Event() - aclose_done = Event() - - class _Handler(logging.Handler): - def emit(self, record): - msg = record.getMessage() - if 'Error in Python agent asyncio event loop' in msg: - holder['errors'].append(msg) - error_logged.set() - - agent_logger = logging.getLogger('skywalking') - handler = _Handler() - agent_logger.addHandler(handler) - previous_level = agent_logger.level - agent_logger.setLevel(logging.ERROR) - - async def cancelled_background(): - asyncio.current_task().cancel() - await asyncio.sleep(0) - - async def yielding_aclose(): - await asyncio.sleep(0.02) - aclose_done.set() - - async def root(): - finished = asyncio.Event() - holder['finished'] = finished - - async def reporter(): - while not finished.is_set(): - await asyncio.sleep(0.05) - - cancelled_task = asyncio.create_task(cancelled_background()) - reporter_task = asyncio.create_task(reporter()) - tasks = {cancelled_task, reporter_task} - await _await_shutdown_or_background_failure(finished, tasks) - await _cancel_pending_tasks(tasks) - await yielding_aclose() - holder['after_wait'] = True - - try: - asyncio.run(root()) - self.assertTrue(holder.get('after_wait')) - self.assertTrue(error_logged.wait(2.0)) - self.assertTrue(aclose_done.wait(2.0)) - self.assertEqual(len(holder['errors']), 1) - self.assertIn('cancelled unexpectedly', holder['errors'][0]) - self.assertTrue(holder['finished'].is_set()) - finally: - agent_logger.removeHandler(handler) - agent_logger.setLevel(previous_level) - - def test_intentional_cleanup_cancellation_is_silent(self): - """Tasks cancelled by _cancel_pending_tasks during cleanup must not ERROR.""" - holder = {'errors': []} - - class _Handler(logging.Handler): - def emit(self, record): - if 'Error in Python agent asyncio event loop' in record.getMessage(): - holder['errors'].append(record.getMessage()) - - agent_logger = logging.getLogger('skywalking') - handler = _Handler() - agent_logger.addHandler(handler) - previous_level = agent_logger.level - agent_logger.setLevel(logging.ERROR) - - async def forever(): - while True: - await asyncio.sleep(0.05) - - async def root(): - tasks = {asyncio.create_task(forever()) for _ in range(2)} - await _cancel_pending_tasks(tasks) - holder['all_done'] = all(task.done() for task in tasks) - - try: - asyncio.run(root()) - self.assertTrue(holder.get('all_done')) - self.assertEqual(holder['errors'], []) - finally: - agent_logger.removeHandler(handler) - agent_logger.setLevel(previous_level) - - def test_shutdown_async_queue_bounded(self): - async def _run(): - q = asyncio.Queue() - await q.put('a') - t0 = time.monotonic() - await _shutdown_async_queue(q, 'test') - self.assertLess(time.monotonic() - t0, 2.0) - self.assertTrue(q.empty()) - - asyncio.run(_run()) - - -if __name__ == '__main__': - unittest.main() +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import asyncio +import logging +import time +import unittest +from queue import Queue +from threading import Event, Thread + +from skywalking.agent import ( + _abandon_async_queue, + _abandon_sync_queue, + _await_shutdown_or_background_failure, + _cancel_pending_tasks, + _join_sync_queue, + _shutdown_async_queue, + _shutdown_sync_queue, +) + + +class TestShutdownQueueHelpers(unittest.TestCase): + + def test_abandon_sync_queue_unblocks_join(self): + q = Queue() + q.put('a') + q.put('b') + self.assertEqual(q.unfinished_tasks, 2) + + hung = Event() + + def _join(): + q.join() + hung.set() + + Thread(target=_join, daemon=True).start() + time.sleep(0.05) + self.assertFalse(hung.is_set()) + + abandoned = _abandon_sync_queue(q) + self.assertEqual(abandoned, 2) + self.assertTrue(hung.wait(1.0)) + self.assertTrue(_join_sync_queue(q, 0.5)) + + def test_shutdown_sync_skips_flush_when_not_ready(self): + q = Queue() + q.put('x') + called = [] + + def report(): + called.append(1) + raise AssertionError('must not flush when may_send is False') + + _shutdown_sync_queue(report, q, 'test', may_send=False) + self.assertEqual(called, []) + self.assertTrue(q.empty()) + self.assertEqual(q.unfinished_tasks, 0) + + def test_shutdown_sync_flush_timeout_then_abandon(self): + q = Queue() + q.put('x') + started = Event() + + def report(): + started.set() + time.sleep(10) # longer than flush budget + + # Temporarily shrink budget via monkeypatch on module constant + import skywalking.agent as agent_mod + + previous = agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC + try: + agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC = 0.2 + t0 = time.monotonic() + _shutdown_sync_queue(report, q, 'test', may_send=True) + elapsed = time.monotonic() - t0 + finally: + agent_mod._SHUTDOWN_FLUSH_TIMEOUT_SEC = previous + + self.assertTrue(started.wait(1.0)) + self.assertLess(elapsed, 2.0) + self.assertTrue(q.empty()) + self.assertEqual(q.unfinished_tasks, 0) + + def test_abandon_async_queue_unblocks_join(self): + async def _run(): + q = asyncio.Queue() + await q.put('a') + await q.put('b') + join_task = asyncio.create_task(q.join()) + await asyncio.sleep(0.05) + self.assertFalse(join_task.done()) + abandoned = await _abandon_async_queue(q) + self.assertEqual(abandoned, 2) + await asyncio.wait_for(join_task, timeout=1.0) + + asyncio.run(_run()) + + def test_cancel_pending_tasks_excludes_caller(self): + """Regression: gathering the caller's own task made shutdown hang until the outer budget.""" + loop = asyncio.new_event_loop() + thread = Thread(target=loop.run_forever, daemon=True) + thread.start() + + async def _forever(): + while True: + await asyncio.sleep(0.05) + + try: + async def _spawn(): + return [asyncio.create_task(_forever()) for _ in range(2)] + + reporter_tasks = asyncio.run_coroutine_threadsafe(_spawn(), loop).result(timeout=2.0) + time.sleep(0.1) + + t0 = time.monotonic() + future = asyncio.run_coroutine_threadsafe(_cancel_pending_tasks(reporter_tasks), loop) + future.result(timeout=2.0) + self.assertLess(time.monotonic() - t0, 2.0) + for task in reporter_tasks: + self.assertTrue(task.cancelled() or task.done()) + finally: + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=2.0) + loop.close() + + def test_async_shutdown_cleanup_runs_inside_asyncio_run_root(self): + """ + Production topology: root waits on _finished, then cancels reporters and + awaits a yielding protocol aclose() before asyncio.run returns. + """ + holder = {} + aclose_entered = Event() + aclose_done = Event() + loop_ready = Event() + + async def yielding_aclose(): + aclose_entered.set() + await asyncio.sleep(0.05) + aclose_done.set() + + async def root(): + finished = asyncio.Event() + holder['finished'] = finished + holder['loop'] = asyncio.get_running_loop() + + async def reporter(): + while not finished.is_set(): + await asyncio.sleep(0.02) + + tasks = {asyncio.create_task(reporter()) for _ in range(2)} + loop_ready.set() + await finished.wait() + await _cancel_pending_tasks(tasks) + await yielding_aclose() + holder['root_finished'] = True + + thread = Thread(target=lambda: asyncio.run(root()), daemon=True) + thread.start() + self.assertTrue(loop_ready.wait(3.0)) + holder['loop'].call_soon_threadsafe(holder['finished'].set) + thread.join(timeout=5.0) + self.assertTrue(aclose_entered.wait(2.0)) + self.assertTrue(aclose_done.wait(2.0)) + self.assertTrue(holder.get('root_finished', False)) + + def test_background_task_failure_is_logged_not_silenced(self): + """Regression: failing background tasks must be observed and logged exactly once.""" + holder = {'errors': []} + error_logged = Event() + aclose_done = Event() + + class _Handler(logging.Handler): + def emit(self, record): + msg = record.getMessage() + if 'Error in Python agent asyncio event loop' in msg: + holder['errors'].append(msg) + error_logged.set() + + agent_logger = logging.getLogger('skywalking') + handler = _Handler() + agent_logger.addHandler(handler) + previous_level = agent_logger.level + agent_logger.setLevel(logging.ERROR) + + async def failing_background(): + await asyncio.sleep(0.02) + raise ValueError('command dispatch failed') + + async def yielding_aclose(): + await asyncio.sleep(0.02) + aclose_done.set() + + async def root(): + finished = asyncio.Event() + holder['finished'] = finished + failing_task = asyncio.create_task(failing_background()) + + async def reporter(): + while not finished.is_set(): + await asyncio.sleep(0.05) + + reporter_task = asyncio.create_task(reporter()) + tasks = {failing_task, reporter_task} + await _await_shutdown_or_background_failure(finished, tasks) + await _cancel_pending_tasks(tasks) + await yielding_aclose() + holder['after_wait'] = True + holder['failing_task'] = failing_task + + try: + asyncio.run(root()) + self.assertTrue(holder.get('after_wait')) + self.assertTrue(error_logged.wait(2.0)) + self.assertTrue(aclose_done.wait(2.0)) + self.assertEqual(len(holder['errors']), 1) + self.assertIn('command dispatch failed', holder['errors'][0]) + self.assertTrue(holder['finished'].is_set()) + self.assertTrue(holder['failing_task'].done()) + self.assertIsInstance(holder['failing_task'].exception(), ValueError) + finally: + agent_logger.removeHandler(handler) + agent_logger.setLevel(previous_level) + + def test_clean_shutdown_does_not_log_normal_background_completion(self): + """Clean shutdown: reporters finish after _finished; cleanup must stay silent.""" + holder = {'errors': []} + + class _Handler(logging.Handler): + def emit(self, record): + if 'Error in Python agent asyncio event loop' in record.getMessage(): + holder['errors'].append(record.getMessage()) + + agent_logger = logging.getLogger('skywalking') + handler = _Handler() + agent_logger.addHandler(handler) + previous_level = agent_logger.level + agent_logger.setLevel(logging.ERROR) + + async def root(): + finished = asyncio.Event() + + async def reporter(): + while not finished.is_set(): + await asyncio.sleep(0.01) + + tasks = {asyncio.create_task(reporter()) for _ in range(2)} + finished.set() + await _await_shutdown_or_background_failure(finished, tasks) + # Give reporters a turn to observe _finished and return normally. + await asyncio.sleep(0.05) + await _cancel_pending_tasks(tasks) + holder['all_done'] = all(task.done() for task in tasks) + + try: + asyncio.run(root()) + self.assertTrue(holder.get('all_done')) + self.assertEqual(holder['errors'], []) + finally: + agent_logger.removeHandler(handler) + agent_logger.setLevel(previous_level) + + def test_unexpected_pre_shutdown_cancellation_is_logged(self): + """Cancellation before shutdown must be logged and trigger orderly cleanup.""" + holder = {'errors': []} + error_logged = Event() + aclose_done = Event() + + class _Handler(logging.Handler): + def emit(self, record): + msg = record.getMessage() + if 'Error in Python agent asyncio event loop' in msg: + holder['errors'].append(msg) + error_logged.set() + + agent_logger = logging.getLogger('skywalking') + handler = _Handler() + agent_logger.addHandler(handler) + previous_level = agent_logger.level + agent_logger.setLevel(logging.ERROR) + + async def cancelled_background(): + asyncio.current_task().cancel() + await asyncio.sleep(0) + + async def yielding_aclose(): + await asyncio.sleep(0.02) + aclose_done.set() + + async def root(): + finished = asyncio.Event() + holder['finished'] = finished + + async def reporter(): + while not finished.is_set(): + await asyncio.sleep(0.05) + + cancelled_task = asyncio.create_task(cancelled_background()) + reporter_task = asyncio.create_task(reporter()) + tasks = {cancelled_task, reporter_task} + await _await_shutdown_or_background_failure(finished, tasks) + await _cancel_pending_tasks(tasks) + await yielding_aclose() + holder['after_wait'] = True + + try: + asyncio.run(root()) + self.assertTrue(holder.get('after_wait')) + self.assertTrue(error_logged.wait(2.0)) + self.assertTrue(aclose_done.wait(2.0)) + self.assertEqual(len(holder['errors']), 1) + self.assertIn('cancelled unexpectedly', holder['errors'][0]) + self.assertTrue(holder['finished'].is_set()) + finally: + agent_logger.removeHandler(handler) + agent_logger.setLevel(previous_level) + + def test_intentional_cleanup_cancellation_is_silent(self): + """Tasks cancelled by _cancel_pending_tasks during cleanup must not ERROR.""" + holder = {'errors': []} + + class _Handler(logging.Handler): + def emit(self, record): + if 'Error in Python agent asyncio event loop' in record.getMessage(): + holder['errors'].append(record.getMessage()) + + agent_logger = logging.getLogger('skywalking') + handler = _Handler() + agent_logger.addHandler(handler) + previous_level = agent_logger.level + agent_logger.setLevel(logging.ERROR) + + async def forever(): + while True: + await asyncio.sleep(0.05) + + async def root(): + tasks = {asyncio.create_task(forever()) for _ in range(2)} + await _cancel_pending_tasks(tasks) + holder['all_done'] = all(task.done() for task in tasks) + + try: + asyncio.run(root()) + self.assertTrue(holder.get('all_done')) + self.assertEqual(holder['errors'], []) + finally: + agent_logger.removeHandler(handler) + agent_logger.setLevel(previous_level) + + def test_shutdown_async_queue_bounded(self): + async def _run(): + q = asyncio.Queue() + await q.put('a') + t0 = time.monotonic() + await _shutdown_async_queue(q, 'test') + self.assertLess(time.monotonic() - t0, 2.0) + self.assertTrue(q.empty()) + + asyncio.run(_run()) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py new file mode 100644 index 000000000..ec1eb3dfd --- /dev/null +++ b/tests/unit/test_tls.py @@ -0,0 +1,602 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import base64 +import os +import ssl +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from skywalking import config +from skywalking.utils import tls as tls_mod +from skywalking.utils.tls import ( + collector_http_scheme, + collector_uses_tls, + normalize_private_key_pem, + requests_tls_settings, + ssl_context_for_collector, + tls_pem_material, +) + +# Self-signed PEMs for unit tests (loadable by OpenSSL; not for production). +_TEST_CA_CERT = b"""-----BEGIN CERTIFICATE----- +MIICxjCCAa6gAwIBAgIUF4Oln8syl8F84oLaGDfp2Y34WNIwDQYJKoZIhvcNAQEL +BQAwHTEbMBkGA1UEAwwSc2t5d2Fsa2luZy10ZXN0LWNhMB4XDTI2MDkxMDIzNTkx +NVoXDTM2MDkwODIzNTkxNVowHTEbMBkGA1UEAwwSc2t5d2Fsa2luZy10ZXN0LWNh +MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA3kWpwJ4D/b7osNRM6khH +22cU4/HMvVZsMqEMl966iDxhOjBsTiY9kUFHdQqqxfyEcSf5cZ9bGgOq+qFE+0ES +O7/xm7mPehMtBfrOIryOLwGru7a09Nt4EzsyNr8Pfa3uUMRGqMjg5BwWL/1/Fl+N +yWNcoe+DzQ1nMu1mOXDgRk+T/Kv4sVrUMUAxbEVr0pYIOBwmksKd1/vgDoZirKD/ +I4natV4RJUkiaDbovE/iUb7D9RqTvKj+xUe9IOY/zRhwZAS9mX3+xSYCS5eik/uP +J0sRt8TBbehtGlbqtfKDqlL75dJxR9noOX69Q4s/xw1bupFvrtkAjfhTTiABC/tJ +4QIDAQABMA0GCSqGSIb3DQEBCwUAA4IBAQBuLlOFq8oN6aymQ6mlaeXIXc+Hs87V +jKxeMkQuEIz5oTqykfz4od7CJHTqaSsJC5o/M7F8KuYHjT9CFxM+bbkCyUkQXPRC +ZEibp8qiqJeUiWnU7bTtoBcY/so5V1DNrer56jajJ3iltnOnw3reiH3BKIjr20hq +jzECpayEsoEFyXUViVsJtd+0A0owJy2h2z9dB6c+C/H5BAuo/M7dbQAIOyQiBXeY +gLcAZ/zj9tMNTxOAcFHppHZS1TVF/Wt5c/shLDxC0XVvutVG8s3q+cqZ/xQyMs9y +wlTT55CMucIIw0KTLNyNCB12fALoaf0AzZbHBCEvTsi4Qf34NW2C7V2s +-----END CERTIFICATE----- +""" + +_TEST_CLIENT_CERT = b"""-----BEGIN CERTIFICATE----- +MIICzjCCAbagAwIBAgIUCipZSsMD9Tl7eGdHHlEujKB/mF8wDQYJKoZIhvcNAQEL +BQAwITEfMB0GA1UEAwwWc2t5d2Fsa2luZy10ZXN0LWNsaWVudDAeFw0yNjA5MTAy +MzU5MTVaFw0zNjA5MDgyMzU5MTVaMCExHzAdBgNVBAMMFnNreXdhbGtpbmctdGVz +dC1jbGllbnQwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDNyXYhjEpM +5H+gOWuEzLQh5cwJtjbLDpCo6756AtffkGxGZI96HFzpBshl8e1AcXWSIdia9P1M +xckVeTgreYHe5YxfYRuhXsOfAYVxM8nb3v1iwieHhetmIHlzNzpGLXluqkwNyBuY +xo3y+AApv6Otxf+v/60Uj6kEjMhJM8O/BEoNwagvl1KA7FknTyJWnQmpHwZpovmP +TNogsjHn1a79MioUq8n1QOCQlfXqssoLllCjQXblv8TiBuaijAXfa66stdoxsQ8+ +GKv+WIC5zQyjhcoKhur37hGlSwSbOKng4MI9zfHMNvPvhRdDachGZ9vl0f+61OLf +aphVI1xn0u1/AgMBAAEwDQYJKoZIhvcNAQELBQADggEBAEptwm4EAQuVUdqTNrPF +47e3E5iZtkBjF2TDFiLJfvjvGZJFj7PbtrFR9UVRxP5l6Y/8GHmLtIvni1C+7HjV +tuW2JpId+miVvM0vHm3FPtjil5JapSDVlNrtkOxGxizOtuf4vx0s1bUlxbDstarx +ASVBFRxdG+XBhAyCC3Jgui2li/mw2oJEk50b37cD9vE8jSuA5qHNLRRqMhRPs+Y4 +CPh4r/MnvhJChhEPzzbUoxejbrccpVBBA2hlwM363tRNr7E3i6tsDFfnChhXlKIH +ypuoPrg+L1WWdF4gkCWhCYdZN7dM0ph1Z+2XZjGZYmk6/US9HOvVImOw39AAr9Ev +bAs= +-----END CERTIFICATE----- +""" + +_TEST_CLIENT_KEY_PKCS1 = b"""-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEAzcl2IYxKTOR/oDlrhMy0IeXMCbY2yw6QqOu+egLX35BsRmSP +ehxc6QbIZfHtQHF1kiHYmvT9TMXJFXk4K3mB3uWMX2EboV7DnwGFcTPJ2979YsIn +h4XrZiB5czc6Ri15bqpMDcgbmMaN8vgAKb+jrcX/r/+tFI+pBIzISTPDvwRKDcGo +L5dSgOxZJ08iVp0JqR8GaaL5j0zaILIx59Wu/TIqFKvJ9UDgkJX16rLKC5ZQo0F2 +5b/E4gbmoowF32uurLXaMbEPPhir/liAuc0Mo4XKCobq9+4RpUsEmzip4ODCPc3x +zDbz74UXQ2nIRmfb5dH/utTi32qYVSNcZ9LtfwIDAQABAoIBADGuZ5Sl1/JEYAOA +mVKQURS189KMaSIQvB/r+ipesVtJ9Lnx4Smr20pu1sa1539dZPMItNCEQPrd7TP/ +9e2ZAh+b742/VfpZdITYyyyPQjaQ2T+UfBKd5DzdrjSAqtLye5SaDI5vNyplFTQJ +Z5CssYlsedQ1t8V1AWsVyezSUHm20iAPFzldYwyhsS9jz0H8EBtMuMU2uo7I6e4a +4WlQAQtzcBNzplcwdAQ9qvba3UFlA62gIsH39Va+QckXROgnISaJMsq1H/RSx12Y +rht8SOxF7GXvjfnesZ6Gfc7gek/8E65ny+CS0xv8xj2HV8W+IrlQFE4pqX8VQ/Qg +OJWdCpECgYEA+rURpz+IAP7aYcGgogKhATJOOJGHqg2grXWUZr6J/NrMI4/ohJg3 +rMoiEJBDb0q/tUxsvuT8aH+UxuZ3gWwcMci5PvldKlrFC+sVpJTlLB+MWbNPWBs/ +dr7riyy/0ZJ5OTt3h/ZsSy2RczFOKWXXNS/r/WvXFtfvV40+McGQ4S0CgYEA0iGf +m9Z0BBpHlwL3sGsZvPI5KflhgMx3D2YDwGonk4fW31k+NcCYR1d+0Ghq0oNyb2qS +09z+ZZA94uiO3t+mtZc5IWLZoJW/dDBiam2tVManluqFRX+i3d2JcJuDNpiI76Wz +Be8NrpG8uOfaO3tovmgwaS1WZ3mBe1R21wBi/NsCgYBRq0x95BdE48B2GeJfBGY4 +go+yo83C2r+d4fCe67D9urTHXOjM0N1KH2qrZKNjDMGYqLXAFc4XqH/pr0f81B/3 +I8Ecv5TW6EzKTiF1xL9G+Vv6GIxfUjkBUL5gTwqJlaKBv1p34xFyB/0avlQM7k0F +2X+RxWCC44LnTW6WPM0aXQKBgQCaGrySLlmRNLCyCCQchr8ueboAlXqzWcArU9aG +g5OYt7OWwz1DcIZ9M6a2Mw28a1g+a7tYkyci1wD76y/0NbNuU4Q7fuI5yfjJvj4+ +7UaD+Nipbj7k9DE+Yx1Lr1EwdfdfQXckb+fp0cnFFYxPuTbdBU4TpINMiaizCQPK +s+bkpwKBgQC0WurqUUJ6EZc6ahUwxeGKVgJcBt4+9Nz+61t0j9mwrga7iIHmOBp/ +hnGpvSFjYbsX7zIPELCZs4GTkpAvyrDLSD1x/ZTLf+cXyX/c/7vF3KtOoKAQT9N0 +VunQSXwz7Gln1ZemLrcAn9XQwovDZGAkT+NrCRda9sFfD8GN3AedUg== +-----END RSA PRIVATE KEY----- +""" + + +class TestCollectorTls(unittest.TestCase): + def setUp(self): + self._saved = ( + config.agent_force_tls, + config.agent_ssl_trusted_ca_path, + config.agent_ssl_cert_chain_path, + config.agent_ssl_key_path, + ) + config.agent_force_tls = False + config.agent_ssl_trusted_ca_path = '' + config.agent_ssl_cert_chain_path = '' + config.agent_ssl_key_path = '' + tls_mod._warned_keys.clear() + tls_mod._cleanup_mtls_temp_files() + tls_mod._mtls_file_cache_key = None + tls_mod._mtls_file_cache = None + + def tearDown(self): + ( + config.agent_force_tls, + config.agent_ssl_trusted_ca_path, + config.agent_ssl_cert_chain_path, + config.agent_ssl_key_path, + ) = self._saved + tls_mod._cleanup_mtls_temp_files() + tls_mod._mtls_file_cache_key = None + tls_mod._mtls_file_cache = None + tls_mod._warned_keys.clear() + + def _write_pem(self, directory: str, name: str, data: bytes) -> Path: + path = Path(directory) / name + path.write_bytes(data) + return path + + def test_plaintext_when_tls_off_and_no_ca(self): + self.assertFalse(collector_uses_tls()) + self.assertIsNone(tls_pem_material()) + self.assertEqual(collector_http_scheme(), 'http://') + + def test_force_tls_uses_system_trust_without_ca(self): + config.agent_force_tls = True + self.assertTrue(collector_uses_tls()) + self.assertEqual(collector_http_scheme(), 'https://') + self.assertEqual(tls_pem_material(), (None, None, None)) + verify, cert = requests_tls_settings() + self.assertTrue(verify) + self.assertIsNone(cert) + + def test_ca_file_enables_tls_without_force_flag(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + self.assertTrue(collector_uses_tls()) + self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) + verify, cert = requests_tls_settings() + self.assertTrue(Path(verify).samefile(ca)) + self.assertIsNone(cert) + + def test_missing_ca_path_warns_and_stays_plaintext(self): + config.agent_ssl_trusted_ca_path = '/nonexistent/skywalking-ca.crt' + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertFalse(collector_uses_tls()) + self.assertIsNone(tls_pem_material()) + self.assertEqual(collector_http_scheme(), 'http://') + self.assertTrue(any('SW_AGENT_SSL_TRUSTED_CA_PATH' in line for line in logs.output)) + self.assertTrue(any('plaintext' in line for line in logs.output)) + + def test_mtls_when_ca_cert_and_key_exist(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + roots, private_key, chain = tls_pem_material() + self.assertEqual(roots, _TEST_CA_CERT) + self.assertEqual(chain, _TEST_CLIENT_CERT) + self.assertIn(b'BEGIN PRIVATE KEY', private_key) + verify, pair = requests_tls_settings() + self.assertTrue(Path(verify).samefile(ca)) + self.assertIsNotNone(pair) + cert_file, key_file = pair + self.assertEqual(Path(cert_file).read_bytes(), _TEST_CLIENT_CERT) + self.assertEqual(Path(key_file).read_bytes(), private_key) + self.assertTrue(all(os.path.exists(p) for p in tls_mod._mtls_temp_files)) + tls_mod._cleanup_mtls_temp_files() + self.assertEqual(tls_mod._mtls_temp_files, []) + + def test_normalize_pkcs1_to_pkcs8(self): + der = b'\x30' + b'\x00' * 31 + body = base64.b64encode(der).decode('ascii') + pkcs1 = ( + '-----BEGIN RSA PRIVATE KEY-----\n' + f'{body}\n' + '-----END RSA PRIVATE KEY-----\n' + ).encode('ascii') + out = normalize_private_key_pem(pkcs1) + text = out.decode('ascii') + self.assertIn('-----BEGIN PRIVATE KEY-----', text) + self.assertIn('-----END PRIVATE KEY-----', text) + self.assertNotIn('BEGIN RSA PRIVATE KEY', text) + + pkcs8 = b'-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n' + self.assertEqual(normalize_private_key_pem(pkcs8), pkcs8) + + def test_normalize_rejects_encrypted_private_key(self): + encrypted = ( + b'-----BEGIN ENCRYPTED PRIVATE KEY-----\n' + b'abc\n' + b'-----END ENCRYPTED PRIVATE KEY-----\n' + ) + with self.assertRaises(ValueError) as ctx: + normalize_private_key_pem(encrypted) + self.assertIn('Passphrase-encrypted', str(ctx.exception)) + + def test_normalize_rejects_legacy_proc_type_encrypted(self): + legacy = ( + b'-----BEGIN RSA PRIVATE KEY-----\n' + b'Proc-Type: 4,ENCRYPTED\n' + b'DEK-Info: AES-256-CBC,0123456789ABCDEF0123456789ABCDEF\n' + b'\n' + b'AAAA\n' + b'-----END RSA PRIVATE KEY-----\n' + ) + with self.assertRaises(ValueError) as ctx: + normalize_private_key_pem(legacy) + self.assertIn('Passphrase-encrypted', str(ctx.exception)) + + def test_mtls_converts_pkcs1_key_for_grpc_and_http(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + + roots, private_key, chain = tls_pem_material() + self.assertEqual(roots, _TEST_CA_CERT) + self.assertEqual(chain, _TEST_CLIENT_CERT) + self.assertIn(b'BEGIN PRIVATE KEY', private_key) + self.assertNotIn(b'BEGIN RSA PRIVATE KEY', private_key) + + verify, pair = requests_tls_settings() + self.assertTrue(Path(verify).samefile(ca)) + self.assertEqual(Path(pair[1]).read_bytes(), private_key) + + def test_missing_key_stays_one_way_tls(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(Path(tmp) / 'missing.pem') + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) + self.assertTrue(any('mTLS' in line for line in logs.output)) + + def test_only_cert_configured_warns(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = '' + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) + self.assertTrue(any('Only one of' in line for line in logs.output)) + + def test_client_certs_ignored_without_ca_file_warns(self): + with tempfile.TemporaryDirectory() as tmp: + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_force_tls = True + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (None, None, None)) + self.assertTrue(any('mTLS is disabled' in line for line in logs.output)) + + def test_symlink_ca_to_file_is_accepted(self): + with tempfile.TemporaryDirectory() as tmp: + real = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + link = Path(tmp) / 'ca-link.crt' + try: + link.symlink_to(real) + except OSError: + self.skipTest('symlinks not available') + config.agent_ssl_trusted_ca_path = str(link) + self.assertTrue(collector_uses_tls()) + self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) + + def test_oversized_pem_is_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_bytes(b'X' * (tls_mod._MAX_PEM_BYTES + 1)) + config.agent_ssl_trusted_ca_path = str(ca) + self.assertTrue(collector_uses_tls()) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertIsNone(tls_pem_material()) + self.assertTrue(any('plaintext' in line for line in logs.output)) + self.assertFalse(any('system trust' in line for line in logs.output)) + self.assertEqual(collector_http_scheme(), 'http://') + verify, cert = requests_tls_settings() + self.assertTrue(verify) + self.assertIsNone(cert) + self.assertIsNone(ssl_context_for_collector()) + + def test_oversized_pem_with_force_tls_uses_system_trust(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_bytes(b'X' * (tls_mod._MAX_PEM_BYTES + 1)) + config.agent_force_tls = True + config.agent_ssl_trusted_ca_path = str(ca) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (None, None, None)) + self.assertTrue(any('process trust store' in line for line in logs.output)) + self.assertEqual(collector_http_scheme(), 'https://') + + def test_bad_ca_content_degrades_to_plaintext(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_text('not-a-pem') + config.agent_ssl_trusted_ca_path = str(ca) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertIsNone(tls_pem_material()) + self.assertEqual(collector_http_scheme(), 'http://') + self.assertIsNone(ssl_context_for_collector()) + self.assertTrue(any('plaintext' in line for line in logs.output)) + verify, cert = requests_tls_settings() + # Material is None → callers that still ask for settings get defaults, + # but HTTP reporter uses scheme from tls_pem_material() → http://. + self.assertTrue(verify) + self.assertIsNone(cert) + + def test_bad_ca_content_with_force_tls_degrades_to_system_trust(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_text('not-a-pem') + config.agent_force_tls = True + config.agent_ssl_trusted_ca_path = str(ca) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (None, None, None)) + ctx = ssl_context_for_collector() + self.assertIsInstance(ctx, ssl.SSLContext) + self.assertTrue(any('process trust store' in line for line in logs.output)) + verify, cert = requests_tls_settings() + self.assertTrue(verify) + self.assertIsNone(cert) + + def test_bad_ca_with_force_and_client_certs_drops_mtls(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_bytes(b'bad') + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_force_tls = True + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (None, None, None)) + self.assertTrue(any('mTLS is disabled' in line for line in logs.output)) + verify, pair = requests_tls_settings() + self.assertTrue(verify) + self.assertIsNone(pair) + + def test_oversized_ca_with_force_and_client_certs_drops_mtls(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + ca.write_bytes(b'X' * (tls_mod._MAX_PEM_BYTES + 1)) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_force_tls = True + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + with self.assertLogs('skywalking', level='WARNING'): + self.assertEqual(tls_pem_material(), (None, None, None)) + verify, pair = requests_tls_settings() + self.assertTrue(verify) + self.assertIsNone(pair) + + def test_invalid_client_material_stays_one_way_tls(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = Path(tmp) / 'client.crt' + key = Path(tmp) / 'client.pem' + crt.write_bytes(b'not-a-cert') + key.write_bytes(b'not-a-key') + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) + self.assertTrue(any('mTLS' in line for line in logs.output)) + + def test_temp_file_failure_stays_one_way_for_requests(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + + def boom(*_a, **_k): + raise OSError(30, 'Read-only file system') + + with patch('tempfile.mkstemp', side_effect=boom), \ + self.assertLogs('skywalking', level='WARNING') as logs: + # Validation also uses mkstemp; either path must stay one-way TLS. + material = tls_pem_material() + if material is not None and material[1] is not None: + verify, pair = requests_tls_settings() + self.assertTrue(Path(verify).samefile(ca)) + self.assertIsNone(pair) + else: + self.assertEqual(material, (_TEST_CA_CERT, None, None)) + self.assertTrue(any('mTLS' in line or 'temp files' in line for line in logs.output)) + + def test_configure_requests_session_never_raises(self): + config.agent_force_tls = True + session = MagicMock() + with patch('skywalking.utils.tls.requests_tls_settings', side_effect=RuntimeError('boom')), \ + self.assertLogs('skywalking', level='WARNING'): + tls_mod.configure_requests_session(session) + self.assertIs(session.verify, True) + self.assertIsNone(session.cert) + + def test_ssl_file_path_rejects_invalid_path(self): + self.assertIsNone(tls_mod.ssl_file_path('a\x00b')) + self.assertIsNone(tls_mod.ssl_file_path('/nonexistent/ca.crt')) + + def test_ssl_file_path_swallows_expanduser_and_resolve_errors(self): + with patch.object(Path, 'expanduser', side_effect=RuntimeError('no home')): + self.assertIsNone(tls_mod.ssl_file_path('~/missing-user-ca.crt')) + + with patch.object(Path, 'resolve', side_effect=RuntimeError('symlink loop')): + self.assertIsNone(tls_mod.ssl_file_path('ca.crt')) + + # HTTP scheme is chosen before configure_requests_session's try/except. + config.agent_ssl_trusted_ca_path = '~/missing-user-ca.crt' + with patch.object(Path, 'expanduser', side_effect=RuntimeError('no home')): + self.assertEqual(collector_http_scheme(), 'http://') + + def test_ca_vanishes_after_exists_check_stays_plaintext_without_force(self): + # Simulate TOCTOU: existence probe said TLS-on, but the CA path is gone + # before material load — without FORCE_TLS must stay plaintext. + config.agent_ssl_trusted_ca_path = '/tmp/vanished-ca.crt' + with patch.object(tls_mod, 'collector_uses_tls', return_value=True), \ + patch.object(tls_mod, 'ssl_file_path', return_value=None): + self.assertIsNone(tls_pem_material()) + self.assertEqual(collector_http_scheme(), 'http://') + + with tempfile.TemporaryDirectory() as tmp: + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + tls_mod._warned_keys.clear() + with patch.object(tls_mod, 'collector_uses_tls', return_value=True), \ + patch.object(tls_mod, 'ssl_file_path', return_value=None), \ + self.assertLogs('skywalking', level='WARNING') as logs: + self.assertIsNone(tls_pem_material()) + self.assertTrue(any('mTLS is disabled' in line for line in logs.output)) + self.assertEqual(collector_http_scheme(), 'http://') + + def test_client_certs_without_ca_warn_when_force_tls_off(self): + with tempfile.TemporaryDirectory() as tmp: + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_force_tls = False + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + with self.assertLogs('skywalking', level='WARNING') as logs: + self.assertIsNone(tls_pem_material()) + self.assertTrue(any('mTLS is disabled' in line for line in logs.output)) + self.assertEqual(collector_http_scheme(), 'http://') + + def test_create_default_context_oserror_degrades(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + + with patch('ssl.create_default_context', side_effect=PermissionError(13, 'Permission denied')), \ + self.assertLogs('skywalking', level='WARNING') as logs: + # Validation and SSLContext build both call create_default_context. + self.assertIsNone(tls_pem_material()) + self.assertTrue(any('plaintext' in line for line in logs.output)) + + config.agent_force_tls = True + tls_mod._warned_keys.clear() + + call_count = {'n': 0} + + def _ctx(*_a, **kwargs): + call_count['n'] += 1 + if kwargs.get('cadata') or kwargs.get('cafile'): + raise PermissionError(13, 'Permission denied') + return ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + with patch('ssl.create_default_context', side_effect=_ctx), \ + self.assertLogs('skywalking', level='WARNING') as logs: + material = tls_pem_material() + self.assertEqual(material, (None, None, None)) + ctx = ssl_context_for_collector() + self.assertIsInstance(ctx, ssl.SSLContext) + self.assertTrue(any('process trust store' in line for line in logs.output)) + + def test_system_trust_failure_after_bad_cadata_does_not_raise(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_force_tls = True + config.agent_ssl_trusted_ca_path = str(ca) + + def _ctx(*_a, **kwargs): + if kwargs.get('cadata') or kwargs.get('cafile'): + raise PermissionError(13, 'Permission denied') + raise PermissionError(13, 'system trust boom') + + # Bypass early CA validation so we exercise ssl_context_for_collector's + # FORCE + cadata-fail → system-trust path. + with patch.object(tls_mod, 'tls_pem_material', return_value=(_TEST_CA_CERT, None, None)), \ + patch('ssl.create_default_context', side_effect=_ctx), \ + self.assertLogs('skywalking', level='WARNING') as logs: + self.assertIsNone(ssl_context_for_collector()) + self.assertTrue(any('plaintext' in line for line in logs.output)) + + def test_after_fork_rebind_clears_child_bookkeeping_without_unlink(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + _verify, pair = requests_tls_settings() + self.assertIsNotNone(pair) + parent_list = tls_mod._mtls_temp_files + cert_file, key_file = pair + self.assertTrue(parent_list) + self.assertTrue(os.path.exists(cert_file)) + + tls_mod._after_fork_in_child() + self.assertIsNot(tls_mod._mtls_temp_files, parent_list) + self.assertEqual(tls_mod._mtls_temp_files, []) + self.assertIsNone(tls_mod._mtls_file_cache) + # Simulate child atexit against the rebound empty list. + tls_mod._cleanup_mtls_temp_files() + self.assertTrue(os.path.exists(cert_file)) + self.assertTrue(os.path.exists(key_file)) + # Parent still holds the original list object with live paths. + self.assertEqual(parent_list, list(pair)) + # Restore parent bookkeeping so tearDown can unlink temps. + tls_mod._mtls_temp_files = parent_list + tls_mod._mtls_file_cache = pair + tls_mod._mtls_file_cache_key = ( + Path(cert_file).read_bytes(), + Path(key_file).read_bytes(), + ) + + @unittest.skipUnless(hasattr(os, 'fork'), 'os.fork required') + def test_fork_child_exit_does_not_delete_parent_mtls_temps(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = self._write_pem(tmp, 'client.pem', _TEST_CLIENT_KEY_PKCS1) + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + verify, pair = requests_tls_settings() + self.assertIsNotNone(pair) + cert_file, key_file = pair + self.assertTrue(os.path.exists(cert_file)) + self.assertTrue(os.path.exists(key_file)) + + pid = os.fork() + if pid == 0: + # Run atexit like a normal shutdown, then hard-exit. Raising + # SystemExit under pytest in a forked child is unreliable + # (non-zero wait status / parent sees the exception). + import atexit + atexit._run_exitfuncs() + os._exit(0) + _pid, status = os.waitpid(pid, 0) + self.assertTrue(os.WIFEXITED(status), status) + self.assertEqual(os.WEXITSTATUS(status), 0) + self.assertTrue(os.path.exists(cert_file), 'parent cert temp deleted by child') + self.assertTrue(os.path.exists(key_file), 'parent key temp deleted by child') + self.assertEqual(tls_mod._mtls_file_cache, pair) + + +if __name__ == '__main__': + unittest.main() From 3d3d5443f2d70d62b08a5672882b0d95fff4f426 Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Sat, 12 Sep 2026 23:20:51 +0800 Subject: [PATCH 2/5] fix: harden TLS PEM parse and CA verify snapshot (P2) Extract PKCS#1 / CERTIFICATE PEM between BEGIN/END only so preamble and UTF-8 BOM cannot break b64decode or aio SSLContext cadata. Always verify HTTP with a process-lifetime CA temp snapshot (not the resolved K8s ..data path) so secret rotation cannot invalidate an open session. --- skywalking/utils/tls.py | 116 ++++++++++++++++++++++++++++++---------- tests/unit/test_tls.py | 103 +++++++++++++++++++++++++++++++++-- 2 files changed, 186 insertions(+), 33 deletions(-) diff --git a/skywalking/utils/tls.py b/skywalking/utils/tls.py index 27b14f3fb..cf36f2a88 100644 --- a/skywalking/utils/tls.py +++ b/skywalking/utils/tls.py @@ -131,6 +131,10 @@ def _load_trusted_ca(path: Path) -> bytes: Raises OSError / ValueError when the file is unreadable, oversized, or not a CA bundle OpenSSL can load — callers degrade instead of handing garbage to gRPC / requests (which may only fail at connect time). + + Returns ASCII PEM with only ``CERTIFICATE`` blocks extracted, so UTF-8 BOM + / preamble accepted by ``cafile`` remain usable for ``cadata`` and temp + verify files. """ data = _read_bytes(path) try: @@ -138,7 +142,7 @@ def _load_trusted_ca(path: Path) -> bytes: except OSError as exc: # ssl.SSLError subclasses OSError on CPython. raise ValueError(f'Invalid trusted CA PEM {path}: {exc}') from exc - return data + return _extract_pem_blocks(data, 'CERTIFICATE') def _validate_client_cert_key(cert_pem: bytes, key_pem: bytes) -> None: @@ -185,6 +189,10 @@ def normalize_private_key_pem(key_pem: bytes) -> bytes: Passphrase-encrypted PEMs (PKCS#8 encrypted or legacy OpenSSL Proc-Type) are rejected with a clear error (not supported). + + Only the bytes between the matching PKCS#1 BEGIN/END delimiters are + decoded — comments or ``openssl rsa -text`` preamble outside the block + must not corrupt the key (OpenSSL accepts such files). """ text = key_pem.decode('utf-8', errors='ignore') if _ENCRYPTED_PEM_HEADER in text: @@ -201,10 +209,14 @@ def normalize_private_key_pem(key_pem: bytes) -> bytes: if _PKCS1_PEM_HEADER not in text: return key_pem - body = text.replace(_PKCS1_PEM_HEADER, '').replace(_PKCS1_PEM_FOOTER, '') + start = text.find(_PKCS1_PEM_HEADER) + end = text.find(_PKCS1_PEM_FOOTER, start) + if start < 0 or end < 0: + raise ValueError('Invalid PKCS#1 private key PEM: missing BEGIN/END delimiters') + body = text[start + len(_PKCS1_PEM_HEADER):end] body = body.replace('\r', '').replace('\n', '').replace(' ', '') try: - pkcs1 = base64.b64decode(body) + pkcs1 = base64.b64decode(body, validate=False) except ValueError as exc: raise ValueError(f'Invalid PKCS#1 private key PEM: {exc}') from exc @@ -229,6 +241,34 @@ def normalize_private_key_pem(key_pem: bytes) -> bytes: return pem.encode('ascii') +def _extract_pem_blocks(data: bytes, label: str) -> bytes: + """ + Return ASCII PEM containing only ``BEGIN/END {label}`` blocks. + + Strips UTF-8 BOM and ignores preamble/comments outside PEM delimiters so + OpenSSL-accepted files (BOM, UTF-8 comments) stay usable for ``cadata`` and + temp-file verify paths that require clean ASCII PEM. + """ + text = data.decode('utf-8', errors='ignore').lstrip('\ufeff') + header = f'-----BEGIN {label}-----' + footer = f'-----END {label}-----' + blocks: List[str] = [] + pos = 0 + while True: + start = text.find(header, pos) + if start < 0: + break + end = text.find(footer, start) + if end < 0: + break + end += len(footer) + blocks.append(text[start:end].strip() + '\n') + pos = end + if not blocks: + raise ValueError(f'No {label} PEM block found') + return ''.join(blocks).encode('ascii') + + def _mtls_material(*, ca_usable: bool) -> Tuple[Optional[bytes], Optional[bytes]]: """ Client certificate_chain and private_key PEM bytes, or (None, None). @@ -282,20 +322,27 @@ def _mtls_material(*, ca_usable: bool) -> Tuple[Optional[bytes], Optional[bytes] return None, None -# Keep mTLS temp PEM paths alive for the process (requests/ssl need file paths). +# Keep mTLS / CA temp PEM paths alive for the process (requests/ssl need file paths). _mtls_temp_files: List[str] = [] _mtls_file_cache_key: Optional[Tuple[bytes, bytes]] = None _mtls_file_cache: Optional[Tuple[str, str]] = None +_ca_file_cache_key: Optional[bytes] = None +_ca_file_cache: Optional[str] = None _atexit_registered = False def _cleanup_mtls_temp_files() -> None: + global _ca_file_cache, _ca_file_cache_key, _mtls_file_cache, _mtls_file_cache_key for path in list(_mtls_temp_files): try: os.unlink(path) except OSError: pass _mtls_temp_files.clear() + _mtls_file_cache = None + _mtls_file_cache_key = None + _ca_file_cache = None + _ca_file_cache_key = None def _after_fork_in_child() -> None: @@ -307,9 +354,12 @@ def _after_fork_in_child() -> None: instead so the child's atexit handler cannot delete the parent's files. """ global _mtls_temp_files, _mtls_file_cache, _mtls_file_cache_key + global _ca_file_cache, _ca_file_cache_key _mtls_temp_files = [] _mtls_file_cache = None _mtls_file_cache_key = None + _ca_file_cache = None + _ca_file_cache_key = None if hasattr(os, 'register_at_fork'): @@ -520,6 +570,21 @@ def collector_http_scheme() -> str: return 'https://' if config.agent_force_tls else 'http://' +def _ca_verify_temp_file(root_certificates: bytes) -> str: + """Process-lifetime temp path for trusted CA bytes (survives K8s secret rotation).""" + global _ca_file_cache_key, _ca_file_cache + if ( + _ca_file_cache is not None + and _ca_file_cache_key == root_certificates + and os.path.exists(_ca_file_cache) + ): + return _ca_file_cache + path = _pem_bytes_to_temp_file(root_certificates, '.crt') + _ca_file_cache_key = root_certificates + _ca_file_cache = path + return path + + def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: """ (verify, cert) for requests.Session. @@ -530,35 +595,29 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: Keeps the same enable/disable decision as grpc_ssl_credentials / tls_pem_material so an unreadable or oversized CA cannot leave HTTP on https:// with a bad verify path. - When custom CA bytes were loaded but the CA path is no longer readable (symlink - race), verify uses a process-local temp PEM of those bytes instead of silently - falling back to the system trust store. + Custom CA always uses a process-lifetime temp snapshot of the validated PEM + bytes (never the resolved symlink target), so Kubernetes secret rotation that + removes the old ``..data`` version cannot invalidate an already-configured + session. Temp-file failures drop client certs only (one-way TLS), with a warning. """ - from skywalking import config - material = tls_pem_material() if material is None: return True, None root_certificates, private_key, certificate_chain = material - ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) if root_certificates is not None: - if ca_path is not None: - verify: object = str(ca_path) - else: - # Prefer in-memory CA over system trust when the path raced away. - try: - verify = _pem_bytes_to_temp_file(root_certificates, '.crt') - except OSError as exc: - _warn_once( - f'requests-ca-temp:{exc}', - 'Failed to persist trusted CA temp file (%s); ' - 'using process trust store for HTTP verify.', - exc, - ) - verify = True + try: + verify: object = _ca_verify_temp_file(root_certificates) + except OSError as exc: + _warn_once( + f'requests-ca-temp:{exc}', + 'Failed to persist trusted CA temp file (%s); ' + 'using process trust store for HTTP verify.', + exc, + ) + verify = True else: verify = True @@ -599,8 +658,9 @@ def ssl_context_for_collector() -> Optional[ssl.SSLContext]: - bad / unparsable custom CA without FORCE_TLS → plaintext (None) - bad client cert/key or temp-file failure → one-way TLS (CA or system trust) - Custom CA is loaded from in-memory PEM bytes (``cadata``) when present so a - path race after ``tls_pem_material`` cannot silently switch to system trust. + Custom CA is loaded from normalized in-memory PEM bytes (``cadata``) when + present so path races and UTF-8 BOM/preamble cannot drop the custom trust + store after ``tls_pem_material`` already accepted the CA. """ from skywalking import config @@ -616,9 +676,9 @@ def _system_trust_context() -> ssl.SSLContext: ctx: Optional[ssl.SSLContext] if root_certificates is not None: try: - # cadata avoids re-reading the CA path (TOCTOU with K8s secret mounts). + # Normalized ASCII PEM from _load_trusted_ca (BOM/preamble stripped). ctx = ssl.create_default_context( - cadata=root_certificates.decode('ascii', errors='strict'), + cadata=root_certificates.decode('ascii'), ) except (OSError, ValueError) as exc: # ssl.SSLError subclasses OSError; ValueError: non-ASCII PEM bytes. diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py index ec1eb3dfd..31531d3cb 100644 --- a/tests/unit/test_tls.py +++ b/tests/unit/test_tls.py @@ -159,7 +159,13 @@ def test_ca_file_enables_tls_without_force_flag(self): self.assertTrue(collector_uses_tls()) self.assertEqual(tls_pem_material(), (_TEST_CA_CERT, None, None)) verify, cert = requests_tls_settings() - self.assertTrue(Path(verify).samefile(ca)) + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).is_file()) + self.assertFalse(Path(verify).samefile(ca)) + self.assertEqual( + Path(verify).read_bytes(), + tls_mod._extract_pem_blocks(_TEST_CA_CERT, 'CERTIFICATE'), + ) self.assertIsNone(cert) def test_missing_ca_path_warns_and_stays_plaintext(self): @@ -184,7 +190,9 @@ def test_mtls_when_ca_cert_and_key_exist(self): self.assertEqual(chain, _TEST_CLIENT_CERT) self.assertIn(b'BEGIN PRIVATE KEY', private_key) verify, pair = requests_tls_settings() - self.assertTrue(Path(verify).samefile(ca)) + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).is_file()) + self.assertFalse(Path(verify).samefile(ca)) self.assertIsNotNone(pair) cert_file, key_file = pair self.assertEqual(Path(cert_file).read_bytes(), _TEST_CLIENT_CERT) @@ -210,6 +218,28 @@ def test_normalize_pkcs1_to_pkcs8(self): pkcs8 = b'-----BEGIN PRIVATE KEY-----\nabc\n-----END PRIVATE KEY-----\n' self.assertEqual(normalize_private_key_pem(pkcs8), pkcs8) + def test_normalize_pkcs1_ignores_preamble_outside_pem_block(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + crt = self._write_pem(tmp, 'client.crt', _TEST_CLIENT_CERT) + key = Path(tmp) / 'key.pem' + key.write_bytes(b'# This is a private key\n' + _TEST_CLIENT_KEY_PKCS1) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_cert_chain(crt, key) + + normalized = normalize_private_key_pem(key.read_bytes()) + normalized_path = Path(tmp) / 'normalized.pem' + normalized_path.write_bytes(normalized) + ctx.load_cert_chain(crt, normalized_path) # must not raise + + config.agent_ssl_trusted_ca_path = str(ca) + config.agent_ssl_cert_chain_path = str(crt) + config.agent_ssl_key_path = str(key) + roots, private_key, chain = tls_pem_material() + self.assertIsNotNone(roots) + self.assertIsNotNone(private_key) + self.assertIsNotNone(chain) + def test_normalize_rejects_encrypted_private_key(self): encrypted = ( b'-----BEGIN ENCRYPTED PRIVATE KEY-----\n' @@ -249,7 +279,9 @@ def test_mtls_converts_pkcs1_key_for_grpc_and_http(self): self.assertNotIn(b'BEGIN RSA PRIVATE KEY', private_key) verify, pair = requests_tls_settings() - self.assertTrue(Path(verify).samefile(ca)) + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).is_file()) + self.assertFalse(Path(verify).samefile(ca)) self.assertEqual(Path(pair[1]).read_bytes(), private_key) def test_missing_key_stays_one_way_tls(self): @@ -420,7 +452,9 @@ def boom(*_a, **_k): material = tls_pem_material() if material is not None and material[1] is not None: verify, pair = requests_tls_settings() - self.assertTrue(Path(verify).samefile(ca)) + # CA snapshot and client temps both need mkstemp; CA may fall + # back to system trust when temp creation fails. + self.assertTrue(verify is True or (isinstance(verify, str) and Path(verify).is_file())) self.assertIsNone(pair) else: self.assertEqual(material, (_TEST_CA_CERT, None, None)) @@ -485,6 +519,58 @@ def test_client_certs_without_ca_warn_when_force_tls_off(self): self.assertTrue(any('mTLS is disabled' in line for line in logs.output)) self.assertEqual(collector_http_scheme(), 'http://') + def test_requests_verify_survives_k8s_style_ca_rotation(self): + import shutil + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for version in ('v1', 'v2'): + (root / version).mkdir() + (root / version / 'ca.crt').write_bytes(_TEST_CA_CERT) + try: + (root / '..data').symlink_to('v1') + ca = root / 'ca.crt' + ca.symlink_to(Path('..data') / 'ca.crt') + except OSError: + self.skipTest('symlinks not available') + if tls_mod.ssl_file_path(str(ca)) is None: + self.skipTest('symlink CA path not readable as regular file') + config.agent_ssl_trusted_ca_path = str(ca) + + verify, _cert = requests_tls_settings() + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).is_file()) + stored = Path(verify) + + try: + (root / '..data-next').symlink_to('v2') + os.replace(root / '..data-next', root / '..data') + shutil.rmtree(root / 'v1') + except OSError: + self.skipTest('symlink rotation not available') + self.assertEqual(ca.read_bytes(), _TEST_CA_CERT) + self.assertTrue(stored.is_file()) + self.assertEqual( + stored.read_bytes(), + tls_mod._extract_pem_blocks(_TEST_CA_CERT, 'CERTIFICATE'), + ) + + def test_aio_context_accepts_bom_and_utf8_preamble_ca(self): + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.crt' + config.agent_ssl_trusted_ca_path = str(ca) + for label, prefix in [ + ('plain', b''), + ('bom', b'\xef\xbb\xbf'), + ('utf8-comment', '# 注释\n'.encode('utf-8')), + ]: + ca.write_bytes(prefix + _TEST_CA_CERT) + ssl.create_default_context(cafile=str(ca)) + tls_mod._warned_keys.clear() + self.assertEqual(collector_http_scheme(), 'https://', label) + ctx = ssl_context_for_collector() + self.assertIsInstance(ctx, ssl.SSLContext, label) + def test_create_default_context_oserror_degrades(self): with tempfile.TemporaryDirectory() as tmp: ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) @@ -547,18 +633,22 @@ def test_after_fork_rebind_clears_child_bookkeeping_without_unlink(self): parent_list = tls_mod._mtls_temp_files cert_file, key_file = pair self.assertTrue(parent_list) + self.assertIn(cert_file, parent_list) + self.assertIn(key_file, parent_list) self.assertTrue(os.path.exists(cert_file)) tls_mod._after_fork_in_child() self.assertIsNot(tls_mod._mtls_temp_files, parent_list) self.assertEqual(tls_mod._mtls_temp_files, []) self.assertIsNone(tls_mod._mtls_file_cache) + self.assertIsNone(tls_mod._ca_file_cache) # Simulate child atexit against the rebound empty list. tls_mod._cleanup_mtls_temp_files() self.assertTrue(os.path.exists(cert_file)) self.assertTrue(os.path.exists(key_file)) # Parent still holds the original list object with live paths. - self.assertEqual(parent_list, list(pair)) + self.assertIn(cert_file, parent_list) + self.assertIn(key_file, parent_list) # Restore parent bookkeeping so tearDown can unlink temps. tls_mod._mtls_temp_files = parent_list tls_mod._mtls_file_cache = pair @@ -566,6 +656,9 @@ def test_after_fork_rebind_clears_child_bookkeeping_without_unlink(self): Path(cert_file).read_bytes(), Path(key_file).read_bytes(), ) + if _verify is not True and isinstance(_verify, str): + tls_mod._ca_file_cache = _verify + tls_mod._ca_file_cache_key = Path(_verify).read_bytes() @unittest.skipUnless(hasattr(os, 'fork'), 'os.fork required') def test_fork_child_exit_does_not_delete_parent_mtls_temps(self): From 1327f8b2b2a6da6f7c52dd44fbadd3ef5413ef8c Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Sun, 13 Sep 2026 10:50:44 +0800 Subject: [PATCH 3/5] fix: keep custom CA on temp fail and TRUSTED CERTIFICATE PEMs (P2) When the HTTP CA snapshot cannot be written, fall back to the still-readable configured CA path instead of Requests' system trust store. Extract both CERTIFICATE and TRUSTED CERTIFICATE blocks (labels/trust attrs preserved) so openssl -trustout CAs remain usable for sync HTTPS; aio falls back to cafile when cadata rejects TRUSTED CERTIFICATE. --- skywalking/utils/tls.py | 106 +++++++++++++++++++++++++++++++++------- tests/unit/test_tls.py | 85 ++++++++++++++++++++++++++++++-- 2 files changed, 167 insertions(+), 24 deletions(-) diff --git a/skywalking/utils/tls.py b/skywalking/utils/tls.py index cf36f2a88..f0c25f96a 100644 --- a/skywalking/utils/tls.py +++ b/skywalking/utils/tls.py @@ -132,9 +132,9 @@ def _load_trusted_ca(path: Path) -> bytes: a CA bundle OpenSSL can load — callers degrade instead of handing garbage to gRPC / requests (which may only fail at connect time). - Returns ASCII PEM with only ``CERTIFICATE`` blocks extracted, so UTF-8 BOM - / preamble accepted by ``cafile`` remain usable for ``cadata`` and temp - verify files. + Returns ASCII PEM with ``CERTIFICATE`` and/or ``TRUSTED CERTIFICATE`` blocks + extracted (labels and trust attributes preserved). UTF-8 BOM / preamble + accepted by ``cafile`` stay usable for ``cadata`` and temp verify files. """ data = _read_bytes(path) try: @@ -142,7 +142,7 @@ def _load_trusted_ca(path: Path) -> bytes: except OSError as exc: # ssl.SSLError subclasses OSError on CPython. raise ValueError(f'Invalid trusted CA PEM {path}: {exc}') from exc - return _extract_pem_blocks(data, 'CERTIFICATE') + return _extract_ca_pem_blocks(data) def _validate_client_cert_key(cert_pem: bytes, key_pem: bytes) -> None: @@ -269,6 +269,41 @@ def _extract_pem_blocks(data: bytes, label: str) -> bytes: return ''.join(blocks).encode('ascii') +# OpenSSL ``openssl x509 -trustout`` emits TRUSTED CERTIFICATE (with trust +# auxiliary). Keep that label — do not relabel as CERTIFICATE. +_CA_PEM_LABELS = ('TRUSTED CERTIFICATE', 'CERTIFICATE') + + +def _extract_ca_pem_blocks(data: bytes) -> bytes: + """ + Return ASCII PEM with ``CERTIFICATE`` / ``TRUSTED CERTIFICATE`` blocks only. + + Preserves each block's PEM label and trust attributes. Strips UTF-8 BOM and + ignores preamble outside delimiters. ``TRUSTED CERTIFICATE`` is searched + first so its header is not confused with plain ``CERTIFICATE``. + """ + text = data.decode('utf-8', errors='ignore').lstrip('\ufeff') + found: List[Tuple[int, str]] = [] + for label in _CA_PEM_LABELS: + header = f'-----BEGIN {label}-----' + footer = f'-----END {label}-----' + pos = 0 + while True: + start = text.find(header, pos) + if start < 0: + break + end = text.find(footer, start) + if end < 0: + break + end += len(footer) + found.append((start, text[start:end].strip() + '\n')) + pos = end + if not found: + raise ValueError('No CERTIFICATE or TRUSTED CERTIFICATE PEM block found') + found.sort(key=lambda item: item[0]) + return ''.join(block for _, block in found).encode('ascii') + + def _mtls_material(*, ca_usable: bool) -> Tuple[Optional[bytes], Optional[bytes]]: """ Client certificate_chain and private_key PEM bytes, or (None, None). @@ -595,13 +630,17 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: Keeps the same enable/disable decision as grpc_ssl_credentials / tls_pem_material so an unreadable or oversized CA cannot leave HTTP on https:// with a bad verify path. - Custom CA always uses a process-lifetime temp snapshot of the validated PEM - bytes (never the resolved symlink target), so Kubernetes secret rotation that + Custom CA prefers a process-lifetime temp snapshot of the validated PEM bytes + (never the resolved symlink target), so Kubernetes secret rotation that removes the old ``..data`` version cannot invalidate an already-configured - session. + session. If the snapshot cannot be written (e.g. read-only temp dir), fall + back to the still-readable configured CA path rather than discarding the + private CA for Requests' default trust store. - Temp-file failures drop client certs only (one-way TLS), with a warning. + Client-cert temp-file failures drop mTLS only (one-way TLS), with a warning. """ + from skywalking import config + material = tls_pem_material() if material is None: return True, None @@ -611,13 +650,23 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: try: verify: object = _ca_verify_temp_file(root_certificates) except OSError as exc: - _warn_once( - f'requests-ca-temp:{exc}', - 'Failed to persist trusted CA temp file (%s); ' - 'using process trust store for HTTP verify.', - exc, - ) - verify = True + ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) + if ca_path is not None: + _warn_once( + f'requests-ca-temp:{exc}', + 'Failed to persist trusted CA temp file (%s); ' + 'falling back to configured CA path for HTTP verify.', + exc, + ) + verify = str(ca_path) + else: + _warn_once( + f'requests-ca-temp:{exc}', + 'Failed to persist trusted CA temp file (%s); ' + 'using process trust store for HTTP verify.', + exc, + ) + verify = True else: verify = True @@ -675,19 +724,38 @@ def _system_trust_context() -> ssl.SSLContext: ctx: Optional[ssl.SSLContext] if root_certificates is not None: + load_exc: Optional[BaseException] = None try: # Normalized ASCII PEM from _load_trusted_ca (BOM/preamble stripped). + # Plain CERTIFICATE works via cadata; TRUSTED CERTIFICATE often needs cafile. ctx = ssl.create_default_context( cadata=root_certificates.decode('ascii'), ) except (OSError, ValueError) as exc: + load_exc = exc + ctx = None + try: + cafile = _ca_verify_temp_file(root_certificates) + ctx = ssl.create_default_context(cafile=cafile) + load_exc = None + except OSError as temp_exc: + load_exc = temp_exc + ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) + if ca_path is not None: + try: + ctx = ssl.create_default_context(cafile=str(ca_path)) + load_exc = None + except OSError as path_exc: + load_exc = path_exc + ctx = None + if load_exc is not None: # ssl.SSLError subclasses OSError; ValueError: non-ASCII PEM bytes. if config.agent_force_tls: _warn_once( - f'ssl-ctx-ca:{exc}', + f'ssl-ctx-ca:{load_exc}', 'Failed to load trusted CA into SSLContext (%s); continuing with ' 'FORCE_TLS and the process trust store (mTLS disabled).', - exc, + load_exc, ) try: ctx = _system_trust_context() @@ -703,10 +771,10 @@ def _system_trust_context() -> ssl.SSLContext: certificate_chain = None else: _warn_once( - f'ssl-ctx-ca-plain:{exc}', + f'ssl-ctx-ca-plain:{load_exc}', 'Failed to load trusted CA into SSLContext (%s); collector stays ' 'plaintext (set SW_AGENT_FORCE_TLS to use the process trust store).', - exc, + load_exc, ) return None else: diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py index 31531d3cb..e3f515214 100644 --- a/tests/unit/test_tls.py +++ b/tests/unit/test_tls.py @@ -164,7 +164,7 @@ def test_ca_file_enables_tls_without_force_flag(self): self.assertFalse(Path(verify).samefile(ca)) self.assertEqual( Path(verify).read_bytes(), - tls_mod._extract_pem_blocks(_TEST_CA_CERT, 'CERTIFICATE'), + tls_mod._extract_ca_pem_blocks(_TEST_CA_CERT), ) self.assertIsNone(cert) @@ -452,13 +452,13 @@ def boom(*_a, **_k): material = tls_pem_material() if material is not None and material[1] is not None: verify, pair = requests_tls_settings() - # CA snapshot and client temps both need mkstemp; CA may fall - # back to system trust when temp creation fails. + # CA snapshot may fall back to the configured path; client temps fail. self.assertTrue(verify is True or (isinstance(verify, str) and Path(verify).is_file())) self.assertIsNone(pair) else: self.assertEqual(material, (_TEST_CA_CERT, None, None)) - self.assertTrue(any('mTLS' in line or 'temp files' in line for line in logs.output)) + self.assertTrue(any('mTLS' in line or 'temp files' in line or 'configured CA path' in line + for line in logs.output)) def test_configure_requests_session_never_raises(self): config.agent_force_tls = True @@ -552,9 +552,84 @@ def test_requests_verify_survives_k8s_style_ca_rotation(self): self.assertTrue(stored.is_file()) self.assertEqual( stored.read_bytes(), - tls_mod._extract_pem_blocks(_TEST_CA_CERT, 'CERTIFICATE'), + tls_mod._extract_ca_pem_blocks(_TEST_CA_CERT), ) + def test_extract_ca_pem_blocks_preserves_trusted_label(self): + mixed = ( + b'\xef\xbb\xbf# preamble \xe6\xb3\xa8\xe9\x87\x8a\n' + b'-----BEGIN TRUSTED CERTIFICATE-----\n' + b'THJ1c3RlZA==\n' + b'-----END TRUSTED CERTIFICATE-----\n' + b'-----BEGIN CERTIFICATE-----\n' + b'Y2VydA==\n' + b'-----END CERTIFICATE-----\n' + ) + out = tls_mod._extract_ca_pem_blocks(mixed) + self.assertTrue(out.startswith(b'-----BEGIN TRUSTED CERTIFICATE-----')) + self.assertIn(b'-----BEGIN CERTIFICATE-----', out) + self.assertIn(b'THJ1c3RlZA==', out) + self.assertNotIn(b'preamble', out) + self.assertNotIn('\ufeff'.encode('utf-8'), out) + + def test_requests_ca_temp_failure_falls_back_to_configured_path(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + resolved = tls_mod.ssl_file_path(str(ca)) + self.assertIsNotNone(resolved) + + def boom(*_a, **_k): + raise OSError(30, 'Read-only file system') + + with patch('tempfile.mkstemp', side_effect=boom), \ + self.assertLogs('skywalking', level='WARNING') as logs: + verify, cert = requests_tls_settings() + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).samefile(resolved)) + self.assertIsNone(cert) + self.assertTrue(any('configured CA path' in line for line in logs.output)) + + def test_trusted_certificate_pem_enables_sync_http_tls(self): + import shutil + import subprocess + + if shutil.which('openssl') is None: + self.skipTest('openssl not available') + with tempfile.TemporaryDirectory() as tmp: + ca = Path(tmp) / 'ca.pem' + trusted = Path(tmp) / 'trusted.pem' + ca.write_bytes(_TEST_CA_CERT) + try: + subprocess.run( + [ + 'openssl', 'x509', '-in', str(ca), '-addtrust', 'serverAuth', + '-trustout', '-out', str(trusted), + ], + check=True, + capture_output=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + self.skipTest(f'openssl trustout failed: {exc}') + text = trusted.read_text(encoding='ascii', errors='ignore') + self.assertIn('BEGIN TRUSTED CERTIFICATE', text) + ssl.create_default_context(cafile=str(trusted)) + config.agent_ssl_trusted_ca_path = str(trusted) + material = tls_pem_material() + self.assertIsNotNone(material) + roots, private_key, chain = material + self.assertIsNotNone(roots) + self.assertIn(b'BEGIN TRUSTED CERTIFICATE', roots) + self.assertNotIn(b'-----BEGIN CERTIFICATE-----', roots) + self.assertIsNone(private_key) + self.assertIsNone(chain) + self.assertEqual(collector_http_scheme(), 'https://') + verify, cert = requests_tls_settings() + self.assertIsInstance(verify, str) + self.assertTrue(Path(verify).is_file()) + self.assertIn(b'BEGIN TRUSTED CERTIFICATE', Path(verify).read_bytes()) + self.assertIsNone(cert) + def test_aio_context_accepts_bom_and_utf8_preamble_ca(self): with tempfile.TemporaryDirectory() as tmp: ca = Path(tmp) / 'ca.crt' From 3b3906deafd31a0a5c0049e1e8380cf5735e1cc2 Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Wed, 16 Sep 2026 08:21:48 +0800 Subject: [PATCH 4/5] fix: keep CA symlink on no-temp fallback and mixed TRUSTED cafile (P2) When the HTTP CA snapshot cannot be written, fall back to the configured path without resolving symlinks so K8s secret rotation cannot invalidate session.verify. Prefer cafile whenever the CA PEM includes TRUSTED CERTIFICATE so aio cadata cannot silently drop trusted blocks from a mixed bundle. --- skywalking/utils/tls.py | 100 +++++++++++++++++++++---------- tests/unit/test_tls.py | 130 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 195 insertions(+), 35 deletions(-) diff --git a/skywalking/utils/tls.py b/skywalking/utils/tls.py index f0c25f96a..15d459f1d 100644 --- a/skywalking/utils/tls.py +++ b/skywalking/utils/tls.py @@ -620,6 +620,34 @@ def _ca_verify_temp_file(root_certificates: bytes) -> str: return path +def _configured_ca_path_for_verify() -> Optional[str]: + """ + Absolute, expanduser'd configured CA path **without** resolving symlinks. + + Validates that a regular file is reachable (via ``ssl_file_path``), but + returns the configured path so Kubernetes ``ca.crt -> ..data/ca.crt`` stays + stable across secret rotation when a temp CA snapshot cannot be written. + ``ssl_file_path`` / ``Path.resolve`` would pin ``session.verify`` to a + versioned ``v1/`` target that rotation deletes. + """ + from skywalking import config + + text = _configured_path(config.agent_ssl_trusted_ca_path) + if not text or ssl_file_path(text) is None: + return None + try: + path = Path(text).expanduser() + # absolute() does not follow symlinks (unlike resolve()). + return str(path.absolute()) + except (OSError, ValueError, RuntimeError): + return None + + +def _ca_pem_has_trusted_certificate(root_certificates: bytes) -> bool: + """True when extracted CA bytes include an OpenSSL TRUSTED CERTIFICATE block.""" + return b'BEGIN TRUSTED CERTIFICATE' in root_certificates + + def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: """ (verify, cert) for requests.Session. @@ -634,13 +662,12 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: (never the resolved symlink target), so Kubernetes secret rotation that removes the old ``..data`` version cannot invalidate an already-configured session. If the snapshot cannot be written (e.g. read-only temp dir), fall - back to the still-readable configured CA path rather than discarding the - private CA for Requests' default trust store. + back to the configured CA path **without resolving symlinks** (still + validated as readable) rather than discarding the private CA for Requests' + default trust store or pinning a deleted ``v1/`` target after rotation. Client-cert temp-file failures drop mTLS only (one-way TLS), with a warning. """ - from skywalking import config - material = tls_pem_material() if material is None: return True, None @@ -650,15 +677,15 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: try: verify: object = _ca_verify_temp_file(root_certificates) except OSError as exc: - ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) - if ca_path is not None: + ca_verify = _configured_ca_path_for_verify() + if ca_verify is not None: _warn_once( f'requests-ca-temp:{exc}', 'Failed to persist trusted CA temp file (%s); ' 'falling back to configured CA path for HTTP verify.', exc, ) - verify = str(ca_path) + verify = ca_verify else: _warn_once( f'requests-ca-temp:{exc}', @@ -707,9 +734,12 @@ def ssl_context_for_collector() -> Optional[ssl.SSLContext]: - bad / unparsable custom CA without FORCE_TLS → plaintext (None) - bad client cert/key or temp-file failure → one-way TLS (CA or system trust) - Custom CA is loaded from normalized in-memory PEM bytes (``cadata``) when - present so path races and UTF-8 BOM/preamble cannot drop the custom trust - store after ``tls_pem_material`` already accepted the CA. + Custom CA is loaded from normalized in-memory PEM bytes (``cadata``) when the + bundle is plain ``CERTIFICATE`` only, so path races and UTF-8 BOM/preamble + cannot drop the custom trust store after ``tls_pem_material`` already accepted + the CA. Bundles that include ``TRUSTED CERTIFICATE`` (``openssl x509 -trustout``) + always use ``cafile``: Python's ``cadata`` loader can silently skip trusted + blocks while still succeeding on ordinary certificates in a mixed bundle. """ from skywalking import config @@ -722,32 +752,38 @@ def ssl_context_for_collector() -> Optional[ssl.SSLContext]: def _system_trust_context() -> ssl.SSLContext: return ssl.create_default_context() + def _load_ca_via_cafile() -> Tuple[Optional[ssl.SSLContext], Optional[BaseException]]: + """Load custom CA through cafile (snapshot, else configured path).""" + try: + cafile = _ca_verify_temp_file(root_certificates) + return ssl.create_default_context(cafile=cafile), None + except OSError as temp_exc: + ca_verify = _configured_ca_path_for_verify() + if ca_verify is not None: + try: + return ssl.create_default_context(cafile=ca_verify), None + except OSError as path_exc: + return None, path_exc + return None, temp_exc + ctx: Optional[ssl.SSLContext] if root_certificates is not None: load_exc: Optional[BaseException] = None - try: - # Normalized ASCII PEM from _load_trusted_ca (BOM/preamble stripped). - # Plain CERTIFICATE works via cadata; TRUSTED CERTIFICATE often needs cafile. - ctx = ssl.create_default_context( - cadata=root_certificates.decode('ascii'), - ) - except (OSError, ValueError) as exc: - load_exc = exc - ctx = None + ctx = None + # TRUSTED CERTIFICATE must not go through cadata: mixed bundles can + # load only the ordinary CERTIFICATE entries and never raise. + if _ca_pem_has_trusted_certificate(root_certificates): + ctx, load_exc = _load_ca_via_cafile() + else: try: - cafile = _ca_verify_temp_file(root_certificates) - ctx = ssl.create_default_context(cafile=cafile) - load_exc = None - except OSError as temp_exc: - load_exc = temp_exc - ca_path = ssl_file_path(config.agent_ssl_trusted_ca_path) - if ca_path is not None: - try: - ctx = ssl.create_default_context(cafile=str(ca_path)) - load_exc = None - except OSError as path_exc: - load_exc = path_exc - ctx = None + # Normalized ASCII PEM from _load_trusted_ca (BOM/preamble stripped). + ctx = ssl.create_default_context( + cadata=root_certificates.decode('ascii'), + ) + except (OSError, ValueError) as exc: + load_exc = exc + ctx = None + ctx, load_exc = _load_ca_via_cafile() if load_exc is not None: # ssl.SSLError subclasses OSError; ValueError: non-ASCII PEM bytes. if config.agent_force_tls: diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py index e3f515214..4f652f21a 100644 --- a/tests/unit/test_tls.py +++ b/tests/unit/test_tls.py @@ -576,8 +576,6 @@ def test_requests_ca_temp_failure_falls_back_to_configured_path(self): with tempfile.TemporaryDirectory() as tmp: ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) config.agent_ssl_trusted_ca_path = str(ca) - resolved = tls_mod.ssl_file_path(str(ca)) - self.assertIsNotNone(resolved) def boom(*_a, **_k): raise OSError(30, 'Read-only file system') @@ -586,10 +584,53 @@ def boom(*_a, **_k): self.assertLogs('skywalking', level='WARNING') as logs: verify, cert = requests_tls_settings() self.assertIsInstance(verify, str) - self.assertTrue(Path(verify).samefile(resolved)) + # Unresolved configured path (absolute), not a deleted resolve() target. + self.assertEqual(Path(verify), Path(ca).expanduser().absolute()) + self.assertTrue(Path(verify).is_file()) self.assertIsNone(cert) self.assertTrue(any('configured CA path' in line for line in logs.output)) + def test_requests_ca_temp_failure_keeps_symlink_across_k8s_rotation(self): + """No-temp fallback + K8s secret rotation must keep the configured symlink.""" + import shutil + + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for version in ('v1', 'v2'): + (root / version).mkdir() + (root / version / 'ca.crt').write_bytes(_TEST_CA_CERT) + try: + (root / '..data').symlink_to('v1') + ca = root / 'ca.crt' + ca.symlink_to(Path('..data') / 'ca.crt') + except OSError: + self.skipTest('symlinks not available') + if tls_mod.ssl_file_path(str(ca)) is None: + self.skipTest('symlink CA path not readable as regular file') + config.agent_ssl_trusted_ca_path = str(ca) + + def boom(*_a, **_k): + raise OSError(30, 'Read-only file system') + + with patch('tempfile.mkstemp', side_effect=boom), \ + self.assertLogs('skywalking', level='WARNING'): + verify, _cert = requests_tls_settings() + self.assertIsInstance(verify, str) + stored = Path(verify) + # Must be the configured symlink, not the resolved v1/ target. + self.assertEqual(stored, ca.expanduser().absolute()) + self.assertNotEqual(stored.parent.name, 'v1') + + try: + (root / '..data-next').symlink_to('v2') + os.replace(root / '..data-next', root / '..data') + shutil.rmtree(root / 'v1') + except OSError: + self.skipTest('symlink rotation not available') + self.assertEqual(ca.read_bytes(), _TEST_CA_CERT) + self.assertTrue(stored.is_file()) + self.assertEqual(stored.read_bytes(), _TEST_CA_CERT) + def test_trusted_certificate_pem_enables_sync_http_tls(self): import shutil import subprocess @@ -630,6 +671,89 @@ def test_trusted_certificate_pem_enables_sync_http_tls(self): self.assertIn(b'BEGIN TRUSTED CERTIFICATE', Path(verify).read_bytes()) self.assertIsNone(cert) + def test_aio_mixed_trusted_and_certificate_bundle_uses_cafile(self): + """cadata must not silently drop TRUSTED blocks from a mixed CA bundle.""" + import shutil + import subprocess + + if shutil.which('openssl') is None: + self.skipTest('openssl not available') + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + ca = root / 'ca.pem' + cert = root / 'server.pem' + trusted = root / 'trusted.pem' + bundle = root / 'bundle.pem' + ca.write_bytes(_TEST_CA_CERT) + cert.write_bytes(_TEST_CLIENT_CERT) + try: + subprocess.run( + [ + 'openssl', 'x509', '-in', str(cert), '-addtrust', 'serverAuth', + '-trustout', '-out', str(trusted), + ], + check=True, + capture_output=True, + ) + except (OSError, subprocess.CalledProcessError) as exc: + self.skipTest(f'openssl trustout failed: {exc}') + bundle.write_bytes(trusted.read_bytes() + ca.read_bytes()) + direct = ssl.create_default_context(cafile=str(bundle)) + self.assertGreaterEqual(direct.cert_store_stats()['x509'], 2) + + config.agent_ssl_trusted_ca_path = str(bundle) + material = tls_pem_material() + self.assertIsNotNone(material) + roots = material[0] + self.assertIn(b'BEGIN TRUSTED CERTIFICATE', roots) + self.assertIn(b'BEGIN CERTIFICATE', roots) + self.assertTrue(tls_mod._ca_pem_has_trusted_certificate(roots)) + + # Prove cadata alone would under-load the mixed bundle. + cadata_only = ssl.create_default_context(cadata=roots.decode('ascii')) + self.assertLess( + cadata_only.cert_store_stats()['x509'], + direct.cert_store_stats()['x509'], + ) + + ctx = ssl_context_for_collector() + self.assertIsInstance(ctx, ssl.SSLContext) + self.assertEqual( + ctx.cert_store_stats()['x509'], + direct.cert_store_stats()['x509'], + ) + + def test_ssl_context_skips_cadata_when_trusted_label_present(self): + """Control-flow guard: TRUSTED marker forces cafile (no openssl required).""" + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + mixed = ( + b'-----BEGIN TRUSTED CERTIFICATE-----\n' + b'MIIB\n' + b'-----END TRUSTED CERTIFICATE-----\n' + ) + _TEST_CA_CERT + self.assertTrue(tls_mod._ca_pem_has_trusted_certificate(mixed)) + calls = [] + real_cdc = ssl.create_default_context + + def spy(*_a, **kwargs): + calls.append(dict(kwargs)) + if 'cadata' in kwargs: + raise AssertionError( + 'cadata must not be used when TRUSTED CERTIFICATE is present' + ) + # Load a known-good CA file regardless of the snapshot path. + return real_cdc(cafile=str(ca)) + + with patch.object(tls_mod, 'tls_pem_material', return_value=(mixed, None, None)), \ + patch.object(tls_mod, '_ca_verify_temp_file', return_value=str(ca)), \ + patch('ssl.create_default_context', side_effect=spy): + ctx = ssl_context_for_collector() + self.assertIsInstance(ctx, ssl.SSLContext) + self.assertTrue(any('cafile' in c for c in calls)) + self.assertFalse(any('cadata' in c for c in calls)) + def test_aio_context_accepts_bom_and_utf8_preamble_ca(self): with tempfile.TemporaryDirectory() as tmp: ca = Path(tmp) / 'ca.crt' From ca6d3be342c233c3cb69bff492473a0bbd056976 Mon Sep 17 00:00:00 2001 From: songzhendong12315 Date: Wed, 16 Sep 2026 09:43:37 +0800 Subject: [PATCH 5/5] fix: polish mTLS temp cleanup and shared HTTP TLS material (P3) Share one PEM material load for HTTP scheme and session/context, prefer configured CA on requests TLS fallback, drop orphaned cert temps and unused _extract_pem_blocks. CA snapshot hot-reload left intentional. --- skywalking/client/http.py | 18 +++--- skywalking/client/http_aio.py | 23 ++++--- skywalking/utils/tls.py | 111 +++++++++++++++++++++------------- tests/unit/test_tls.py | 55 +++++++++++++++++ 4 files changed, 149 insertions(+), 58 deletions(-) diff --git a/skywalking/client/http.py b/skywalking/client/http.py index c529fe96a..d79993464 100644 --- a/skywalking/client/http.py +++ b/skywalking/client/http.py @@ -21,7 +21,7 @@ from skywalking import config from skywalking.client import ServiceManagementClient, TraceSegmentReportService, LogDataReportService from skywalking.loggings import logger, logger_debug_enabled -from skywalking.utils.tls import collector_http_scheme, configure_requests_session +from skywalking.utils.tls import collector_http_scheme, configure_requests_session, safe_tls_pem_material class HttpServiceManagementClient(ServiceManagementClient): @@ -29,11 +29,13 @@ def __init__(self): super().__init__() self.instance_properties = self.get_instance_properties() - proto = collector_http_scheme() + # One material load shared by scheme + session (avoid scheme/settings TOCTOU). + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_instance_props = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/reportProperties" self.url_heart_beat = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/keepAlive" self.session = requests.Session() - configure_requests_session(self.session) + configure_requests_session(self.session, material) def send_instance_props(self): res = self.session.post(self.url_instance_props, json={ @@ -63,10 +65,11 @@ def send_heart_beat(self): class HttpTraceSegmentReportService(TraceSegmentReportService): def __init__(self): - proto = collector_http_scheme() + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/segment" self.session = requests.Session() - configure_requests_session(self.session) + configure_requests_session(self.session, material) def report(self, generator): for segment in generator: @@ -116,10 +119,11 @@ def report(self, generator): class HttpLogDataReportService(LogDataReportService): def __init__(self): - proto = collector_http_scheme() + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/logs" self.session = requests.Session() - configure_requests_session(self.session) + configure_requests_session(self.session, material) def report(self, generator): log_batch = [json.loads(json_format.MessageToJson(log_data)) for log_data in generator] diff --git a/skywalking/client/http_aio.py b/skywalking/client/http_aio.py index 038a4e4b8..248797ae9 100644 --- a/skywalking/client/http_aio.py +++ b/skywalking/client/http_aio.py @@ -21,11 +21,12 @@ from skywalking import config from skywalking.client import ServiceManagementClientAsync, TraceSegmentReportServiceAsync, LogDataReportServiceAsync from skywalking.loggings import logger, logger_debug_enabled -from skywalking.utils.tls import collector_http_scheme, ssl_context_for_collector +from skywalking.utils import tls as tls_mod +from skywalking.utils.tls import collector_http_scheme, safe_tls_pem_material, ssl_context_for_collector -def _aiohttp_session(): - ssl_ctx = ssl_context_for_collector() +def _aiohttp_session(material=tls_mod._MATERIAL_UNSET): + ssl_ctx = ssl_context_for_collector(material) if ssl_ctx is None: return aiohttp.ClientSession() return aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_ctx)) @@ -36,11 +37,13 @@ def __init__(self): super().__init__() self.instance_properties = self.get_instance_properties() - proto = collector_http_scheme() + # One material load shared by scheme + SSLContext (avoid scheme/settings TOCTOU). + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_instance_props = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/reportProperties" self.url_heart_beat = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/management/keepAlive" # self.client = httpx.AsyncClient() - self.client = _aiohttp_session() + self.client = _aiohttp_session(material) async def send_instance_props(self): @@ -73,10 +76,11 @@ async def send_heart_beat(self): class HttpTraceSegmentReportServiceAsync(TraceSegmentReportServiceAsync): def __init__(self): - proto = collector_http_scheme() + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/segment" # self.client = httpx.AsyncClient() - self.client = _aiohttp_session() + self.client = _aiohttp_session(material) async def report(self, generator): async for segment in generator: @@ -127,10 +131,11 @@ async def report(self, generator): class HttpLogDataReportServiceAsync(LogDataReportServiceAsync): def __init__(self): - proto = collector_http_scheme() + material = safe_tls_pem_material() + proto = collector_http_scheme(material) self.url_report = f"{proto}{config.agent_collector_backend_services.rstrip('/')}/v3/logs" # self.client = httpx.AsyncClient() - self.client = _aiohttp_session() + self.client = _aiohttp_session(material) async def report(self, generator): log_batch = [json.loads(json_format.MessageToJson(log_data)) async for log_data in generator] diff --git a/skywalking/utils/tls.py b/skywalking/utils/tls.py index 15d459f1d..779ea71aa 100644 --- a/skywalking/utils/tls.py +++ b/skywalking/utils/tls.py @@ -241,38 +241,13 @@ def normalize_private_key_pem(key_pem: bytes) -> bytes: return pem.encode('ascii') -def _extract_pem_blocks(data: bytes, label: str) -> bytes: - """ - Return ASCII PEM containing only ``BEGIN/END {label}`` blocks. - - Strips UTF-8 BOM and ignores preamble/comments outside PEM delimiters so - OpenSSL-accepted files (BOM, UTF-8 comments) stay usable for ``cadata`` and - temp-file verify paths that require clean ASCII PEM. - """ - text = data.decode('utf-8', errors='ignore').lstrip('\ufeff') - header = f'-----BEGIN {label}-----' - footer = f'-----END {label}-----' - blocks: List[str] = [] - pos = 0 - while True: - start = text.find(header, pos) - if start < 0: - break - end = text.find(footer, start) - if end < 0: - break - end += len(footer) - blocks.append(text[start:end].strip() + '\n') - pos = end - if not blocks: - raise ValueError(f'No {label} PEM block found') - return ''.join(blocks).encode('ascii') - - # OpenSSL ``openssl x509 -trustout`` emits TRUSTED CERTIFICATE (with trust # auxiliary). Keep that label — do not relabel as CERTIFICATE. _CA_PEM_LABELS = ('TRUSTED CERTIFICATE', 'CERTIFICATE') +# Sentinel: optional ``material`` args mean "load via tls_pem_material()". +_MATERIAL_UNSET = object() + def _extract_ca_pem_blocks(data: bytes) -> bytes: """ @@ -432,6 +407,20 @@ def _pem_bytes_to_temp_file(data: bytes, suffix: str) -> str: return path +def _discard_mtls_temp_path(path: Optional[str]) -> None: + """Unlink a temp PEM and drop it from the process cleanup list.""" + if not path: + return + try: + os.unlink(path) + except OSError: + pass + try: + _mtls_temp_files.remove(path) + except ValueError: + pass + + def _mtls_cert_key_files( certificate_chain: bytes, private_key: bytes, @@ -440,6 +429,8 @@ def _mtls_cert_key_files( Write normalized PEM bytes to temp files (cached per material). Returns None when temp files cannot be created (caller stays one-way TLS). + If the key temp write fails after the cert was written, the orphaned cert + file is unlinked immediately (not left until process exit). """ global _mtls_file_cache_key, _mtls_file_cache @@ -447,12 +438,13 @@ def _mtls_cert_key_files( if _mtls_file_cache is not None and _mtls_file_cache_key == cache_key: return _mtls_file_cache + cert_path = None try: - paths = ( - _pem_bytes_to_temp_file(certificate_chain, '.crt'), - _pem_bytes_to_temp_file(private_key, '.pem'), - ) + cert_path = _pem_bytes_to_temp_file(certificate_chain, '.crt') + key_path = _pem_bytes_to_temp_file(private_key, '.pem') + paths = (cert_path, key_path) except OSError as exc: + _discard_mtls_temp_path(cert_path) _warn_once( f'mtls-temp:{exc}', 'Failed to write mTLS cert/key temp files (%s); staying on one-way TLS.', @@ -519,6 +511,27 @@ def tls_pem_material() -> Optional[Tuple[Optional[bytes], Optional[bytes], Optio return root_certificates, private_key, certificate_chain +def safe_tls_pem_material() -> Optional[Tuple[Optional[bytes], Optional[bytes], Optional[bytes]]]: + """ + ``tls_pem_material()`` that never raises into HTTP reporter ``__init__``. + + On unexpected failure: FORCE_TLS → system-trust tuple; else plaintext None. + """ + try: + return tls_pem_material() + except Exception as exc: # noqa: BLE001 - never fail host process start + from skywalking import config + + _warn_once( + f'tls-material:{exc}', + 'Failed to load collector TLS material (%s); falling back.', + exc, + ) + if config.agent_force_tls: + return None, None, None + return None + + def grpc_ssl_credentials(): """ grpc.ChannelCredentials for TLS/mTLS, or None for plaintext. @@ -585,15 +598,20 @@ def grpc_ssl_credentials(): return None -def collector_http_scheme() -> str: +def collector_http_scheme(material=_MATERIAL_UNSET) -> str: """ ``https://`` when TLS material is enabled, else ``http://``. + Pass ``material`` from a shared ``tls_pem_material()`` call so HTTP URL + scheme and session TLS settings cannot disagree across a TOCTOU window. + Never raises into reporter ``__init__`` (HTTP clients call this before other TLS helpers that may wrap failures). """ try: - return 'https://' if tls_pem_material() is not None else 'http://' + if material is _MATERIAL_UNSET: + material = tls_pem_material() + return 'https://' if material is not None else 'http://' except Exception as exc: # noqa: BLE001 - never fail host process start from skywalking import config @@ -648,13 +666,16 @@ def _ca_pem_has_trusted_certificate(root_certificates: bytes) -> bool: return b'BEGIN TRUSTED CERTIFICATE' in root_certificates -def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: +def requests_tls_settings(material=_MATERIAL_UNSET) -> Tuple[object, Optional[Tuple[str, str]]]: """ (verify, cert) for requests.Session. verify is True (system CAs), a CA file path, or unused for plaintext callers. cert is (cert_path, key_path) when mTLS files are present. + Pass ``material`` from a shared ``tls_pem_material()`` call (with + ``collector_http_scheme``) so scheme and verify cannot disagree. + Keeps the same enable/disable decision as grpc_ssl_credentials / tls_pem_material so an unreadable or oversized CA cannot leave HTTP on https:// with a bad verify path. @@ -668,7 +689,8 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: Client-cert temp-file failures drop mTLS only (one-way TLS), with a warning. """ - material = tls_pem_material() + if material is _MATERIAL_UNSET: + material = tls_pem_material() if material is None: return True, None @@ -704,9 +726,9 @@ def requests_tls_settings() -> Tuple[object, Optional[Tuple[str, str]]]: return verify, pair -def configure_requests_session(session) -> None: +def configure_requests_session(session, material=_MATERIAL_UNSET) -> None: try: - verify, cert = requests_tls_settings() + verify, cert = requests_tls_settings(material) except Exception as exc: # noqa: BLE001 - never fail host process start from skywalking import config @@ -716,8 +738,9 @@ def configure_requests_session(session) -> None: exc, ) if config.agent_force_tls or ssl_file_path(config.agent_ssl_trusted_ca_path): - # Prefer system-trust https over aborting agent start. - session.verify = True + # Prefer the configured private CA over discarding it for system trust. + ca_verify = _configured_ca_path_for_verify() + session.verify = ca_verify if ca_verify is not None else True session.cert = None return session.verify = verify @@ -725,10 +748,13 @@ def configure_requests_session(session) -> None: session.cert = cert -def ssl_context_for_collector() -> Optional[ssl.SSLContext]: +def ssl_context_for_collector(material=_MATERIAL_UNSET) -> Optional[ssl.SSLContext]: """ stdlib SSLContext for aiohttp, or None when the collector stays plaintext. + Pass ``material`` from a shared ``tls_pem_material()`` call (with + ``collector_http_scheme``) so scheme and SSLContext cannot disagree. + Degrade on SSLError / OSError (never raise into agent bootstrap): - bad / unparsable custom CA + FORCE_TLS → process trust store, no client cert - bad / unparsable custom CA without FORCE_TLS → plaintext (None) @@ -743,7 +769,8 @@ def ssl_context_for_collector() -> Optional[ssl.SSLContext]: """ from skywalking import config - material = tls_pem_material() + if material is _MATERIAL_UNSET: + material = tls_pem_material() if material is None: return None diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py index 4f652f21a..d6623d51b 100644 --- a/tests/unit/test_tls.py +++ b/tests/unit/test_tls.py @@ -889,6 +889,61 @@ def test_fork_child_exit_does_not_delete_parent_mtls_temps(self): self.assertTrue(os.path.exists(key_file), 'parent key temp deleted by child') self.assertEqual(tls_mod._mtls_file_cache, pair) + def test_mtls_partial_temp_failure_does_not_orphan_cert(self): + # Second mkstemp (key) fails after cert was written → cert must be unlinked. + calls = {'n': 0} + real_mkstemp = tempfile.mkstemp + + def mkstemp_fail_on_key(*args, **kwargs): + calls['n'] += 1 + if calls['n'] == 1: + return real_mkstemp(*args, **kwargs) + raise OSError(30, 'Read-only file system') + + with patch('tempfile.mkstemp', side_effect=mkstemp_fail_on_key), \ + self.assertLogs('skywalking', level='WARNING') as logs: + pair = tls_mod._mtls_cert_key_files(_TEST_CLIENT_CERT, _TEST_CLIENT_KEY_PKCS1) + self.assertIsNone(pair) + self.assertEqual(tls_mod._mtls_temp_files, []) + self.assertTrue(any('temp files' in line for line in logs.output)) + + def test_configure_requests_session_fallback_prefers_configured_ca(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + session = MagicMock() + with patch('skywalking.utils.tls.requests_tls_settings', side_effect=RuntimeError('boom')), \ + self.assertLogs('skywalking', level='WARNING'): + tls_mod.configure_requests_session(session) + self.assertEqual(session.verify, str(Path(ca).expanduser().absolute())) + self.assertIsNone(session.cert) + + def test_http_scheme_and_settings_share_one_material(self): + with tempfile.TemporaryDirectory() as tmp: + ca = self._write_pem(tmp, 'ca.crt', _TEST_CA_CERT) + config.agent_ssl_trusted_ca_path = str(ca) + material = tls_pem_material() + self.assertIsNotNone(material) + + calls = {'n': 0} + real = tls_mod.tls_pem_material + + def counting(): + calls['n'] += 1 + return real() + + with patch.object(tls_mod, 'tls_pem_material', side_effect=counting): + # Shared material: helpers must not reload. + self.assertEqual(collector_http_scheme(material), 'https://') + verify, cert = requests_tls_settings(material) + self.assertIsInstance(verify, str) + self.assertIsNone(cert) + self.assertIsInstance(ssl_context_for_collector(material), ssl.SSLContext) + self.assertEqual(calls['n'], 0) + + def test_extract_pem_blocks_removed(self): + self.assertFalse(hasattr(tls_mod, '_extract_pem_blocks')) + if __name__ == '__main__': unittest.main()