From 9a3f608d118944b956a4de732e982f83cee7e0a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ren=C3=A9=20Treffer?= Date: Wed, 26 Aug 2026 13:49:45 +0200 Subject: [PATCH] Add resctrlfs package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parse the monitoring data of the resctrl filesystem, the kernel interface to Intel RDT, AMD PQoS and ARM MPAM. All vendors share the same driver and file layout, so one parser serves all vendors. MonData returns the per domain counters of the root control group (llc_occupancy, mbm_total_bytes, mbm_local_bytes). A counter is nil when the hardware lacks the feature or reports "Unavailable". L3MonInfo exposes the monitoring capabilities from info/L3_MON. This patch only adds read support for monitoring data. Signed-off-by: René Treffer --- internal/fs/fs.go | 3 + resctrlfs/fs.go | 63 ++++++++ resctrlfs/fs_test.go | 36 +++++ resctrlfs/info.go | 66 +++++++++ resctrlfs/info_test.go | 63 ++++++++ resctrlfs/mon_data.go | 134 ++++++++++++++++++ resctrlfs/mon_data_test.go | 106 ++++++++++++++ .../sys/fs/resctrl/info/L3_MON/mon_features | 3 + .../sys/fs/resctrl/info/L3_MON/num_rmids | 1 + .../resctrl/mon_data/mon_L3_00/llc_occupancy | 1 + .../mon_data/mon_L3_00/mbm_local_bytes | 1 + .../mon_data/mon_L3_00/mbm_total_bytes | 1 + .../resctrl/mon_data/mon_L3_01/llc_occupancy | 1 + .../mon_data/mon_L3_01/mbm_local_bytes | 1 + .../mon_data/mon_L3_01/mbm_total_bytes | 1 + 15 files changed, 481 insertions(+) create mode 100644 resctrlfs/fs.go create mode 100644 resctrlfs/fs_test.go create mode 100644 resctrlfs/info.go create mode 100644 resctrlfs/info_test.go create mode 100644 resctrlfs/mon_data.go create mode 100644 resctrlfs/mon_data_test.go create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/mon_features create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/num_rmids create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/llc_occupancy create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_local_bytes create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_total_bytes create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/llc_occupancy create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_local_bytes create mode 100644 resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_total_bytes diff --git a/internal/fs/fs.go b/internal/fs/fs.go index e7ccad66..bdd8010a 100644 --- a/internal/fs/fs.go +++ b/internal/fs/fs.go @@ -31,6 +31,9 @@ const ( // DefaultSelinuxMountPoint is the common mount point of the selinuxfs. DefaultSelinuxMountPoint = "/sys/fs/selinux" + + // DefaultResctrlMountPoint is the common mount point of the resctrl filesystem. + DefaultResctrlMountPoint = "/sys/fs/resctrl" ) // FS represents a pseudo-filesystem, normally /proc or /sys, which provides an diff --git a/resctrlfs/fs.go b/resctrlfs/fs.go new file mode 100644 index 00000000..1417566b --- /dev/null +++ b/resctrlfs/fs.go @@ -0,0 +1,63 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +// Package resctrlfs provides access to the monitoring data of the resctrl +// filesystem. +// +// resctrl is the kernel interface to the cache and memory bandwidth resource +// control of the CPU: Intel calls it RDT (Resource Director Technology), AMD +// calls it PQoS (Platform Quality of Service), Arm calls it MPAM (Memory System +// Resource Partitioning and Monitoring). All vendors are served by the same +// kernel driver, so the file layout is identical. +// +// The filesystem is not mounted by default. Mount it with: +// +// mount -t resctrl resctrl /sys/fs/resctrl +// +// This package only reads monitoring data. It does not configure allocation. +// +// The filesystem is documented in the kernel tree: +// - https://docs.kernel.org/filesystems/resctrl.html +// - https://docs.kernel.org/arch/arm64/mpam.html +package resctrlfs + +import ( + "github.com/prometheus/procfs/internal/fs" +) + +// FS represents the pseudo-filesystem resctrl, which provides an interface to +// the cache and memory bandwidth monitoring of the CPU. +type FS struct { + resctrl fs.FS +} + +// DefaultMountPoint is the common mount point of the resctrl filesystem. +const DefaultMountPoint = fs.DefaultResctrlMountPoint + +// NewDefaultFS returns a new FS mounted under the default mountPoint. It will error +// if the mount point can't be read. +func NewDefaultFS() (FS, error) { + return NewFS(DefaultMountPoint) +} + +// NewFS returns a new FS mounted under the given mountPoint. It will error +// if the mount point can't be read. +func NewFS(mountPoint string) (FS, error) { + fs, err := fs.NewFS(mountPoint) + if err != nil { + return FS{}, err + } + return FS{fs}, nil +} diff --git a/resctrlfs/fs_test.go b/resctrlfs/fs_test.go new file mode 100644 index 00000000..defbf85d --- /dev/null +++ b/resctrlfs/fs_test.go @@ -0,0 +1,36 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package resctrlfs + +import "testing" + +const ( + resctrlTestFixtures = "testdata/fixtures" + DefaultMountPoint +) + +func TestNewFS(t *testing.T) { + if _, err := NewFS("foobar"); err == nil { + t.Error("want NewFS to fail for non-existing mount point") + } + + if _, err := NewFS("fs.go"); err == nil { + t.Error("want NewFS to fail if mount point is not a directory") + } + + if _, err := NewFS(resctrlTestFixtures); err != nil { + t.Error("want NewFS to succeed if mount point exists") + } +} diff --git a/resctrlfs/info.go b/resctrlfs/info.go new file mode 100644 index 00000000..39e61645 --- /dev/null +++ b/resctrlfs/info.go @@ -0,0 +1,66 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package resctrlfs + +import ( + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +const l3MonInfoPath = "info/L3_MON" + +// L3MonInfo describes the L3 monitoring capabilities from info/L3_MON. +type L3MonInfo struct { + // NumRMIDs is the number of resource monitoring ids the hardware offers. + // One RMID is used per monitoring group, so this is the upper limit of + // groups that can be monitored at the same time. + NumRMIDs uint64 + // MonFeatures lists the counters the hardware supports, in the order the + // kernel reports them, e.g. "llc_occupancy", "mbm_total_bytes", + // "mbm_local_bytes". + MonFeatures []string +} + +// L3MonInfo returns the L3 monitoring capabilities of the CPU. +// +// It errors if info/L3_MON is missing, which means the CPU or the kernel does +// not support L3 monitoring. The error wraps the underlying os error, so it can +// be tested with os.IsNotExist. +func (fs FS) L3MonInfo() (L3MonInfo, error) { + var info L3MonInfo + + numRMIDs, err := util.ReadUintFromFile(fs.resctrl.Path(l3MonInfoPath, "num_rmids")) + if err != nil { + return info, err + } + info.NumRMIDs = numRMIDs + + data, err := util.ReadFileNoStat(fs.resctrl.Path(l3MonInfoPath, "mon_features")) + if err != nil { + return info, err + } + + for _, line := range strings.Split(string(data), "\n") { + feature := strings.TrimSpace(line) + if feature == "" { + continue + } + info.MonFeatures = append(info.MonFeatures, feature) + } + + return info, nil +} diff --git a/resctrlfs/info_test.go b/resctrlfs/info_test.go new file mode 100644 index 00000000..83728904 --- /dev/null +++ b/resctrlfs/info_test.go @@ -0,0 +1,63 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package resctrlfs + +import ( + "os" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestL3MonInfo(t *testing.T) { + fs, err := NewFS(resctrlTestFixtures) + if err != nil { + t.Fatalf("failed to open filesystem: %v", err) + } + + got, err := fs.L3MonInfo() + if err != nil { + t.Fatalf("failed to read info/L3_MON: %v", err) + } + + want := L3MonInfo{ + NumRMIDs: 256, + MonFeatures: []string{ + "llc_occupancy", + "mbm_total_bytes", + "mbm_local_bytes", + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("unexpected L3 monitoring info (-want +got):\n%s", diff) + } +} + +func TestL3MonInfoWithoutMonitoring(t *testing.T) { + fs, err := NewFS(t.TempDir()) + if err != nil { + t.Fatalf("failed to open filesystem: %v", err) + } + + _, err = fs.L3MonInfo() + if err == nil { + t.Fatal("want L3MonInfo to fail if info/L3_MON is missing") + } + if !os.IsNotExist(err) { + t.Errorf("want a not exist error, got %v", err) + } +} diff --git a/resctrlfs/mon_data.go b/resctrlfs/mon_data.go new file mode 100644 index 00000000..785bc7fa --- /dev/null +++ b/resctrlfs/mon_data.go @@ -0,0 +1,134 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package resctrlfs + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +const monDataPath = "mon_data" + +// MonData holds the monitoring counters of one domain of one resource, +// read from mon_data/mon__. +type MonData struct { + Resource string // e.g. "L3" + ID string // domain id with leading zeros stripped, e.g. "0" + // Counters are nil when the file is absent (feature not supported) + // or reads "Unavailable" (no sample available right now). For example + // Arm MPAM never offers MBMLocalBytes, because the hardware can not + // tell local from remote traffic. + LLCOccupancy *uint64 + MBMTotalBytes *uint64 + MBMLocalBytes *uint64 +} + +// MonData returns the monitoring counters of the root control group, one entry +// per monitoring domain. A domain is a socket, or a CCX on AMD. +// +// It errors only if the mon_data directory itself can't be read. A domain +// counter that is missing, unreadable or unavailable stays nil, because the +// hardware may support only some of the features and samples are not always +// available. A mount without any monitoring domain returns an empty slice. +func (fs FS) MonData() ([]MonData, error) { + path := fs.resctrl.Path(monDataPath) + + dirs, err := os.ReadDir(path) + if err != nil { + return nil, fmt.Errorf("failed to list monitoring domains at %q: %w", path, err) + } + + // os.ReadDir sorts by filename, so the result is ordered by domain. + mons := make([]MonData, 0, len(dirs)) + for _, d := range dirs { + resource, id, ok := parseMonDirName(d.Name()) + if !ok { + continue + } + + domain := fs.resctrl.Path(monDataPath, d.Name()) + mons = append(mons, MonData{ + Resource: resource, + ID: id, + LLCOccupancy: readCounter(domain, "llc_occupancy"), + // The mbm counters are hardware registers of limited width. They + // are free running and wrap around; this library reports the raw + // value and leaves the wrap handling to the caller. + MBMTotalBytes: readCounter(domain, "mbm_total_bytes"), + MBMLocalBytes: readCounter(domain, "mbm_local_bytes"), + }) + } + + return mons, nil +} + +// readCounter reads a single counter file of a monitoring domain. It returns +// nil if the file does not exist, can not be read, or does not hold a number. +// The kernel writes the literal "Unavailable" when the hardware can not +// deliver a sample, which is a normal and transient state. +func readCounter(domain, name string) *uint64 { + value, err := util.ReadUintFromFile(filepath.Join(domain, name)) + if err != nil { + return nil + } + return &value +} + +// parseMonDirName splits a mon_data directory name into resource and domain id. +// The name is "mon__", for example "mon_L3_00". The resource name +// may itself contain underscores, so the split is at the last one. The id is +// zero padded by the kernel; the padding is stripped, so "00" becomes "0". +// The second return value is false for a name that is not a monitoring domain. +func parseMonDirName(name string) (string, string, bool) { + rest, ok := strings.CutPrefix(name, "mon_") + if !ok { + return "", "", false + } + + i := strings.LastIndex(rest, "_") + if i <= 0 { + return "", "", false + } + + resource, id := rest[:i], rest[i+1:] + if !isDigits(id) { + return "", "", false + } + + id = strings.TrimLeft(id, "0") + if id == "" { + id = "0" + } + + return resource, id, true +} + +// isDigits reports whether s is a non-empty run of decimal digits. +func isDigits(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} diff --git a/resctrlfs/mon_data_test.go b/resctrlfs/mon_data_test.go new file mode 100644 index 00000000..13ff5798 --- /dev/null +++ b/resctrlfs/mon_data_test.go @@ -0,0 +1,106 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build linux + +package resctrlfs + +import ( + "testing" + + "github.com/google/go-cmp/cmp" +) + +func uint64Ptr(v uint64) *uint64 { + return &v +} + +func TestMonData(t *testing.T) { + fs, err := NewFS(resctrlTestFixtures) + if err != nil { + t.Fatalf("failed to open filesystem: %v", err) + } + + got, err := fs.MonData() + if err != nil { + t.Fatalf("failed to read mon_data: %v", err) + } + + want := []MonData{ + { + Resource: "L3", + ID: "0", + LLCOccupancy: uint64Ptr(44040192), + MBMTotalBytes: uint64Ptr(315294410752), + MBMLocalBytes: uint64Ptr(210196273664), + }, + { + Resource: "L3", + ID: "1", + LLCOccupancy: uint64Ptr(8388608), + MBMTotalBytes: uint64Ptr(105098136832), + // mbm_local_bytes reads "Unavailable" in the fixture. + MBMLocalBytes: nil, + }, + } + + if diff := cmp.Diff(want, got); diff != "" { + t.Errorf("unexpected mon_data (-want +got):\n%s", diff) + } +} + +func TestMonDataWithoutMonData(t *testing.T) { + fs, err := NewFS(t.TempDir()) + if err != nil { + t.Fatalf("failed to open filesystem: %v", err) + } + + if _, err := fs.MonData(); err == nil { + t.Error("want MonData to fail if the mon_data directory is missing") + } +} + +func TestParseMonDirName(t *testing.T) { + for _, test := range []struct { + name string + wantResource string + wantID string + wantOK bool + }{ + {name: "mon_L3_00", wantResource: "L3", wantID: "0", wantOK: true}, + {name: "mon_L3_01", wantResource: "L3", wantID: "1", wantOK: true}, + {name: "mon_L3_12", wantResource: "L3", wantID: "12", wantOK: true}, + {name: "mon_MB_0", wantResource: "MB", wantID: "0", wantOK: true}, + {name: "mon_L3_MON_3", wantResource: "L3_MON", wantID: "3", wantOK: true}, + {name: "mon_L3"}, + {name: "mon_L3_"}, + {name: "mon__0"}, + {name: "mon_L3_xy"}, + {name: "L3_00"}, + {name: "mon_"}, + {name: ""}, + } { + t.Run(test.name, func(t *testing.T) { + resource, id, ok := parseMonDirName(test.name) + if ok != test.wantOK { + t.Fatalf("want ok %v, got %v", test.wantOK, ok) + } + if resource != test.wantResource { + t.Errorf("want resource %q, got %q", test.wantResource, resource) + } + if id != test.wantID { + t.Errorf("want id %q, got %q", test.wantID, id) + } + }) + } +} diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/mon_features b/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/mon_features new file mode 100644 index 00000000..0c57b8d8 --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/mon_features @@ -0,0 +1,3 @@ +llc_occupancy +mbm_total_bytes +mbm_local_bytes diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/num_rmids b/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/num_rmids new file mode 100644 index 00000000..9183bf03 --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/info/L3_MON/num_rmids @@ -0,0 +1 @@ +256 diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/llc_occupancy b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/llc_occupancy new file mode 100644 index 00000000..317ea811 --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/llc_occupancy @@ -0,0 +1 @@ +44040192 diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_local_bytes b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_local_bytes new file mode 100644 index 00000000..5cb81362 --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_local_bytes @@ -0,0 +1 @@ +210196273664 diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_total_bytes b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_total_bytes new file mode 100644 index 00000000..d91f52eb --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_00/mbm_total_bytes @@ -0,0 +1 @@ +315294410752 diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/llc_occupancy b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/llc_occupancy new file mode 100644 index 00000000..16edb4cc --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/llc_occupancy @@ -0,0 +1 @@ +8388608 diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_local_bytes b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_local_bytes new file mode 100644 index 00000000..ba54161f --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_local_bytes @@ -0,0 +1 @@ +Unavailable diff --git a/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_total_bytes b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_total_bytes new file mode 100644 index 00000000..47cf3d0c --- /dev/null +++ b/resctrlfs/testdata/fixtures/sys/fs/resctrl/mon_data/mon_L3_01/mbm_total_bytes @@ -0,0 +1 @@ +105098136832