diff --git a/LAYOUT.md b/LAYOUT.md index 019296a..8f665c4 100644 --- a/LAYOUT.md +++ b/LAYOUT.md @@ -7,7 +7,7 @@ linux-kernel-internals/ ├── kxray/ # Python: trace, BTF, /proc and dump analysis │ ├── btf/ # BTF reader: types, fields, offsets, holes, type tags │ ├── trace/ # ftrace function_graph, function and trace_event parsers -│ ├── proc/ # /proc and /sys snapshot parsers +│ ├── proc/ # /proc and /sys snapshot parsers, and the ABI stability ledger │ ├── source/ # kernel tree navigation, Kconfig, MAINTAINERS │ ├── models/ # the shared model everything else renders │ ├── replay/ # recorded Tier 1 session playback @@ -75,6 +75,12 @@ Three of the five ride along in BTF as a `type_tag` record. `__iomem` does not, `kxray/models.py` holds `TraceLog` under all three parsers. The banner at the top of a trace says how many events the kernel produced and how many survived, and a trace where those differ has holes in it that nothing in the body of the file admits to, so it is read rather than skipped. The per tracer subclasses add only what is actually different, which is the list of things on the lines. +`kxray/proc/` is five readers for four file shapes, because `/proc` has fewer shapes in it than it has files. `keyed.py` reads `Key: value`, which is `meminfo` and a process status file. `percpu.py` reads a label and one count per CPU, which is `interrupts` and `softirqs`. `maps.py` reads one record per line with positional columns. `pidstat.py` reads the single line file, which is its own shape only because of what a command name is allowed to contain. `version.py` reads one line of free text with two useful things in it. + +`kxray/proc/stability.py` is the reason the package is shaped that way rather than being five loose functions. Every reader returns something carrying a `Promise` saying what the kernel tree says about the file it read, taken from `Documentation/ABI` and citing the file that makes the claim. That matters because of a number: on 7.2.2 there are 685 files under `Documentation/ABI` and six of them describe a path in `/proc`, and not one of those six is a file anybody reads. `meminfo`, `interrupts`, `maps` and `/proc//stat` are all undocumented, which is not the same as unstable and is worth telling a reader before they lean on one. Two paths are worse than undocumented and this project reads both: the closing section of that README names Kconfig, calling out `/proc/config.gz`, and kernel symbols, which is `/proc/kallsyms`, as things that must not under any circumstances be considered stable. + +`kxray/proc/pidstat.py` is a file of its own for one reason, and the reason is a real capture. The kernel prints the command name in brackets and does not escape it, so a process whose executable is called `od) d ma` prints as `37 (od) d ma) R 1 0 ...`, and a whitespace split puts the state two fields to the left of where it belongs and reports a process as being in a state that does not exist. `corpora/proc/tier0/odd-comm-stat.txt` is that line off the pinned kernel. The parser takes the first opening bracket and the last closing bracket, which is what `procps` has done for decades, and keeps what the naive split would have said so that a lesson can show both answers rather than assert that the trap is real. + `kxray/layout.py` is the arithmetic that turns a tree of frames into rectangles. It is in `kxray` for the same reason. A widget and an animation of the same trace call it and get the same answer, so the wide box is in the same place in both. `kxshapes/` is the next step up from that. It is the nine shapes every picture in this book is built out of, held as plain data rather than as drawing: a frame card, a layer band, an object box, a pointer thread, an ops plug, a trace cell, a CPU lane, a context badge and a memory slot. A test asserts there are exactly nine, because a closed set is the point. Each shape works out its own rows, its own labels and its own alt text, and neither renderer is allowed to work any of that out again. It is a package of its own rather than a module inside either renderer, and that is the whole reason it exists. If the arithmetic lived in `kxwidgets` then `kxmanim` would have to redo it, and two renderers doing their own arithmetic are two renderers that can disagree, in the worst possible way, which is that both pictures look fine and one of them is wrong. diff --git a/corpora/BASELINE.toml b/corpora/BASELINE.toml index c1b5c2a..52d0562 100644 --- a/corpora/BASELINE.toml +++ b/corpora/BASELINE.toml @@ -12,10 +12,10 @@ schema = 1 [totals] -artefacts = 24 -lines = 1856 -read = 1480 -skipped = 207 +artefacts = 32 +lines = 1982 +read = 1604 +skipped = 209 unparsed = 0 [[artefact]] @@ -105,6 +105,15 @@ read = 34 skipped = 1 unparsed = 0 +[[artefact]] +path = "corpora/proc/tier0/interrupts.txt" +reader = "proc-percpu" +lines = 8 +found = 7 +read = 7 +skipped = 1 +unparsed = 0 + [[artefact]] path = "corpora/proc/tier0/lockdep-stats-after.txt" reader = "lockdep-stats" @@ -123,6 +132,24 @@ read = 51 skipped = 1 unparsed = 0 +[[artefact]] +path = "corpora/proc/tier0/meminfo.txt" +reader = "proc-keyed" +lines = 47 +found = 47 +read = 47 +skipped = 0 +unparsed = 0 + +[[artefact]] +path = "corpora/proc/tier0/odd-comm-stat.txt" +reader = "proc-pidstat" +lines = 1 +found = 50 +read = 1 +skipped = 0 +unparsed = 0 + [[artefact]] path = "corpora/proc/tier0/ring-overrun.txt" reader = "tracefs-stats" @@ -132,6 +159,51 @@ read = 6 skipped = 2 unparsed = 0 +[[artefact]] +path = "corpora/proc/tier0/self-maps.txt" +reader = "proc-maps" +lines = 7 +found = 7 +read = 7 +skipped = 0 +unparsed = 0 + +[[artefact]] +path = "corpora/proc/tier0/self-stat.txt" +reader = "proc-pidstat" +lines = 1 +found = 50 +read = 1 +skipped = 0 +unparsed = 0 + +[[artefact]] +path = "corpora/proc/tier0/self-status.txt" +reader = "proc-keyed" +lines = 50 +found = 50 +read = 50 +skipped = 0 +unparsed = 0 + +[[artefact]] +path = "corpora/proc/tier0/softirqs.txt" +reader = "proc-percpu" +lines = 11 +found = 10 +read = 10 +skipped = 1 +unparsed = 0 + +[[artefact]] +path = "corpora/proc/tier0/version.txt" +reader = "proc-version" +lines = 1 +found = 3 +read = 1 +skipped = 0 +unparsed = 0 + [[artefact]] path = "corpora/traces/handwritten/page-fault.txt" reader = "function_graph" diff --git a/corpora/proc/tier0/README.md b/corpora/proc/tier0/README.md new file mode 100644 index 0000000..e021880 --- /dev/null +++ b/corpora/proc/tier0/README.md @@ -0,0 +1,89 @@ +# Tier 0 snapshots of /proc + +These are copies of files out of `/proc` and `/sys` on the pinned kernel. A trace is a recording of something happening. These are the opposite: a look at what the kernel was willing to say about itself at one moment, with nothing happening at all. + +Every file here is real, off the 7.2.2 built for 32-bit x86 with one processor, running under v86. Every `.meta.toml` says `evidence = true`, so a lesson may cite one. Each also records the file in `/proc` it is a copy of, under `path`, and the level the kernel tree gives that path, under `stability`. + +## The thing to read before reading any of them + +Almost none of these files are documented. + +The kernel keeps its own record of what it promises in `Documentation/ABI`, four directories deep, one per level, defined in `Documentation/ABI/README`. On 7.2.2 that tree has 685 files in it. Six of them describe a path in `/proc`, and those six are `/proc/i8k`, `/proc/diskstats`, `/proc/pid/smaps_rollup` and the three files under `/proc/*/attr`. Not `meminfo`. Not `interrupts`. Not `/proc//stat`, which is the file behind every process monitor ever written. Not `maps`. + +That is worth sitting with rather than being alarmed by. Those files have had the same shape for many years and will keep it, because changing one would break userspace and that is the rule nobody gets to bend. What is missing is anybody having written down which part of the shape you may lean on. So this project reads them, and `kxray.proc.stability` attaches the answer to every read, and printing one of these objects tells you it is leaning on custom rather than on a promise. + +Two paths this project reads are stronger than undocumented and worse. The closing section of `Documentation/ABI/README` names, as things that "should not under any circumstances be considered stable", Kconfig, calling out `/proc/config.gz` by name, and kernel symbols, saying not to rely on "the presence, absence, location, or type of any kernel symbol". The second of those is `/proc/kallsyms`, which is next door in `../handwritten/` and which `kxray.kallsyms` reads anyway. Counting ops tables by name on a machine in front of you is a fine thing to do. Shipping the same code inside a tool is not, and now the ledger says so. + +## Taking one again + +``` +node kxbox/web/headless.js sh 'cat /proc/meminfo' +``` + +No setup, no tracer, nothing to turn on or put back. These are the cheapest artefacts in the corpus to refresh, and the ones most likely to be different after a kernel bump, which is the point of having them. + +## version.txt + +One line. The release and the build number are worth pulling out and the rest is not: it holds the user and host that built the kernel, then the whole compiler and linker banner, with brackets inside brackets in it, and nothing anywhere promises its shape. + +It is also the cheapest confirmation that the kernel running is the kernel the profile asked for. `PREEMPT` is in there because `kxbox/kernel/pin.toml` asked for `CONFIG_PREEMPT=y`, and if it ever stops being in there, half the claims in the concurrency lessons are about a different machine. + +`kxray.proc.version` turns the release into a tuple of numbers, because string comparison says 6.9 is newer than 6.10 and it is not. + +## meminfo.txt + +All of memory as the kernel accounts for it, on a box given 100 MiB. + +The unit says `kB` and means KiB. `MemTotal` is 102308 kB, and 102308 times 1024 is a shade under 100 MiB while 102308 times 1000 is nowhere near it. The kernel has spelled it that way since the beginning and every tool on the machine agrees with it, so the parser keeps the kernel's spelling and multiplies by 1024. + +The key list is per config. There is no `HugePages_Total` here and there is a `GPUActive`, and a machine built differently prints a different set. Nothing in `kxray.proc` requires a key to be there, and asking for one that is not raises rather than returning zero. + +## interrupts.txt and softirqs.txt + +Read these two together or the pair is wasted. + +`interrupts.txt` counts the hardware asking for attention. There is one column because this box has one CPU. A laptop prints one column per possible CPU, which is not the same as the number online and not the same as the number in the machine, and the header is the only place that number is already worked out. That is why `kxray.proc.percpu` reads the header and refuses to guess. + +The rows underneath the numbered ones are per architecture and per config. Two here, `NMI` and `TLB`. An x86-64 desktop prints around fifteen and an arm64 machine prints a different set again, so there is no list of them anywhere in the code. + +`softirqs.txt` counts the work that answering an interrupt did not do itself. Ten vectors, seven of which have never fired on this idle box. The two that have are `TIMER` and `RCU`, which is exactly the pair that `../../traces/tier0/flat-interrupt.txt` catches in the act: `raise_softirq` inside the hardware handler with interrupts off, then `handle_softirqs` four lines later with interrupts back on. That trace is the gap happening once. These two files are the same gap counted since boot. + +## self-maps.txt + +The whole address space of one process, which was the `cat` that read the file. Seven lines, and four things in them. + +`/bin/busybox` appears twice, once `r-xp` and once `rw-p`. A program's text and its data are one file mapped two ways with different permissions, and that is true of every program on every Linux machine rather than being a busybox quirk. + +Line three has no name at all. That is anonymous memory, and it is what a first write has to go and find a page for, which is the entire subject of `blueprints/page-fault.md`. + +Line three also ends in a space, and that is the trap. The kernel pads every line out to a fixed column before printing the path, and when there is no path the padding is printed anyway. So that line has five whitespace separated fields and every other line has six, and code that reaches for field six works on every maps file it has ever seen until it meets an anonymous mapping. Do not let an editor strip the trailing whitespace from this file. + +The two gaps between the mappings are most of the address space. From `0814c000` to `b7f8f000` is about 2.7 GiB of nothing, and a fault anywhere in it is a segmentation fault. + +## self-stat.txt and odd-comm-stat.txt + +The same file twice, for two processes with different names, and the pair is the point. + +`self-stat.txt` is the ordinary case: a process called `cat`. Fifty two fields on one line, which is exactly what Table 1-4 of `Documentation/filesystems/proc.rst` lists. That table is headed "as of 2.6.30-rc7" and still describes 7.2.2 without an error in it, for a file that has no ABI entry at all. Splitting this line on whitespace gives the right answer. + +`odd-comm-stat.txt` is the same file for a process whose executable is named `od) d ma`: + +``` +37 (od) d ma) R 1 0 0 0 -1 4194304 37 0 0 0 0 1 0 0 20 0 1 0 265 ... +``` + +The kernel prints the command name in brackets and does not escape it. `line.split()` on that gives `37`, `(od)`, `d`, `ma)`, `R`, and every field after the name has slid two places along. The state, field three, comes back as `d`, which is not a state any process is ever in. Nothing raises. All the numbers are still numbers. A monitor reading this would carry on reporting nonsense. + +The correct parse is the one `procps` has used for decades and it is not clever: first opening bracket, last closing bracket, and the fields are what is left. `kxray.proc.pidstat` does that, and keeps what the naive split would have said, so a lesson can print the two answers next to each other instead of asking anybody to take the trap on trust. + +Getting the capture needed a process with a name like that, and on a busybox rootfs that means a shell script. busybox dispatches on its own `argv[0]` and refuses to run under a name that is not one of its applets, so a copy of `/bin/sleep` called `od) d ma` exits immediately with "applet not found". A script gets its `comm` from the script's own filename, so the name sticks. + +One number ties this file to the one above it. `vsize` in `self-stat.txt` is 1298432, and the sizes of the seven mappings in `self-maps.txt` add up to 1298432, because `vsize` is that sum. Two files, two readers, one fact, and a test that checks they still agree. + +## self-status.txt + +The same process as `self-stat.txt`, printed for a person instead of for a program. Fifty keys, tab separated where meminfo uses spaces, and three of them break the idea that a value is a number. + +`Uid` has four values on one line: real, effective, saved and filesystem. `State` has a letter and then the same state spelled out in brackets. `Groups` is empty, and the kernel prints the key and the separator anyway. + +So the reader keeps a tuple of words per key and offers a number only when there is exactly one word and it is one. A model with `value: int` on it would have to throw two of those three away and would be wrong about the third. diff --git a/corpora/proc/tier0/interrupts.meta.toml b/corpora/proc/tier0/interrupts.meta.toml new file mode 100644 index 0000000..3d0238d --- /dev/null +++ b/corpora/proc/tier0/interrupts.meta.toml @@ -0,0 +1,36 @@ +source = "tier0" +evidence = true + +describes = "every interrupt the machine has taken since boot, one column per CPU" +path = "/proc/interrupts" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/interrupts" +setup = [] + +# The column count is the fact worth taking away. There is one column here because the box has one +# CPU. A laptop prints one per possible CPU, which is not the same as the number online and not the +# same as the number the hardware has. Only the header knows, which is why kxray.proc.percpu reads +# the header and refuses to guess. +# +# The rows underneath the numbered ones are per architecture and per config. Two here, NMI and TLB. +# An x86-64 desktop prints around fifteen and an arm64 machine prints a different set. There is no +# list of them in kxray for that reason. +# +# Read this next to softirqs.txt. This file counts the hardware asking. That one counts the work the +# answering deferred. +timings_are_real = false + +unparsed_lines = 0 +cpus = ["CPU0"] +rows = 7 +hardware_rows = 5 +named_rows = ["NMI", "TLB"] diff --git a/corpora/proc/tier0/interrupts.txt b/corpora/proc/tier0/interrupts.txt new file mode 100644 index 0000000..c364387 --- /dev/null +++ b/corpora/proc/tier0/interrupts.txt @@ -0,0 +1,8 @@ + CPU0 + 0: 316 XT-PIC timer + 1: 10 XT-PIC i8042 + 2: 0 XT-PIC cascade + 4: 436 XT-PIC ttyS0 + 12: 115 XT-PIC i8042 + NMI: 0 Non-maskable interrupts + TLB: 0 TLB shootdowns diff --git a/corpora/proc/tier0/meminfo.meta.toml b/corpora/proc/tier0/meminfo.meta.toml new file mode 100644 index 0000000..fbcd007 --- /dev/null +++ b/corpora/proc/tier0/meminfo.meta.toml @@ -0,0 +1,34 @@ +source = "tier0" +evidence = true + +describes = "the whole of memory as the kernel accounts for it, on a box given 100 MiB" +path = "/proc/meminfo" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/meminfo" +setup = [] + +# Two things this file is here to show. +# +# The unit says kB and means KiB. MemTotal is 102308 kB on a machine given 100 MiB, and 102308 +# times 1024 is a shade under 100 MiB while 102308 times 1000 is nowhere near it. The kernel has +# spelled it that way since the beginning and every tool on the machine agrees, so kxray keeps the +# kernel's spelling and does the multiplication by 1024. +# +# The key list is per config. There is no HugePages_Total here and there is a GPUActive, and a +# machine with transparent huge pages configured differently prints a different set again. Nothing +# in kxray.proc requires a key to be present. +timings_are_real = false + +unparsed_lines = 0 +keys = 47 +mem_total_kb = 102308 +mem_total_bytes = 104763392 diff --git a/corpora/proc/tier0/meminfo.txt b/corpora/proc/tier0/meminfo.txt new file mode 100644 index 0000000..282a035 --- /dev/null +++ b/corpora/proc/tier0/meminfo.txt @@ -0,0 +1,47 @@ +MemTotal: 102308 kB +MemFree: 97572 kB +MemAvailable: 95668 kB +Buffers: 0 kB +Cached: 1204 kB +SwapCached: 0 kB +Active: 1232 kB +Inactive: 0 kB +Active(anon): 1232 kB +Inactive(anon): 0 kB +Active(file): 0 kB +Inactive(file): 0 kB +Unevictable: 0 kB +Mlocked: 0 kB +SwapTotal: 0 kB +SwapFree: 0 kB +Dirty: 0 kB +Writeback: 0 kB +AnonPages: 48 kB +Mapped: 680 kB +Shmem: 1204 kB +KReclaimable: 0 kB +Slab: 2676 kB +SReclaimable: 0 kB +SUnreclaim: 2676 kB +KernelStack: 224 kB +PageTables: 64 kB +SecPageTables: 0 kB +NFS_Unstable: 0 kB +Bounce: 0 kB +WritebackTmp: 0 kB +CommitLimit: 51152 kB +Committed_AS: 1508 kB +VmallocTotal: 917496 kB +VmallocUsed: 12 kB +VmallocChunk: 0 kB +Percpu: 32 kB +AnonHugePages: 0 kB +ShmemHugePages: 0 kB +ShmemPmdMapped: 0 kB +FileHugePages: 0 kB +FilePmdMapped: 0 kB +Balloon: 0 kB +GPUActive: 0 kB +GPUReclaim: 0 kB +DirectMap4k: 8192 kB +DirectMap4M: 106496 kB diff --git a/corpora/proc/tier0/odd-comm-stat.meta.toml b/corpora/proc/tier0/odd-comm-stat.meta.toml new file mode 100644 index 0000000..3c78619 --- /dev/null +++ b/corpora/proc/tier0/odd-comm-stat.meta.toml @@ -0,0 +1,49 @@ +source = "tier0" +evidence = true + +describes = "the same file for a process whose command name has a closing bracket and two spaces in it" +path = "/proc/self/stat" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +# Why a shell script rather than a renamed binary. busybox dispatches on its own argv[0] and +# refuses to run under a name that is not one of its applets, so a copy of /bin/sleep called +# `od) d ma` exits immediately with "applet not found". A script gets its comm from the script's +# own filename, so the name sticks and the process stays alive long enough to be read. +command = "cat /proc/$!/stat" +setup = [ + "printf '#!/bin/sh\\nsleep 30\\n' > '/tmp/od) d ma'", + "chmod +x '/tmp/od) d ma'", + "'/tmp/od) d ma' &", + "sleep 1", +] + +# This is the file the trap lives in, on a real kernel, with the line the kernel actually printed: +# +# 37 (od) d ma) R 1 0 0 0 -1 4194304 37 0 ... +# +# The kernel does not escape the command name. `line.split()` gives 37, (od), d, ma), R, and every +# field after the name has slid two places along. The state, field three, comes back as `d`, which +# is not a state any process is ever in. Nothing raises and the numbers are all still numbers. +# +# The correct parse is the one procps has used for decades: first opening bracket, last closing +# bracket, and the fields are what is left. kxray.proc.pidstat does that, and keeps what the naive +# split would have said in `naive` so a lesson can print the two answers next to each other. +timings_are_real = false + +unparsed_lines = 0 +pid = 37 +comm = "od) d ma" +state = "R" +naive_state = "d" +naive_fields = 54 +fields = 50 +extra_fields = 0 +trapped_by_naive_split = true diff --git a/corpora/proc/tier0/odd-comm-stat.txt b/corpora/proc/tier0/odd-comm-stat.txt new file mode 100644 index 0000000..d269eb1 --- /dev/null +++ b/corpora/proc/tier0/odd-comm-stat.txt @@ -0,0 +1 @@ +37 (od) d ma) R 1 0 0 0 -1 4194304 37 0 0 0 0 1 0 0 20 0 1 0 265 1253376 148 4294967295 134512640 135563644 3217822688 0 0 0 2147221247 6 65536 0 0 0 17 0 0 0 0 0 0 135569392 135577561 136048640 3217829795 3217829817 3217829817 3217829870 0 diff --git a/corpora/proc/tier0/self-maps.meta.toml b/corpora/proc/tier0/self-maps.meta.toml new file mode 100644 index 0000000..e2a0e54 --- /dev/null +++ b/corpora/proc/tier0/self-maps.meta.toml @@ -0,0 +1,42 @@ +source = "tier0" +evidence = true + +describes = "the whole address space of one process, which was cat reading this file" +path = "/proc/self/maps" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/self/maps" +setup = [] + +# Seven lines and four things in them. +# +# /bin/busybox appears twice, once r-xp and once rw-p, because a program's text and its data are +# one file mapped two ways with different permissions. That is not a quirk of busybox, it is what +# every dynamically linked program on every Linux machine looks like. +# +# Line three has no name. That is anonymous memory, and it is what a first write has to go and find +# a page for, which is the whole subject of blueprints/page-fault.md. +# +# Line three also ends in a space. The kernel pads every line out to a fixed column before printing +# the path, and when there is no path the padding is still there, so the line has five whitespace +# separated fields and every other line has six. Code that reaches for field six works until it +# meets this line. Do not let an editor strip the trailing whitespace from this file. +# +# The two gaps between the mappings are most of the address space. 0814c000 to b7f8f000 is about +# 2.7 GiB of nothing, and a fault anywhere in it is a segmentation fault. +timings_are_real = false + +unparsed_lines = 0 +regions = 7 +anonymous_regions = 1 +named_regions = ["[vvar]", "[vvar_vclock]", "[vdso]", "[stack]"] +gaps = 2 +total_bytes = 1298432 diff --git a/corpora/proc/tier0/self-maps.txt b/corpora/proc/tier0/self-maps.txt new file mode 100644 index 0000000..7b91d73 --- /dev/null +++ b/corpora/proc/tier0/self-maps.txt @@ -0,0 +1,7 @@ +08048000-08149000 r-xp 00000000 00:03 11 /bin/busybox +08149000-0814c000 rw-p 00100000 00:03 11 /bin/busybox +b7f8f000-b7f9f000 rw-p 00000000 00:00 0 +b7f9f000-b7fa3000 r--p 00000000 00:00 0 [vvar] +b7fa3000-b7fa5000 r--p 00000000 00:00 0 [vvar_vclock] +b7fa5000-b7fa7000 r-xp 00000000 00:00 0 [vdso] +bfa7b000-bfa9c000 rw-p 00000000 00:00 0 [stack] diff --git a/corpora/proc/tier0/self-stat.meta.toml b/corpora/proc/tier0/self-stat.meta.toml new file mode 100644 index 0000000..39825be --- /dev/null +++ b/corpora/proc/tier0/self-stat.meta.toml @@ -0,0 +1,39 @@ +source = "tier0" +evidence = true + +describes = "the one line file behind ps and top, for an ordinary process with an ordinary name" +path = "/proc/self/stat" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/self/stat" +setup = [] + +# The boring case, kept so the interesting one has something to be compared against. The command is +# `cat`, which has no space and no bracket in it, so a whitespace split reads this line correctly +# and so does a careful parser. odd-comm-stat.txt is the same file for a process where those two +# disagree. +# +# Fifty two fields, which is exactly what Table 1-4 of Documentation/filesystems/proc.rst lists. The +# table is headed "as of 2.6.30-rc7" and still describes 7.2.2 without an error in it. +# +# total_bytes here is vsize, and it is the same number as total_bytes in self-maps.meta.toml. That +# is not a coincidence and it is worth checking: vsize is the sum of the sizes of the mappings, so +# the two files are two views of one fact and the tests compare them. +timings_are_real = false + +unparsed_lines = 0 +pid = 37 +comm = "cat" +state = "R" +fields = 50 +extra_fields = 0 +trapped_by_naive_split = false +vsize = 1298432 diff --git a/corpora/proc/tier0/self-stat.txt b/corpora/proc/tier0/self-stat.txt new file mode 100644 index 0000000..b54cf66 --- /dev/null +++ b/corpora/proc/tier0/self-stat.txt @@ -0,0 +1 @@ +37 (cat) R 1 0 0 0 -1 4194304 24 0 0 0 0 0 0 0 20 0 1 0 428 1298432 128 4294967295 134512640 135563644 3213804608 0 0 0 0 0 0 0 0 0 17 0 0 0 0 0 0 135569392 135577561 145113088 3213811626 3213811646 3213811646 3213811699 0 diff --git a/corpora/proc/tier0/self-status.meta.toml b/corpora/proc/tier0/self-status.meta.toml new file mode 100644 index 0000000..662560b --- /dev/null +++ b/corpora/proc/tier0/self-status.meta.toml @@ -0,0 +1,37 @@ +source = "tier0" +evidence = true + +describes = "the same process as self-stat.txt, printed for a person to read instead of a program" +path = "/proc/self/status" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/self/status" +setup = [] + +# Fifty keys, separated by tabs rather than by the spaces meminfo uses, and three of them break the +# idea that a value is a number. +# +# Uid has four values on one line: real, effective, saved and filesystem. State has a letter and +# then the same state spelled out in brackets. Groups is empty, and the kernel still prints the key +# and the separator. +# +# So kxray.proc.keyed keeps a tuple of words and offers a number only when there is exactly one +# word and it is one. A model with value: int on it would have to throw two of those three away. +# +# The key list is per config and per version. untag_mask, THP_enabled and the two Speculation lines +# are all here because of how this kernel was built, and another machine will differ. +timings_are_real = false + +unparsed_lines = 0 +keys = 50 +uid_values = 4 +state_values = 2 +empty_keys = ["Groups"] diff --git a/corpora/proc/tier0/self-status.txt b/corpora/proc/tier0/self-status.txt new file mode 100644 index 0000000..b0645f5 --- /dev/null +++ b/corpora/proc/tier0/self-status.txt @@ -0,0 +1,50 @@ +Name: cat +Umask: 0022 +State: R (running) +Tgid: 37 +Ngid: 0 +Pid: 37 +PPid: 1 +TracerPid: 0 +Uid: 0 0 0 0 +Gid: 0 0 0 0 +FDSize: 32 +Groups: +Kthread: 0 +VmPeak: 1268 kB +VmSize: 1268 kB +VmLck: 0 kB +VmPin: 0 kB +VmHWM: 508 kB +VmRSS: 508 kB +RssAnon: 16 kB +RssFile: 4 kB +RssShmem: 488 kB +VmData: 76 kB +VmStk: 132 kB +VmExe: 1028 kB +VmLib: 8 kB +VmPTE: 12 kB +VmSwap: 0 kB +CoreDumping: 0 +THP_enabled: 1 +untag_mask: 0xffffffff +Threads: 1 +SigQ: 0/795 +SigPnd: 0000000000000000 +ShdPnd: 0000000000000000 +SigBlk: 0000000000000000 +SigIgn: 0000000000000000 +SigCgt: 0000000000000000 +CapInh: 0000000000000000 +CapPrm: 000001ffffffffff +CapEff: 000001ffffffffff +CapBnd: 000001ffffffffff +CapAmb: 0000000000000000 +NoNewPrivs: 0 +Speculation_Store_Bypass: vulnerable +SpeculationIndirectBranch: always enabled +Cpus_allowed: 1 +Cpus_allowed_list: 0 +voluntary_ctxt_switches: 0 +nonvoluntary_ctxt_switches: 1 diff --git a/corpora/proc/tier0/softirqs.meta.toml b/corpora/proc/tier0/softirqs.meta.toml new file mode 100644 index 0000000..ae079bc --- /dev/null +++ b/corpora/proc/tier0/softirqs.meta.toml @@ -0,0 +1,33 @@ +source = "tier0" +evidence = true + +describes = "the deferred half of interrupt handling, counted since boot" +path = "/proc/softirqs" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/softirqs" +setup = [] + +# Ten vectors, and on an idle emulated box seven of them have never fired. The two that have are +# TIMER and RCU, which is the same pair that corpora/traces/tier0/flat-interrupt.txt catches in the +# act: raise_softirq inside the hardware handler, then handle_softirqs four lines later with +# interrupts back on. This file is that gap counted since boot instead of watched once. +# +# The counts move every boot, so the tests check the shape and the ordering of the vectors rather +# than the numbers. The numbers below are what this particular boot had. +timings_are_real = false + +unparsed_lines = 0 +cpus = ["CPU0"] +rows = 10 +quiet_rows = 7 +rcu = 246 +timer = 26 diff --git a/corpora/proc/tier0/softirqs.txt b/corpora/proc/tier0/softirqs.txt new file mode 100644 index 0000000..31ba16f --- /dev/null +++ b/corpora/proc/tier0/softirqs.txt @@ -0,0 +1,11 @@ + CPU0 + HI: 0 + TIMER: 26 + NET_TX: 0 + NET_RX: 0 + BLOCK: 0 + IRQ_POLL: 0 + TASKLET: 1 + SCHED: 0 + HRTIMER: 0 + RCU: 246 diff --git a/corpora/proc/tier0/version.meta.toml b/corpora/proc/tier0/version.meta.toml new file mode 100644 index 0000000..6da1e6f --- /dev/null +++ b/corpora/proc/tier0/version.meta.toml @@ -0,0 +1,30 @@ +source = "tier0" +evidence = true + +describes = "the kernel's own banner, which is the first thing worth reading on a machine you have not seen before" +path = "/proc/version" +stability = "undocumented" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-05" +tier = 0 + +command = "cat /proc/version" +setup = [] + +# One line. The release and the build number are worth pulling out and the middle is not: it holds +# the user and host that built the kernel, then the whole compiler and linker banner, with brackets +# inside brackets. Nothing promises its shape. +# +# `PREEMPT` in the trailing part is what pin.toml asked for, so this file is also the cheapest +# confirmation that the running kernel is the one the profile describes. +timings_are_real = false + +unparsed_lines = 0 +release = "7.2.2" +parts = [7, 2, 2] +build = "#1" diff --git a/corpora/proc/tier0/version.txt b/corpora/proc/tier0/version.txt new file mode 100644 index 0000000..207e250 --- /dev/null +++ b/corpora/proc/tier0/version.txt @@ -0,0 +1 @@ +Linux version 7.2.2 (kxbox@kxbox) (i686-linux-gnu-gcc (Debian 14.2.0-19) 14.2.0, GNU ld (GNU Binutils for Debian) 2.44) #1 PREEMPT @0 diff --git a/kxray/models.py b/kxray/models.py index aa99c5b..b64a04a 100644 --- a/kxray/models.py +++ b/kxray/models.py @@ -653,6 +653,521 @@ def table(self, *fields: str) -> str: return "\n".join(out) +# --------------------------------------------------------------------------------------------- +# What comes out of /proc and /sys. Parsed by kxray.proc, and every one of these carries the +# promise the kernel makes about the file it was read from, because most of these files are not +# promised anything at all and a reader should be told that before it leans on one. + + +# The four levels Documentation/ABI/README defines, one directory each. +STABLE = "stable" +TESTING = "testing" +OBSOLETE = "obsolete" +REMOVED = "removed" + +# The two levels that are not in that README as directories but are the honest answer for most of +# what this project reads. +# +# UNDOCUMENTED means no file under Documentation/ABI describes this path. That is not the same as +# being unstable. Almost all of /proc is here, including every file in this corpus, and the rule +# that userspace does not get broken still applies to them in practice. What is missing is the +# written promise, which means nobody has said what part of the file you may depend on. +# +# NOT_ABI is stronger and rarer. Documentation/ABI/README has a closing section that names two +# things as "notable bits of non-ABI, which should not under any circumstances be considered +# stable", and this project reads both of them. +UNDOCUMENTED = "undocumented" +NOT_ABI = "not-abi" + +LEVELS = (STABLE, TESTING, OBSOLETE, REMOVED, UNDOCUMENTED, NOT_ABI) + + +@dataclass(frozen=True) +class Promise: + """What the kernel tree says about one path, and where it says it. + + This exists so that a parser cannot report a value without also reporting what kind of value + it is. A number off `/sys/kernel/btf/vmlinux` and a number off `/proc/kallsyms` look the same + coming out of Python and they are not the same kind of fact: one has a documented interface + behind it and the other is explicitly named as something you must not depend on. + + `entry` is the file in the kernel source that carries the claim, so any of this can be + checked rather than believed. + """ + + kind: str + entry: str = "" + note: str = "" + pattern: str = "" + + @property + def documented(self) -> bool: + """Whether anything under Documentation/ABI describes this path at all.""" + return self.kind in (STABLE, TESTING, OBSOLETE, REMOVED) + + @property + def dependable(self) -> bool: + """Whether a tool may rest on the shape of this file across kernels. + + True only for `stable` and `testing`, which are the two levels whose README text says + userspace may rely on them. Everything else is a maybe, and a maybe dressed up as a yes is + how a tool ends up quietly wrong on somebody else's machine. + """ + return self.kind in (STABLE, TESTING) + + def __str__(self) -> str: + where = f" ({self.entry})" if self.entry else "" + return f"{self.kind}{where}" + + +@dataclass +class ProcFile: + """One file out of /proc or /sys, read. + + Every reader in `kxray.proc` returns something built on this, so `path` and `promise` are + always there to be printed next to whatever was found. + """ + + source: str = "" + path: str = "" + promise: Promise = field(default_factory=lambda: Promise(UNDOCUMENTED)) + lines: Lines = field(default_factory=Lines) + + def banner(self) -> str: + """One line saying where this came from and what it is worth.""" + return f"{self.path or self.source}: {self.promise}" + + +@dataclass(frozen=True) +class KeyValue: + """One `Key: value` line, with the value left as the words the kernel wrote. + + `values` is a tuple rather than a string because the kernel does not stick to one value per + key. `Uid:` in `/proc/self/status` has four of them, `State:` has a letter and a word in + brackets, and `MemTotal:` has a number and a unit. A model with a single `value: int` on it + would have to throw two of those three away. + """ + + key: str + values: tuple[str, ...] + unit: str = "" + line: int = 0 + + @property + def text(self) -> str: + return " ".join(self.values) + + @property + def number(self) -> int | None: + """The value as an integer, when there is exactly one and it is one. + + None for `State: R (running)` and for `Uid: 0 0 0 0`, on purpose. A caller that wants a + number out of those has to say which part it means. + """ + if len(self.values) != 1: + return None + try: + return int(self.values[0], 0) + except ValueError: + return None + + def __str__(self) -> str: + unit = f" {self.unit}" if self.unit else "" + return f"{self.key}: {self.text}{unit}" + + +@dataclass +class KeyedFile(ProcFile): + """A whole file of `Key: value` lines, in the order the kernel printed them.""" + + entries: tuple[KeyValue, ...] = () + + @property + def keys(self) -> tuple[str, ...]: + return tuple(one.key for one in self.entries) + + def get(self, key: str) -> KeyValue | None: + return next((one for one in self.entries if one.key == key), None) + + def number(self, key: str) -> int | None: + found = self.get(key) + return found.number if found is not None else None + + def __getitem__(self, key: str) -> KeyValue: + found = self.get(key) + if found is None: + raise KeyError(f"{self.path} has no {key} line on this kernel") + return found + + def __contains__(self, key: str) -> bool: + return self.get(key) is not None + + def table(self, *keys: str) -> str: + wanted = [one for one in self.entries if not keys or one.key in keys] + rows = [("key", "value", "unit")] + rows += [(one.key, one.text, one.unit) for one in wanted] + return grid(rows) + + +@dataclass(frozen=True) +class Counter: + """One row of a file that counts something once per CPU. + + `/proc/interrupts` and `/proc/softirqs` are the same shape: a label, then one number for each + CPU, then in the interrupts case some text saying what the line is about. + """ + + label: str + counts: tuple[int, ...] = () + detail: str = "" + line: int = 0 + + @property + def total(self) -> int: + return sum(self.counts) + + @property + def fired(self) -> bool: + return self.total > 0 + + def on(self, cpu: int) -> int: + return self.counts[cpu] + + def __str__(self) -> str: + detail = f" {self.detail}" if self.detail else "" + return f"{self.label}: {self.total}{detail}" + + +@dataclass +class CounterFile(ProcFile): + """A whole per CPU counter file. + + `cpus` comes off the header row rather than from anywhere else, because the number of columns + is the number of CPUs the kernel is willing to print and nothing else knows that number. A + reader that assumed one column would misread every desktop and a reader that assumed the + machine's CPU count would misread a kernel that prints only the online ones. + """ + + cpus: tuple[str, ...] = () + counters: tuple[Counter, ...] = () + + @property + def cpu_count(self) -> int: + return len(self.cpus) + + @property + def labels(self) -> tuple[str, ...]: + return tuple(one.label for one in self.counters) + + def get(self, label: str) -> Counter | None: + return next((one for one in self.counters if one.label == label), None) + + def total(self, label: str) -> int: + found = self.get(label) + return found.total if found is not None else 0 + + def busiest(self, limit: int = 5) -> list[Counter]: + return sorted(self.counters, key=lambda one: -one.total)[:limit] + + def quiet(self) -> list[Counter]: + """The rows that never fired, which on a small machine is most of them.""" + return [one for one in self.counters if not one.fired] + + def table(self, limit: int = 0) -> str: + wanted = self.busiest(limit) if limit else list(self.counters) + rows = [("label", *self.cpus, "detail")] + for one in wanted: + rows.append((one.label, *[str(count) for count in one.counts], one.detail)) + return grid(rows) + + +@dataclass(frozen=True) +class Region: + """One mapping in an address space, as `/proc//maps` prints it.""" + + start: int + end: int + perms: str + offset: int + dev: str + inode: int + path: str = "" + line: int = 0 + + @property + def size(self) -> int: + return self.end - self.start + + @property + def pages(self) -> int: + """How many 4 KiB pages this covers, which is the unit the fault handler works in.""" + return self.size // 4096 + + @property + def readable(self) -> bool: + return self.perms[0] == "r" + + @property + def writable(self) -> bool: + return self.perms[1] == "w" + + @property + def executable(self) -> bool: + return self.perms[2] == "x" + + @property + def private(self) -> bool: + return self.perms[3] == "p" + + @property + def special(self) -> bool: + """Whether the kernel named this rather than a file, so `[stack]` or `[vdso]`.""" + return self.path.startswith("[") and self.path.endswith("]") + + @property + def anonymous(self) -> bool: + """No file behind it. This is the memory a first write has to go and find a page for.""" + return not self.path + + @property + def label(self) -> str: + return self.path or "anonymous" + + def holds(self, address: int) -> bool: + return self.start <= address < self.end + + def __str__(self) -> str: + return f"{self.start:08x}-{self.end:08x} {self.perms} {self.label}" + + +@dataclass +class AddressSpace(ProcFile): + """Every mapping one process has, in the order the kernel walked them, which is by address.""" + + pid: int = 0 + regions: tuple[Region, ...] = () + + @property + def total_size(self) -> int: + return sum(one.size for one in self.regions) + + def find(self, needle: str) -> list[Region]: + return [one for one in self.regions if needle in one.label] + + def at(self, address: int) -> Region | None: + """Which mapping an address is in, or None for a hole. + + A fault on an address with no mapping is the segmentation fault case, so None here is a + real answer rather than a lookup failure. + """ + return next((one for one in self.regions if one.holds(address)), None) + + def executable(self) -> list[Region]: + return [one for one in self.regions if one.executable] + + def named(self) -> list[Region]: + return [one for one in self.regions if one.special] + + def gaps(self) -> list[tuple[int, int]]: + """The unmapped stretches between mappings, as (start, size). + + Most of a 32-bit address space is gap, and seeing that written down as numbers is the + quickest way to stop thinking of an address space as a block of memory. + """ + found = [] + for before, after in zip(self.regions, self.regions[1:], strict=False): + if after.start > before.end: + found.append((before.end, after.start - before.end)) + return found + + def table(self) -> str: + rows = [("start", "end", "perms", "size", "pages", "what")] + for one in self.regions: + rows.append( + ( + f"{one.start:08x}", + f"{one.end:08x}", + one.perms, + str(one.size), + str(one.pages), + one.label, + ) + ) + return grid(rows) + + +# The names of the fields in `/proc//stat`, in order, from Table 1-4 of +# Documentation/filesystems/proc.rst. The table is headed "as of 2.6.30-rc7" and it still +# describes 7.2.2 exactly, all 52 of them, which is worth noticing: the file has no entry under +# Documentation/ABI at all and has not moved a field in fifteen years anyway. +# +# The three placeholders are printed as a literal 0 by the kernel. The first used to be the wchan +# address and proc.rst says to read `/proc//wchan` instead. +STAT_FIELDS = ( + "pid", + "tcomm", + "state", + "ppid", + "pgrp", + "sid", + "tty_nr", + "tty_pgrp", + "flags", + "min_flt", + "cmin_flt", + "maj_flt", + "cmaj_flt", + "utime", + "stime", + "cutime", + "cstime", + "priority", + "nice", + "num_threads", + "it_real_value", + "start_time", + "vsize", + "rss", + "rsslim", + "start_code", + "end_code", + "start_stack", + "esp", + "eip", + "pending", + "blocked", + "sigign", + "sigcatch", + "placeholder_wchan", + "placeholder_2", + "placeholder_3", + "exit_signal", + "task_cpu", + "rt_priority", + "policy", + "blkio_ticks", + "gtime", + "cgtime", + "start_data", + "end_data", + "start_brk", + "arg_start", + "arg_end", + "env_start", + "env_end", + "exit_code", +) + + +@dataclass +class PidStat(ProcFile): + """The one line file, read the only way it can be read correctly. + + The second field is the command name in brackets and the kernel does not escape it, so a + process called `od) d ma` prints as `37 (od) d ma) R 1 0 ...` and a whitespace split lands on + the wrong field from there on. `naive` keeps what that split would have said so a lesson can + show the two side by side instead of asserting that the trap is real. + """ + + pid: int = 0 + comm: str = "" + values: dict[str, str] = field(default_factory=dict) + extra: tuple[str, ...] = () + naive: tuple[str, ...] = () + + @property + def state(self) -> str: + return self.values.get("state", "") + + @property + def ppid(self) -> int: + return int(self.values.get("ppid", 0)) + + @property + def threads(self) -> int: + return int(self.values.get("num_threads", 0)) + + @property + def faults(self) -> tuple[int, int]: + """Minor and major faults, which is the pair `blueprints/page-fault.md` counts.""" + return int(self.values.get("min_flt", 0)), int(self.values.get("maj_flt", 0)) + + def number(self, name: str) -> int | None: + raw = self.values.get(name) + if raw is None: + return None + try: + return int(raw) + except ValueError: + return None + + @property + def naive_state(self) -> str: + """What `line.split()[2]` would have said the state was. + + Equal to `state` on almost every process on almost every machine, which is exactly why the + wrong parse survives so long in so much code. + """ + return self.naive[2] if len(self.naive) > 2 else "" + + def table(self, *names: str) -> str: + wanted = names or STAT_FIELDS + rows = [("field", "value")] + for name in wanted: + if name == "tcomm": + rows.append((name, f"({self.comm})")) + elif name in self.values: + rows.append((name, self.values[name])) + return grid(rows) + + +@dataclass +class Version(ProcFile): + """`/proc/version`, taken apart as far as it can honestly be taken apart. + + The release and the build number are worth pulling out and the rest is not. What sits between + them is the user and host that built it and the whole compiler and linker banner, in + parentheses, with parentheses inside it, and there is no promise anywhere about its shape. + `rest` keeps it as text rather than pretending otherwise. + """ + + release: str = "" + build: str = "" + rest: str = "" + text: str = "" + + @property + def parts(self) -> tuple[int, ...]: + """The release as numbers, for comparing kernels. Trailing junk is dropped.""" + found = [] + for piece in self.release.split("."): + digits = "" + for char in piece: + if not char.isdigit(): + break + digits += char + if not digits: + break + found.append(int(digits)) + return tuple(found) + + def at_least(self, *wanted: int) -> bool: + return self.parts[: len(wanted)] >= tuple(wanted) + + +def grid(rows: list[tuple[str, ...]]) -> str: + """A header row, a rule, then the rest, every column as wide as its widest cell.""" + width = max(len(row) for row in rows) + padded = [tuple(list(row) + [""] * (width - len(row))) for row in rows] + widths = [max(len(row[i]) for row in padded) for i in range(width)] + out = [] + for index, row in enumerate(padded): + out.append(" ".join(cell.ljust(widths[i]) for i, cell in enumerate(row)).rstrip()) + if index == 0: + out.append(" ".join("-" * one for one in widths)) + return "\n".join(out) + + # --------------------------------------------------------------------------------------------- # What the kernel knows about its own types. Parsed by kxray.btf, rendered by kxwidgets and by # the generated sections of a blueprint. diff --git a/kxray/proc/__init__.py b/kxray/proc/__init__.py new file mode 100644 index 0000000..81c6105 --- /dev/null +++ b/kxray/proc/__init__.py @@ -0,0 +1,76 @@ +"""The /proc and /sys files the lessons read, each one knowing what it is worth. + + from kxray import proc + + print(proc.read("corpora/proc/tier0/meminfo.txt", "/proc/meminfo").table()) + print(proc.stability.table()) + +Five readers for four file shapes, because /proc has fewer shapes in it than it has files. +`keyed` handles `Key: value`, which is meminfo and a process status file. `percpu` handles a +label and one count per CPU, which is interrupts and softirqs. `maps` handles one record per line +with positional columns. `pidstat` handles the single line with the command name in it, which is +its own shape because of what the command name is allowed to contain. `version` is one line of +free text with two useful things in it. + +Every one of them returns something carrying a `promise`, from `kxray.proc.stability`, saying +what the kernel tree says about the file it read. That is not decoration. On Linux 7.2.2 there +are 685 files under `Documentation/ABI` and six of them describe a path in /proc, and not one of +those six is a file in this corpus. Everything here is read on custom rather than on a promise, +and the object says so when you print it. +""" + +from __future__ import annotations + +from pathlib import Path + +from kxray.models import ProcFile +from kxray.proc import keyed, maps, percpu, pidstat, stability, version + +__all__ = [ + "READERS", + "keyed", + "maps", + "percpu", + "pidstat", + "read", + "reader_for", + "stability", + "version", +] + +# Which reader opens which kernel path. Patterns rather than names, because a per process file is +# read under a pid and under `self` and under `thread-self` and it is the same file either way. +READERS = ( + ("/proc/version", version), + ("/proc/meminfo", keyed), + ("/proc/interrupts", percpu), + ("/proc/softirqs", percpu), + ("/proc/*/stat", pidstat), + ("/proc/*/maps", maps), + ("/proc/*/status", keyed), +) + + +def reader_for(kernel_path: str): + """The module that reads this path, or None when nothing here does. + + None rather than a guess. A file shape that has not been looked at is not a `Key: value` file + merely because most of them are, and returning `keyed` for anything unrecognised would turn a + missing reader into a quiet half correct one. + """ + from fnmatch import fnmatch + + return next((module for pattern, module in READERS if fnmatch(kernel_path, pattern)), None) + + +def read(path: Path | str, kernel_path: str) -> ProcFile: + """Read a captured file with whatever reader its kernel path calls for. + + `kernel_path` is passed separately because a capture on disk is called something else. The + file in the corpus is `self-maps.txt` and what it is, is `/proc/self/maps`, and only the + second of those decides how to read it or what it is worth. + """ + module = reader_for(kernel_path) + if module is None: + raise LookupError(f"nothing in kxray.proc reads {kernel_path}") + return module.parse_file(path, kernel_path) diff --git a/kxray/proc/keyed.py b/kxray/proc/keyed.py new file mode 100644 index 0000000..7b8cba6 --- /dev/null +++ b/kxray/proc/keyed.py @@ -0,0 +1,111 @@ +"""The `Key: value` files, which is most of what people mean when they say they read /proc. + + from kxray.proc import keyed + + mem = keyed.parse_file("corpora/proc/tier0/meminfo.txt", "/proc/meminfo") + print(mem.number("MemTotal"), mem["MemTotal"].unit) + +`/proc/meminfo` and `/proc//status` are the same file shape with different separators, so +they are read by the same code. One key per line, a colon, then the value. + +Three things make this less simple than it sounds, and all three are visible in the two captures +in `corpora/proc/tier0/`. + +The value is not always one number. `Uid:` in a status file is four numbers, the real, effective, +saved and filesystem user IDs, all on one line. `State:` is a letter and then the same state +spelled out in brackets. `Groups:` on a process in no supplementary groups is empty, and the +kernel still prints the key and a separator. So the model keeps a tuple of words and offers a +number only when there is exactly one word and it is one. + +The unit is a lie that everybody has agreed to. The kernel writes `kB` and means KiB: the pinned +box reports `MemTotal: 102308 kB` for a machine given 100 MiB, and 102308 times 1024 is a hair +under 100 MiB while 102308 times 1000 is not. `unit` keeps the kernel's spelling, because +changing it here would mean this file disagreeing with every other tool on the machine, and +`bytes()` does the multiplication with the right factor. + +The set of keys is per config and per version. This box has `untag_mask` and `THP_enabled` in a +status file and no `HugePages_Total` in meminfo, and another machine will differ in both +directions. So nothing here has a required key list, and a missing key raises a KeyError naming +the kernel rather than returning zero. +""" + +from __future__ import annotations + +from pathlib import Path + +from kxray.models import READ, SKIPPED, UNPARSED, KeyedFile, KeyValue, Lines +from kxray.proc.stability import classify + +# The only unit the kernel ever writes in these files. It means 1024 bytes. +UNITS = ("kB",) + +KIB = 1024 + + +def _read_line(line: str, number: int) -> KeyValue | None: + """One line, or None when there is no key on it.""" + key, sep, rest = line.partition(":") + if not sep or not key.strip() or ":" in key: + return None + words = rest.split() + unit = "" + if words and words[-1] in UNITS: + unit = words[-1] + words = words[:-1] + return KeyValue(key=key.strip(), values=tuple(words), unit=unit, line=number) + + +def parse(text: str, path: str = "", source: str = "") -> KeyedFile: + """Every `Key: value` line in the file, in the order the kernel wrote them. + + Order is kept because it carries meaning. The status file groups memory keys together and the + meminfo file puts the totals first, and a dict would throw that away for no gain. + """ + entries = [] + lines = Lines() + for number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + lines.count(SKIPPED) + continue + found = _read_line(line, number) + if found is None: + lines.count(UNPARSED) + continue + lines.count(READ) + entries.append(found) + return KeyedFile( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + entries=tuple(entries), + ) + + +def parse_file(path: Path | str, kernel_path: str = "") -> KeyedFile: + found = Path(path) + return parse(found.read_text(encoding="utf-8"), kernel_path, found.as_posix()) + + +def account(text: str) -> Lines: + return parse(text).lines + + +def bytes_of(found: KeyedFile, key: str) -> int | None: + """One key as a count of bytes, doing the KiB multiplication the kernel's spelling hides. + + None when the key is not there or has no single number, which is the same answer `number` + gives and for the same reason. + """ + entry = found.get(key) + if entry is None or entry.number is None: + return None + return entry.number * KIB if entry.unit == "kB" else entry.number + + +def report(found: KeyedFile) -> str: + """What was read and what it is worth, for the top of a lesson.""" + lines = [found.banner(), f"keys: {len(found.entries)}", f"lines: {found.lines}"] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/proc/maps.py b/kxray/proc/maps.py new file mode 100644 index 0000000..51c9559 --- /dev/null +++ b/kxray/proc/maps.py @@ -0,0 +1,139 @@ +"""An address space, one line per mapping, from `/proc//maps`. + + from kxray.proc import maps + + space = maps.parse_file("corpora/proc/tier0/self-maps.txt", "/proc/self/maps") + print(space.table()) + print(space.at(0x08048100)) + +This is the file that turns an address space from an idea into a list you can count. Seven lines +on the pinned box, for a process running `cat`: + + 08048000-08149000 r-xp 00000000 00:03 11 /bin/busybox + 08149000-0814c000 rw-p 00100000 00:03 11 /bin/busybox + b7f8f000-b7f9f000 rw-p 00000000 00:00 0 + b7f9f000-b7fa3000 r--p 00000000 00:00 0 [vvar] + b7fa3000-b7fa5000 r--p 00000000 00:00 0 [vvar_vclock] + b7fa5000-b7fa7000 r-xp 00000000 00:00 0 [vdso] + bfa7b000-bfa9c000 rw-p 00000000 00:00 0 [stack] + +Everything a page fault lesson needs is in there. The same file appears twice with different +permissions, once executable and once writable, because a program's text and its data are one +file mapped two ways. The third line has no name at all, which is the anonymous memory a first +write has to go and find a page for. Three of them are the kernel handing the process pieces of +itself. And between `0814c000` and `b7f8f000` there is nothing, which is most of the address +space and is the point. + +The parse has one trap in it and it is in the file above, invisible in a browser. An anonymous +mapping has no path, and the kernel pads the line out to a fixed column before printing the path +it does not have, so the line ends in a space and has five fields on it rather than six. Code +that says `line.split()[5]` works on every line of every maps file until it meets one, and then +raises IndexError from inside something that was doing fine a moment ago. The regex below has the +path as optional, which is the shape the file actually has. + +The bracketed names are not a fixed list. This kernel prints `[vvar]`, `[vvar_vclock]`, `[vdso]` +and `[stack]`, and other kernels print `[heap]`, `[vsyscall]` and names this one has never heard +of. So nothing here matches on them. `special` asks whether the kernel named the mapping instead +of a file, which stays true whatever the name turns out to be. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from kxray.models import READ, SKIPPED, UNPARSED, AddressSpace, Lines, Region +from kxray.proc.stability import classify + +# Two addresses, four permission characters, a file offset, a device, an inode, and then a path +# that is often not there. The trailing group is deliberately loose: it holds a filename with +# spaces in it as happily as it holds `[stack]`, and both are things the kernel prints. +LINE_RE = re.compile( + r"^(?P[0-9a-fA-F]+)-(?P[0-9a-fA-F]+)\s+" + r"(?P[rwxsp-]{4})\s+" + r"(?P[0-9a-fA-F]+)\s+" + r"(?P[0-9a-fA-F]+:[0-9a-fA-F]+)\s+" + r"(?P\d+)" + r"(?:\s+(?P.*))?$" +) + +PAGE = 4096 + + +def _read_line(line: str, number: int) -> Region | None: + match = LINE_RE.match(line) + if match is None: + return None + return Region( + start=int(match["start"], 16), + end=int(match["end"], 16), + perms=match["perms"], + offset=int(match["offset"], 16), + dev=match["dev"], + inode=int(match["inode"]), + path=(match["path"] or "").strip(), + line=number, + ) + + +def parse(text: str, path: str = "", source: str = "", pid: int = 0) -> AddressSpace: + """Every mapping, in the order the kernel walked them, which is by address. + + The order is not incidental. The kernel keeps mappings in a tree sorted by address and walks + it, so the file comes out sorted, which is what makes `gaps` meaningful and what lets `at` be + a plain scan rather than a lookup. + """ + regions = [] + lines = Lines() + for number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + lines.count(SKIPPED) + continue + found = _read_line(line, number) + if found is None: + lines.count(UNPARSED) + continue + lines.count(READ) + regions.append(found) + return AddressSpace( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + pid=pid, + regions=tuple(regions), + ) + + +def parse_file(path: Path | str, kernel_path: str = "", pid: int = 0) -> AddressSpace: + found = Path(path) + return parse(found.read_text(encoding="utf-8"), kernel_path, found.as_posix(), pid) + + +def account(text: str) -> Lines: + return parse(text).lines + + +def naive_fields(line: str) -> list[str]: + """What a whitespace split gives for one line, which is the wrong parse worth showing. + + Six fields for a mapping with a path and five for one without, from a file where the two look + identical unless you count the trailing spaces. This is here so a lesson can print the two + counts next to each other rather than asserting that the difference exists. + """ + return line.split() + + +def report(space: AddressSpace) -> str: + anonymous = [one for one in space.regions if one.anonymous] + lines = [ + space.banner(), + f"regions: {len(space.regions)}", + f"mapped: {space.total_size // PAGE} pages", + f"anon: {len(anonymous)} with no file behind them", + f"named: {', '.join(one.path for one in space.named()) or 'none'}", + f"gaps: {len(space.gaps())} unmapped stretches between them", + ] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/proc/percpu.py b/kxray/proc/percpu.py new file mode 100644 index 0000000..9287eab --- /dev/null +++ b/kxray/proc/percpu.py @@ -0,0 +1,145 @@ +"""The counter files, where the number of columns is a fact about the machine. + + from kxray.proc import percpu + + irqs = percpu.parse_file("corpora/proc/tier0/interrupts.txt", "/proc/interrupts") + print(irqs.cpu_count, irqs.total("0")) + +`/proc/interrupts` and `/proc/softirqs` are one shape: a header naming the CPUs, then rows of a +label, one count for each CPU, and in the interrupts case some trailing text saying what the line +is for. Reading one takes four lines of Python. Reading one without hard coding anything is the +part worth writing down. + +The column count comes off the header and nowhere else. Not from `os.cpu_count()`, not from +`/proc/cpuinfo`, and certainly not from a constant. The pinned box has one CPU and prints one +column. A sixteen thread laptop prints sixteen. A kernel that has offlined a CPU prints a column +for it anyway, because these are per possible CPU, and one that was built with a lower +`CONFIG_NR_CPUS` prints fewer than the hardware has. The header is the only place all of that is +already resolved. + +The rows below the numbered interrupts are per architecture and per config. This box, being +32-bit x86 under an emulator with one CPU and no local APIC to speak of, prints two of them: +`NMI` and `TLB`. An ordinary x86-64 desktop prints somewhere around fifteen, including `LOC`, +`RES`, `CAL` and `TRM`, and an arm64 machine prints a different set again. So there is no list of +them here. A row is a row, its label is whatever the kernel wrote, and `detail` keeps the text. + +What the two files are for is easier to see together than apart. `/proc/interrupts` counts the +hardware asking for attention. `/proc/softirqs` counts the deferred work that the answering did +not do itself. `corpora/traces/tier0/flat-interrupt.txt` is that gap happening, four lines apart, +with timestamps on it. These two files are the same gap counted since boot. +""" + +from __future__ import annotations + +from pathlib import Path + +from kxray.models import READ, SKIPPED, UNPARSED, Counter, CounterFile, Lines +from kxray.proc.stability import classify + + +def _header(line: str) -> tuple[str, ...] | None: + """The CPU names off the first row, or None when this is not that row. + + The kernel writes the header as leading whitespace and then `CPU0 CPU1 ...` with no label and + no colon, which is what makes it recognisable without matching on the word CPU: it is the one + line in the file that has no colon on it. + """ + if ":" in line or not line.strip(): + return None + names = tuple(line.split()) + return names or None + + +def _read_line(line: str, cpus: int, number: int) -> Counter | None: + """One row, or None when the counts are not counts.""" + label, sep, rest = line.partition(":") + if not sep or not label.strip(): + return None + words = rest.split(None, cpus) + if len(words) < cpus: + return None + counts = [] + for word in words[:cpus]: + try: + counts.append(int(word)) + except ValueError: + return None + detail = words[cpus].strip() if len(words) > cpus else "" + return Counter(label=label.strip(), counts=tuple(counts), detail=detail, line=number) + + +def parse(text: str, path: str = "", source: str = "") -> CounterFile: + """The header, then every row that fits it. + + A row is only read once the header has been seen, because until then there is no way to know + where the counts stop and the description starts. `NMI: 0 Non-maskable interrupts` splits into + a label and four words, and only the column count says that one of them is a number and three + of them are prose. + """ + cpus: tuple[str, ...] = () + counters = [] + lines = Lines() + for number, line in enumerate(text.splitlines(), start=1): + if not line.strip(): + lines.count(SKIPPED) + continue + if not cpus: + found_cpus = _header(line) + if found_cpus is not None: + cpus = found_cpus + lines.count(SKIPPED) + continue + # A row before the header is a row nobody can read. With no column count there is no + # way to say where the numbers stop, so this is unparsed rather than read with an empty + # list of counts, which is the shape that would sail through and mean nothing. + lines.count(UNPARSED) + continue + found = _read_line(line, len(cpus), number) + if found is None: + lines.count(UNPARSED) + continue + lines.count(READ) + counters.append(found) + return CounterFile( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + cpus=cpus, + counters=tuple(counters), + ) + + +def parse_file(path: Path | str, kernel_path: str = "") -> CounterFile: + found = Path(path) + return parse(found.read_text(encoding="utf-8"), kernel_path, found.as_posix()) + + +def account(text: str) -> Lines: + return parse(text).lines + + +def hardware(found: CounterFile) -> list[Counter]: + """The rows whose label is an interrupt number rather than a name. + + Those are the lines with a device on the end of them. The named rows underneath are the + kernel's own counters and are a different kind of thing, even though the file prints them the + same way. + """ + return [one for one in found.counters if one.label.isdigit()] + + +def named(found: CounterFile) -> list[Counter]: + return [one for one in found.counters if not one.label.isdigit()] + + +def report(found: CounterFile) -> str: + lines = [ + found.banner(), + f"cpus: {found.cpu_count} column(s): {', '.join(found.cpus)}", + f"rows: {len(found.counters)}, of which {len(found.quiet())} never fired", + f"lines: {found.lines}", + ] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/proc/pidstat.py b/kxray/proc/pidstat.py new file mode 100644 index 0000000..3ba88c3 --- /dev/null +++ b/kxray/proc/pidstat.py @@ -0,0 +1,142 @@ +"""One line, fifty two fields, and the oldest parsing trap in /proc. + + from kxray.proc import pidstat + + stat = pidstat.parse_file("corpora/proc/tier0/odd-comm-stat.txt", "/proc/self/stat") + print(stat.state, stat.naive_state) + +`/proc//stat` is the file behind `ps`, behind `top`, and behind most of the process metrics +anything has ever collected. It is one line of space separated values, so the obvious way to read +it is `line.split()`, and that is wrong. + +The second field is the command name and the kernel prints it in brackets without escaping it. +Command names come from the filename of whatever was executed, so they can contain spaces, and +they can contain a closing bracket. Here is a real line off the pinned box, from a process whose +executable is named `od) d ma`: + + 37 (od) d ma) R 1 0 0 0 -1 4194304 37 0 0 0 0 1 0 0 20 0 1 0 265 ... + +`line.split()` on that gives `37`, `(od)`, `d`, `ma)`, `R`, and everything after has slid two +places along. The state, which every reader of this file wants and which is meant to be field +three, is now `d`. Nothing raises. The numbers are all still numbers. A monitor reading this +would report a running process as being in a state that does not exist and carry on. + +The fix is not clever and has been in `procps` for decades: the command is everything between the +first opening bracket and the last closing bracket, and the fields are what is left. `parse` does +that, and it also keeps what the naive split would have said, in `naive`, so a lesson can print +the two answers side by side instead of asking anybody to take this on trust. + +`corpora/proc/tier0/odd-comm-stat.txt` is that capture. Making it needed a process with a name +like that, which on a busybox rootfs means a shell script, because busybox dispatches on its own +argv[0] and refuses to run under a name that is not an applet. The kernel takes `comm` from the +script's filename, so the script gets the name and the trap fires. + +The field names come from Table 1-4 of `Documentation/filesystems/proc.rst`. That table is headed +"as of 2.6.30-rc7" and it still describes 7.2.2 correctly, all fifty two fields in the same order, +which is a good thing to sit with for a moment. This file has no entry under `Documentation/ABI`. +Nothing promises its shape. It has not moved a field in fifteen years regardless, because too much +depends on it, and that is what the rule about not breaking userspace looks like from the outside. +""" + +from __future__ import annotations + +from pathlib import Path + +from kxray.models import READ, SKIPPED, STAT_FIELDS, UNPARSED, Lines, PidStat +from kxray.proc.stability import classify + + +def split_comm(text: str) -> tuple[str, str, str] | None: + """The line in three parts: before the command, the command, after it. + + First opening bracket, last closing bracket. Not a regex, because the regex that gets this + right is harder to read than the two index calls, and the one that is pleasant to read is the + greedy one that gets it wrong. + """ + opened = text.find("(") + closed = text.rfind(")") + if opened < 0 or closed < opened: + return None + return text[:opened], text[opened + 1 : closed], text[closed + 1 :] + + +def parse(text: str, path: str = "", source: str = "") -> PidStat: + """The one line, with the command lifted out before anything is split. + + Extra fields beyond the fifty two the documentation names go into `extra` rather than being + dropped. The kernel has only ever appended to this line, so a newer kernel adding one is the + expected way for this to change, and finding them in `extra` is how anybody would notice. + """ + lines = Lines() + body = text.strip() + if not body: + lines.count(SKIPPED) + return PidStat(source=source, path=path, promise=classify(path), lines=lines) + + for _ in text.splitlines()[1:]: + lines.count(SKIPPED) + + parts = split_comm(body) + if parts is None: + lines.count(UNPARSED) + return PidStat(source=source, path=path, promise=classify(path), lines=lines) + + head, comm, tail = parts + try: + pid = int(head.strip()) + except ValueError: + lines.count(UNPARSED) + return PidStat(source=source, path=path, promise=classify(path), lines=lines) + + rest = tail.split() + names = STAT_FIELDS[2:] + values = dict(zip(names, rest, strict=False)) + extra = tuple(rest[len(names) :]) + lines.count(READ) + return PidStat( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + pid=pid, + comm=comm, + values=values, + extra=extra, + naive=tuple(body.split()), + ) + + +def parse_file(path: Path | str, kernel_path: str = "") -> PidStat: + found = Path(path) + return parse(found.read_text(encoding="utf-8"), kernel_path, found.as_posix()) + + +def account(text: str) -> Lines: + return parse(text).lines + + +def trapped(stat: PidStat) -> bool: + """Whether the naive split would have got this line wrong. + + True when the command name contains a space or a closing bracket, which is the whole of the + trap. On almost every process on almost every machine this is False, and that is the reason + the wrong parse keeps shipping. + """ + return " " in stat.comm or ")" in stat.comm + + +def report(stat: PidStat) -> str: + lines = [ + stat.banner(), + f"pid: {stat.pid}", + f"comm: {stat.comm!r}", + f"state: {stat.state}", + f"fields: {len(stat.values)} named, {len(stat.extra)} beyond what proc.rst lists", + ] + if trapped(stat): + lines.append(f"naive: a whitespace split would call the state {stat.naive_state!r}") + else: + lines.append("naive: a whitespace split would have got this line right") + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/proc/stability.py b/kxray/proc/stability.py new file mode 100644 index 0000000..2a2e7f1 --- /dev/null +++ b/kxray/proc/stability.py @@ -0,0 +1,164 @@ +"""What the kernel promises about a file, before anything reads it. + + from kxray.proc import stability + + stability.classify("/proc/meminfo") undocumented + stability.classify("/sys/kernel/btf/vmlinux") testing (Documentation/ABI/testing/...) + print(stability.table()) + +Every reader in this package attaches one of these to what it returns. The reason is narrow and +worth stating plainly: this project teaches people to read files that mostly carry no promise at +all, and the difference between a file that is documented and a file that merely happens to work +is not visible from the file itself. + +The kernel keeps its own answer in `Documentation/ABI`, with one directory per level, and +`Documentation/ABI/README` defines them. `stable` will be kept working for at least two years and +in practice forever. `testing` may gain features but will not break under you. `obsolete` is on +its way out with a date attached. `removed` is a record of things that are gone. + +Then there is what is not in that tree. On Linux 7.2.2 there are 685 files under +`Documentation/ABI` and exactly six of them describe a path in `/proc`: `/proc/i8k`, +`/proc/diskstats`, `/proc/pid/smaps_rollup`, and the three `/proc/*/attr` files. Not +`/proc/meminfo`. Not `/proc/interrupts`. Not `/proc//stat`, which is the file every process +monitor ever written reads. Not `/proc//maps`. None of the files in this project's corpus. + +That is `undocumented`, and it is not the same as unstable. Those files have been the same shape +for many years and breaking them would break userspace, which is the one rule that does not bend. +What is missing is anybody having written down which part of the shape you may lean on. So a +reader may lean on them, and a reader should be told it is leaning on custom rather than on a +promise. + +One level is stronger than that, and this project reads two files that fall under it. The last +section of `Documentation/ABI/README` names, as "notable bits of non-ABI, which should not under +any circumstances be considered stable", both Kconfig, naming `/proc/config.gz` outright, and +kernel symbols, saying not to rely on "the presence, absence, location, or type of any kernel +symbol". `/proc/kallsyms` is exactly the second one. `kxray.kallsyms` reads it anyway, because +counting ops tables by name is a fine thing to do to a machine in front of you, and the ledger +now says out loud that the same code has no business inside a tool somebody deploys. + +The rules below are patterns rather than exact paths, matched in order, first match wins. Each +carries the file in the kernel tree that makes the claim, so none of this has to be believed. +""" + +from __future__ import annotations + +from fnmatch import fnmatch + +from kxray.models import ( + NOT_ABI, + OBSOLETE, + STABLE, + TESTING, + UNDOCUMENTED, + Promise, + grid, +) + +# How many files were in `Documentation/ABI` on the pinned kernel, and how many of them described +# a path in /proc. Checked against the 7.2.2 source tree, and worth checking again after a bump. +ABI_FILES = 685 +ABI_PROC_ENTRIES = 6 + +README = "Documentation/ABI/README" + +# Pattern, level, the file in the kernel tree that says so, and why in one sentence. Order +# matters. The specific paths come before the directory wildcards, and the two catch-alls are +# last. +RULES: tuple[tuple[str, str, str, str], ...] = ( + ( + "/proc/kallsyms", + NOT_ABI, + README, + "the README says not to rely on the presence, absence, location or type of any kernel " + "symbol, and this file is nothing but those", + ), + ( + "/proc/config.gz", + NOT_ABI, + README, + "the README names Kconfig as non-ABI and names this file while doing it", + ), + ( + "/proc/*/loginuid", + STABLE, + "Documentation/ABI/stable/procfs-audit_loginuid", + "the one file in /proc this project could read that carries a stable promise, kept here " + "as the counterexample", + ), + ( + "/proc/*/smaps_rollup", + TESTING, + "Documentation/ABI/testing/procfs-smaps_rollup", + "one of the six /proc paths the ABI tree describes at all", + ), + ( + "/proc/diskstats", + TESTING, + "Documentation/ABI/testing/procfs-diskstats", + "one of the six /proc paths the ABI tree describes at all", + ), + ( + "/sys/kernel/btf/*", + TESTING, + "Documentation/ABI/testing/sysfs-kernel-btf", + "documented since 5.5, and the reason a BTF dump is safe to build a lesson on", + ), + ( + "/sys/kernel/debug/tracing/*", + OBSOLETE, + "Documentation/ABI/obsolete/automount-tracefs-debugfs", + "the debugfs copy of tracefs, which that entry says should be gone by January 2030", + ), + ( + "/sys/kernel/tracing/*", + UNDOCUMENTED, + "", + "tracefs itself has no ABI entry, only the debugfs path it replaced does, so the " + "interface every tracing lesson here uses is described in Documentation/trace/ftrace.rst " + "and nowhere that carries a level", + ), + ( + "/proc/*", + UNDOCUMENTED, + "", + "no file under Documentation/ABI describes this path, which is true of nearly all of /proc", + ), + ( + "/sys/*", + UNDOCUMENTED, + "", + "no file under Documentation/ABI describes this path", + ), +) + + +def classify(path: str) -> Promise: + """What is promised about `path`, with the pattern that decided it. + + An unrecognised path comes back as `undocumented` with no pattern rather than raising. That is + the honest answer for a path nobody here has looked up, and it is also the safe one, because + the levels this returns are only ever used to decide how much to lean on something. + """ + for pattern, kind, entry, note in RULES: + if fnmatch(path, pattern): + return Promise(kind=kind, entry=entry, note=note, pattern=pattern) + return Promise(kind=UNDOCUMENTED, note="not looked up") + + +def dependable(path: str) -> bool: + return classify(path).dependable + + +def table() -> str: + """The whole ledger, for printing at the top of a lesson.""" + rows = [("path", "level", "written down in")] + for pattern, kind, entry, _ in RULES: + rows.append((pattern, kind, entry or "nothing")) + return grid(rows) + + +def explain(path: str) -> str: + """One paragraph on why `path` is at the level it is.""" + found = classify(path) + where = f", from {found.entry}" if found.entry else "" + return f"{path} is {found.kind}{where}: {found.note}" diff --git a/kxray/proc/version.py b/kxray/proc/version.py new file mode 100644 index 0000000..0006b29 --- /dev/null +++ b/kxray/proc/version.py @@ -0,0 +1,99 @@ +"""`/proc/version`, taken apart as far as it can honestly be taken apart. + + from kxray.proc import version + + banner = version.parse_file("corpora/proc/tier0/version.txt", "/proc/version") + print(banner.release, banner.parts, banner.at_least(6, 0)) + +One line, and everything that reads it reads it with a regex: + + Linux version 7.2.2 (kxbox@kxbox) (i686-linux-gnu-gcc (Debian 14.2.0-19) 14.2.0, + GNU ld (GNU Binutils for Debian) 2.44) #1 PREEMPT @0 + +which is one line in the file and is wrapped here to fit. + +The release is worth pulling out, because a lesson that says a thing is true of 6.1 and not 5.15 +has to be able to check. The build number after the hash is worth pulling out. The bit in the +middle is the user and host that built the kernel and then the entire compiler and linker banner, +which has brackets inside brackets in it, and there is no promise anywhere about its shape. So it +stays as text under `rest`, and anybody who wants the compiler out of it can decide for themselves +how much they trust what they find. + +`parts` is the release as a tuple of numbers, which is the only comparison that behaves. String +comparison says 6.9 is newer than 6.10 and it is not. Anything that is not a number ends the +tuple, so `6.1.0-13-amd64` gives `(6, 1, 0)` and a distribution's suffix does not turn into +nonsense. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +from kxray.models import READ, SKIPPED, UNPARSED, Lines, Version +from kxray.proc.stability import classify + +# `Linux version `. The word Linux is not assumed, because the same +# banner format is used by kernels that call themselves something else and the interesting field +# is the second one either way. +BANNER_RE = re.compile(r"^\S+ version (?P\S+)\s*(?P.*)$") + +# The build count and whatever the build appended to it, which on the pinned kernel is +# `#1 PREEMPT @0`. Its shape is set by the build and is not worth relying on beyond the number. +BUILD_RE = re.compile(r"(#\d+\S*)") + + +def parse(text: str, path: str = "", source: str = "") -> Version: + lines = Lines() + body = "" + for line in text.splitlines(): + if body or not line.strip(): + lines.count(SKIPPED) + continue + body = line + + found = BANNER_RE.match(body) if body else None + if found is None: + if body: + lines.count(UNPARSED) + return Version( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + text=body, + ) + + lines.count(READ) + build = BUILD_RE.search(found["rest"]) + return Version( + source=source, + path=path, + promise=classify(path) if path else classify(""), + lines=lines, + release=found["release"], + build=build.group(1) if build else "", + rest=found["rest"].strip(), + text=body, + ) + + +def parse_file(path: Path | str, kernel_path: str = "") -> Version: + found = Path(path) + return parse(found.read_text(encoding="utf-8"), kernel_path, found.as_posix()) + + +def account(text: str) -> Lines: + return parse(text).lines + + +def report(banner: Version) -> str: + lines = [ + banner.banner(), + f"release: {banner.release} {banner.parts}", + f"build: {banner.build or 'not printed'}", + f"rest: {banner.rest[:60]}{'...' if len(banner.rest) > 60 else ''}", + ] + text = "\n".join(lines) + print(text) + return text diff --git a/tests/test_proc.py b/tests/test_proc.py new file mode 100644 index 0000000..60011aa --- /dev/null +++ b/tests/test_proc.py @@ -0,0 +1,505 @@ +"""Tests for the /proc readers and for the stability ledger they all carry. + +The corpus tests read the eight committed captures and compare them against the numbers in their +own metadata, so a kernel bump that changes any of them fails here rather than in a lesson. The +rest are small strings, and most of them are about the ways one of these files stops being the +simple thing it looks like: a value that is four values, a line with a field missing off the end +of it, and a command name with a bracket in it. +""" + +import tomllib +from pathlib import Path + +import pytest + +from kxray import models, proc +from kxray.proc import keyed, maps, percpu, pidstat, stability, version + +ROOT = Path(__file__).resolve().parents[1] +TIER0 = ROOT / "corpora" / "proc" / "tier0" + +# The captures this package reads, with the file in /proc each one is a copy of. The lockdep and +# tracefs captures in the same directory belong to other readers and are not in here. +CAPTURES = { + "version.txt": "/proc/version", + "meminfo.txt": "/proc/meminfo", + "interrupts.txt": "/proc/interrupts", + "softirqs.txt": "/proc/softirqs", + "self-maps.txt": "/proc/self/maps", + "self-stat.txt": "/proc/self/stat", + "self-status.txt": "/proc/self/status", + "odd-comm-stat.txt": "/proc/self/stat", +} + + +def meta(name): + return tomllib.loads((TIER0 / name).with_suffix(".meta.toml").read_text()) + + +@pytest.fixture(scope="module") +def mem(): + return keyed.parse_file(TIER0 / "meminfo.txt", "/proc/meminfo") + + +@pytest.fixture(scope="module") +def status(): + return keyed.parse_file(TIER0 / "self-status.txt", "/proc/self/status") + + +@pytest.fixture(scope="module") +def irqs(): + return percpu.parse_file(TIER0 / "interrupts.txt", "/proc/interrupts") + + +@pytest.fixture(scope="module") +def softirqs(): + return percpu.parse_file(TIER0 / "softirqs.txt", "/proc/softirqs") + + +@pytest.fixture(scope="module") +def space(): + return maps.parse_file(TIER0 / "self-maps.txt", "/proc/self/maps") + + +@pytest.fixture(scope="module") +def ordinary(): + return pidstat.parse_file(TIER0 / "self-stat.txt", "/proc/self/stat") + + +@pytest.fixture(scope="module") +def odd(): + return pidstat.parse_file(TIER0 / "odd-comm-stat.txt", "/proc/self/stat") + + +# -- the committed captures -- + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_capture_reads_with_nothing_left_over(name): + found = proc.read(TIER0 / name, CAPTURES[name]) + assert found.lines.total == len((TIER0 / name).read_text().splitlines()) + assert found.lines.unparsed == meta(name)["unparsed_lines"] + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_capture_says_which_file_it_is_a_copy_of(name): + assert meta(name)["path"] == CAPTURES[name] + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_capture_records_the_level_the_ledger_gives_it(name): + assert meta(name)["stability"] == stability.classify(CAPTURES[name]).kind + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_capture_is_evidence_off_the_pinned_kernel(name): + got = meta(name) + assert got["evidence"] is True + assert got["kernel"] == "7.2.2" + assert got["arch"] == "i386" + + +def test_the_router_sends_each_capture_to_a_reader(): + for name, kernel_path in CAPTURES.items(): + assert proc.reader_for(kernel_path) is not None, name + + +def test_the_router_would_rather_say_nothing_than_guess(): + assert proc.reader_for("/proc/schedstat") is None + with pytest.raises(LookupError): + proc.read(TIER0 / "meminfo.txt", "/proc/schedstat") + + +# -- what the ledger says -- + + +def test_nothing_in_this_corpus_is_documented_anywhere(): + # The claim the whole package is built on. If a kernel release ever writes an ABI entry for one + # of these, this test is where anybody finds out, and that would be good news. + for kernel_path in CAPTURES.values(): + assert stability.classify(kernel_path).kind == models.UNDOCUMENTED + + +def test_undocumented_is_not_the_same_as_unusable(): + found = stability.classify("/proc/meminfo") + assert found.documented is False + assert found.dependable is False + assert "Documentation/ABI describes this path" in found.note + + +def test_the_two_files_the_readme_names_as_non_abi(): + for kernel_path in ("/proc/kallsyms", "/proc/config.gz"): + found = stability.classify(kernel_path) + assert found.kind == models.NOT_ABI + assert found.entry == "Documentation/ABI/README" + + +def test_btf_is_the_one_thing_here_that_carries_a_promise(): + found = stability.classify("/sys/kernel/btf/vmlinux") + assert found.kind == models.TESTING + assert found.dependable is True + + +def test_tracefs_is_undocumented_and_its_debugfs_copy_is_obsolete(): + assert stability.classify("/sys/kernel/tracing/trace").kind == models.UNDOCUMENTED + assert stability.classify("/sys/kernel/debug/tracing/trace").kind == models.OBSOLETE + + +def test_the_specific_rules_come_before_the_wildcards(): + # /proc/kallsyms matches both its own rule and the /proc/* catch-all, and order is the only + # thing that decides which one answers. + assert stability.classify("/proc/kallsyms").pattern == "/proc/kallsyms" + assert stability.classify("/proc/meminfo").pattern == "/proc/*" + + +def test_a_path_nobody_looked_up_is_undocumented_rather_than_an_error(): + found = stability.classify("/etc/passwd") + assert found.kind == models.UNDOCUMENTED + assert found.pattern == "" + + +def test_every_level_in_the_ledger_is_one_of_the_six(): + for _, kind, _, _ in stability.RULES: + assert kind in models.LEVELS + + +def test_every_documented_rule_names_the_file_that_says_so(): + for pattern, kind, entry, note in stability.RULES: + assert note, pattern + if kind in (models.STABLE, models.TESTING, models.OBSOLETE, models.NOT_ABI): + assert entry.startswith("Documentation/ABI/"), pattern + + +def test_the_ledger_prints_as_a_table(): + printed = stability.table() + assert "written down in" in printed + assert "nothing" in printed + + +# -- key and value files -- + + +def test_meminfo_reads_every_line(mem): + assert len(mem.entries) == meta("meminfo.txt")["keys"] + assert mem.lines.unparsed == 0 + + +def test_the_kernel_writes_kb_and_means_kib(mem): + assert mem["MemTotal"].unit == "kB" + assert mem.number("MemTotal") == meta("meminfo.txt")["mem_total_kb"] + assert keyed.bytes_of(mem, "MemTotal") == meta("meminfo.txt")["mem_total_bytes"] + + +def test_a_missing_key_raises_and_says_which_kernel(mem): + with pytest.raises(KeyError): + mem["HugePages_Total"] + assert mem.number("HugePages_Total") is None + assert "HugePages_Total" not in mem + + +def test_a_status_file_has_values_that_are_not_one_number(status): + got = meta("self-status.txt") + assert len(status["Uid"].values) == got["uid_values"] + assert len(status["State"].values) == got["state_values"] + assert status["Uid"].number is None + assert status["State"].number is None + + +def test_a_key_with_nothing_after_it_is_still_a_key(status): + for key in meta("self-status.txt")["empty_keys"]: + assert key in status + assert status[key].values == () + assert status[key].number is None + + +def test_order_is_kept_because_the_kernel_groups_related_keys(status): + keys = list(status.keys) + assert keys.index("VmSize") < keys.index("VmRSS") < keys.index("Threads") + + +def test_a_line_with_no_colon_is_unparsed(): + found = keyed.parse("MemTotal: 8 kB\nthis is not a key\n") + assert found.lines.read == 1 + assert found.lines.unparsed == 1 + assert found.lines.total == 2 + + +def test_a_hex_value_still_comes_back_as_a_number(): + found = keyed.parse("untag_mask:\t0xffffffff\n") + assert found.number("untag_mask") == 0xFFFFFFFF + + +def test_bytes_of_leaves_a_unitless_number_alone(): + found = keyed.parse("Threads:\t4\n") + assert keyed.bytes_of(found, "Threads") == 4 + assert keyed.bytes_of(found, "Nothing") is None + + +# -- per cpu counter files -- + + +def test_the_column_count_comes_off_the_header(irqs, softirqs): + assert irqs.cpus == tuple(meta("interrupts.txt")["cpus"]) + assert softirqs.cpus == tuple(meta("softirqs.txt")["cpus"]) + assert irqs.cpu_count == 1 + + +def test_interrupts_splits_into_hardware_lines_and_kernel_lines(irqs): + got = meta("interrupts.txt") + assert len(irqs.counters) == got["rows"] + assert len(percpu.hardware(irqs)) == got["hardware_rows"] + assert [one.label for one in percpu.named(irqs)] == got["named_rows"] + + +def test_a_hardware_line_keeps_the_device_on_the_end_of_it(irqs): + timer = irqs.get("0") + assert timer is not None + assert "XT-PIC" in timer.detail + assert timer.total > 0 + + +def test_the_deferred_work_is_mostly_timers_and_rcu(softirqs): + got = meta("softirqs.txt") + assert len(softirqs.counters) == got["rows"] + assert len(softirqs.quiet()) == got["quiet_rows"] + assert softirqs.total("RCU") == got["rcu"] + assert softirqs.total("TIMER") == got["timer"] + + +def test_softirq_rows_have_no_description_because_the_kernel_prints_none(softirqs): + assert all(one.detail == "" for one in softirqs.counters) + + +def test_the_column_count_decides_where_the_numbers_stop(): + # The same row read as one CPU and as two. With two columns the word `Non-maskable` is asked to + # be a number and the row does not parse, which is the failure worth having rather than a row + # that quietly keeps half its counts. + row = " NMI: 7 Non-maskable interrupts" + one = percpu.parse(" CPU0\n" + row + "\n") + assert one.get("NMI").counts == (7,) + assert one.get("NMI").detail == "Non-maskable interrupts" + two = percpu.parse(" CPU0 CPU1\n" + row + "\n") + assert two.lines.unparsed == 1 + + +def test_a_counter_file_with_no_header_reads_nothing(): + found = percpu.parse(" 0: 12 XT-PIC timer\n") + assert found.cpus == () + assert found.lines.unparsed == 1 + + +def test_counts_add_up_across_cpus(): + found = percpu.parse(" CPU0 CPU1\n RCU: 4 6\n") + assert found.total("RCU") == 10 + assert found.get("RCU").on(1) == 6 + assert found.get("RCU").fired is True + + +# -- an address space -- + + +def test_the_capture_has_the_regions_its_metadata_claims(space): + got = meta("self-maps.txt") + assert len(space.regions) == got["regions"] + assert [one.path for one in space.named()] == got["named_regions"] + assert len(space.gaps()) == got["gaps"] + assert space.total_size == got["total_bytes"] + + +def test_the_program_is_mapped_twice_with_different_permissions(space): + both = space.find("busybox") + assert len(both) == 2 + assert both[0].executable and not both[0].writable + assert both[1].writable and not both[1].executable + + +def test_the_anonymous_region_is_the_one_with_no_name(space): + anonymous = [one for one in space.regions if one.anonymous] + assert len(anonymous) == meta("self-maps.txt")["anonymous_regions"] + assert anonymous[0].label == "anonymous" + assert anonymous[0].special is False + + +def test_a_line_with_no_path_still_has_all_its_fields(): + # The trap, stated as a test. The kernel pads to a fixed column and prints nothing, so the line + # ends in a space and a whitespace split comes back one field short. + line = "b7f8f000-b7f9f000 rw-p 00000000 00:00 0 " + assert len(maps.naive_fields(line)) == 5 + found = maps.parse(line + "\n") + assert found.lines.read == 1 + assert found.regions[0].path == "" + + +def test_a_line_with_a_path_has_six_fields_and_looks_identical(): + line = "08048000-08149000 r-xp 00000000 00:03 11 /bin/busybox" + assert len(maps.naive_fields(line)) == 6 + assert maps.parse(line + "\n").regions[0].path == "/bin/busybox" + + +def test_a_filename_with_a_space_in_it_is_kept_whole(): + line = "08048000-08149000 r-xp 00000000 00:03 11 /tmp/od) d ma" + assert maps.parse(line + "\n").regions[0].path == "/tmp/od) d ma" + + +def test_an_address_is_either_in_a_region_or_in_a_hole(space): + text = space.regions[0] + assert space.at(text.start) is text + assert space.at(text.end - 1) is text + assert space.at(text.end) is not text + assert space.at(0x50000000) is None + + +def test_most_of_the_address_space_is_gap(space): + biggest = max(size for _, size in space.gaps()) + assert biggest > space.total_size * 100 + + +def test_sizes_come_out_in_whole_pages(space): + for one in space.regions: + assert one.size % maps.PAGE == 0 + assert one.pages == one.size // maps.PAGE + + +def test_the_table_names_every_region(space): + printed = space.table() + for one in space.regions: + assert one.label in printed + + +# -- the one line file -- + + +def test_the_ordinary_case_reads_the_way_anybody_would_expect(ordinary): + got = meta("self-stat.txt") + assert ordinary.pid == got["pid"] + assert ordinary.comm == got["comm"] + assert ordinary.state == got["state"] + assert len(ordinary.values) == got["fields"] + assert ordinary.extra == () + assert pidstat.trapped(ordinary) is got["trapped_by_naive_split"] + + +def test_the_naive_split_agrees_on_the_ordinary_case(ordinary): + # Which is exactly why the wrong parse survives. It is right almost always. + assert ordinary.naive_state == ordinary.state + + +def test_the_bracket_in_a_command_name_moves_every_field_after_it(odd): + got = meta("odd-comm-stat.txt") + assert odd.comm == got["comm"] + assert odd.state == got["state"] + assert odd.naive_state == got["naive_state"] + assert odd.state != odd.naive_state + assert len(odd.naive) == got["naive_fields"] + assert pidstat.trapped(odd) is True + + +def test_both_captures_have_the_same_fields_despite_the_names(ordinary, odd): + assert len(ordinary.values) == len(odd.values) == 50 + assert set(ordinary.values) == set(odd.values) + + +def test_the_field_names_are_the_ones_the_documentation_lists(): + assert len(models.STAT_FIELDS) == 52 + assert models.STAT_FIELDS[:3] == ("pid", "tcomm", "state") + assert models.STAT_FIELDS[-1] == "exit_code" + + +def test_the_mappings_and_the_one_line_file_agree_on_the_size(space, ordinary): + # vsize is the sum of the sizes of the mappings, so two files taken from two runs of the same + # program are two views of one fact. If either reader drifts, they stop agreeing. + assert ordinary.number("vsize") == space.total_size + + +def test_a_field_the_kernel_added_lands_in_extra(): + # Fifty named fields after the command, and then one more that no kernel prints yet. + line = "1 (init) " + " ".join(["S", *["0"] * 49, "88"]) + found = pidstat.parse(line) + assert found.comm == "init" + assert len(found.values) == 50 + assert found.extra == ("88",) + + +def test_a_line_with_no_bracket_is_unparsed(): + found = pidstat.parse("37 cat R 1 0\n") + assert found.lines.unparsed == 1 + assert found.pid == 0 + + +def test_an_empty_file_is_skipped_rather_than_failed(): + found = pidstat.parse("") + assert found.lines.skipped == 1 + assert found.lines.unparsed == 0 + + +def test_split_comm_takes_the_first_bracket_and_the_last(): + assert pidstat.split_comm("7 (a) b) R 1") == ("7 ", "a) b", " R 1") + assert pidstat.split_comm("no brackets here") is None + + +def test_the_faults_pair_is_the_one_the_page_fault_blueprint_counts(ordinary): + minor, major = ordinary.faults + assert minor > 0 + assert major == 0 + + +# -- the banner -- + + +def test_the_release_comes_out_and_the_compiler_stays_text(): + got = meta("version.txt") + banner = version.parse_file(TIER0 / "version.txt", "/proc/version") + assert banner.release == got["release"] + assert list(banner.parts) == got["parts"] + assert banner.build == got["build"] + assert "gcc" in banner.rest + + +def test_the_running_kernel_is_the_one_the_profile_asked_for(): + banner = version.parse_file(TIER0 / "version.txt", "/proc/version") + assert "PREEMPT" in banner.rest + + +def test_versions_compare_as_numbers_and_not_as_strings(): + older = version.parse("Linux version 6.9.0 (a@b) (gcc) #1\n") + newer = version.parse("Linux version 6.10.0 (a@b) (gcc) #1\n") + assert newer.parts > older.parts + assert newer.release < older.release + assert newer.at_least(6, 10) is True + assert older.at_least(6, 10) is False + + +def test_a_distribution_suffix_stops_the_tuple_rather_than_breaking_it(): + banner = version.parse("Linux version 6.1.0-13-amd64 (a@b) (gcc) #1 SMP\n") + assert banner.parts == (6, 1, 0) + + +def test_a_banner_that_is_not_one_is_unparsed(): + banner = version.parse("something else entirely\n") + assert banner.lines.unparsed == 1 + assert banner.release == "" + assert banner.parts == () + + +# -- what every reader has in common -- + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_reader_prints_a_banner_naming_the_file_and_the_level(name): + found = proc.read(TIER0 / name, CAPTURES[name]) + printed = found.banner() + assert CAPTURES[name] in printed + assert models.UNDOCUMENTED in printed + + +@pytest.mark.parametrize("name", sorted(CAPTURES)) +def test_every_reader_accounts_for_every_line(name): + text = (TIER0 / name).read_text() + module = proc.reader_for(CAPTURES[name]) + assert module.account(text).total == len(text.splitlines()) + + +def test_a_reader_given_no_path_still_works_and_says_it_looked_nothing_up(): + found = keyed.parse("MemTotal: 8 kB\n") + assert found.promise.kind == models.UNDOCUMENTED + assert found.promise.pattern == "" diff --git a/tools/baseline.py b/tools/baseline.py index 04b9ea1..bc39727 100644 --- a/tools/baseline.py +++ b/tools/baseline.py @@ -45,6 +45,11 @@ from kxray import kallsyms, lockdep, tracefs from kxray.btf import reader as btf from kxray.models import Lines +from kxray.proc import keyed as proc_keyed +from kxray.proc import maps as proc_maps +from kxray.proc import percpu as proc_percpu +from kxray.proc import pidstat as proc_pidstat +from kxray.proc import version as proc_version from kxray.trace import events, formats, parse_file from kxray.trace import function as trace_function @@ -68,6 +73,16 @@ ("corpora/proc/*/lockdep_stats.txt", "lockdep-stats"), ("corpora/proc/*/lockdep-stats-*.txt", "lockdep-stats"), ("corpora/proc/*/ring-overrun.txt", "tracefs-stats"), + # The rest of /proc, routed by what the file is rather than by what it is called, which is why + # `self-status.txt` and `meminfo.txt` land on the same reader and `self-stat.txt` does not. The + # lockdep patterns above have to stay in front of the `*-stat.txt` one. + ("corpora/proc/*/version.txt", "proc-version"), + ("corpora/proc/*/meminfo.txt", "proc-keyed"), + ("corpora/proc/*/self-status.txt", "proc-keyed"), + ("corpora/proc/*/interrupts.txt", "proc-percpu"), + ("corpora/proc/*/softirqs.txt", "proc-percpu"), + ("corpora/proc/*/self-maps.txt", "proc-maps"), + ("corpora/proc/*/*-stat.txt", "proc-pidstat"), ("corpora/oops/*/*.txt", "lockdep-splat"), ("corpora/btf/*/*.btf", "btf"), ("corpora/experiments/*/*.txt", "none"), @@ -123,6 +138,47 @@ def _event_format(path: Path) -> tuple[int, Lines | None]: return len(formats.parse_file(path).fields), formats.account(path.read_text(encoding="utf-8")) +def _kernel_path(path: Path) -> str: + """Which file in /proc this artefact is a copy of, from its own metadata. + + The readers need it, because what a file is called on disk does not decide how it is read or + what it is worth. `self-maps.txt` is `/proc/self/maps`, and only the second of those two names + reaches the stability ledger. + """ + meta = path.with_suffix(".meta.toml") + if not meta.exists(): + return "" + return str(tomllib.loads(meta.read_text(encoding="utf-8")).get("path", "")) + + +def _proc_keyed(path: Path) -> tuple[int, Lines | None]: + found = proc_keyed.parse_file(path, _kernel_path(path)) + return len(found.entries), found.lines + + +def _proc_percpu(path: Path) -> tuple[int, Lines | None]: + found = proc_percpu.parse_file(path, _kernel_path(path)) + return len(found.counters), found.lines + + +def _proc_maps(path: Path) -> tuple[int, Lines | None]: + found = proc_maps.parse_file(path, _kernel_path(path)) + return len(found.regions), found.lines + + +def _proc_pidstat(path: Path) -> tuple[int, Lines | None]: + found = proc_pidstat.parse_file(path, _kernel_path(path)) + # The named fields rather than one, because one line that read is not the number that would + # move. A kernel that appends a field puts it in `extra`, and counting the named ones plus the + # extras is how that shows up here at all. + return len(found.values) + len(found.extra), found.lines + + +def _proc_version(path: Path) -> tuple[int, Lines | None]: + found = proc_version.parse_file(path, _kernel_path(path)) + return len(found.parts), found.lines + + def _kallsyms(path: Path) -> tuple[int, Lines | None]: text = path.read_text(encoding="utf-8") return len(kallsyms.parse(text)), kallsyms.account(text) @@ -168,6 +224,11 @@ def _unread(path: Path) -> tuple[int, Lines | None]: "function": _function_flat, "events": _events, "event-format": _event_format, + "proc-keyed": _proc_keyed, + "proc-percpu": _proc_percpu, + "proc-maps": _proc_maps, + "proc-pidstat": _proc_pidstat, + "proc-version": _proc_version, "kallsyms": _kallsyms, "lockdep-classes": _lockdep_classes, "lockdep-stats": _lockdep_stats,