From f70e14c6a4bcfdf128eb2ef80e1d30dd2d7b231c Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:59:20 +0700 Subject: [PATCH] Read the kernel's own source files Adds kxray.source: six modules for the text files a kernel ships with, and the context hash that stops a citation going quietly stale. Grounded on five real files out of the pinned 7.2.2 tarball, committed under corpora/source/pinned so the parsers have something to run against in CI and in a notebook that has downloaded nothing. --- LAYOUT.md | 12 +- blueprints/page-fault.refs.toml | 28 + blueprints/write-path.refs.toml | 37 + corpora/BASELINE.toml | 59 +- corpora/README.md | 9 + corpora/proc/tier0/kallsyms-write.meta.toml | 41 + corpora/proc/tier0/kallsyms-write.txt | 8 + corpora/source/pinned/MAINTAINERS.excerpt | 296 +++ corpora/source/pinned/MAINTAINERS.meta.toml | 44 + corpora/source/pinned/README.md | 74 + .../x86/entry/syscalls/syscall_32.meta.toml | 26 + .../arch/x86/entry/syscalls/syscall_32.tbl | 479 +++++ .../x86/entry/syscalls/syscall_64.meta.toml | 26 + .../arch/x86/entry/syscalls/syscall_64.tbl | 443 ++++ corpora/source/pinned/fs/read_write.c | 1822 +++++++++++++++++ corpora/source/pinned/fs/read_write.meta.toml | 32 + .../source/pinned/kernel/Kconfig.meta.toml | 25 + corpora/source/pinned/kernel/Kconfig.preempt | 194 ++ kxray/source/__init__.py | 47 + kxray/source/citations.py | 148 ++ kxray/source/kconfig.py | 362 ++++ kxray/source/maintainers.py | 393 ++++ kxray/source/symbols.py | 195 ++ kxray/source/syscalls.py | 239 +++ kxray/source/tree.py | 172 ++ lessons/C09/refs.toml | 3 + lessons/S05/refs.toml | 3 + lessons/Z02/refs.toml | 2 + tests/test_refcheck.py | 100 +- tests/test_source.py | 493 +++++ tools/baseline.py | 29 + tools/refcheck.py | 104 +- 32 files changed, 5910 insertions(+), 35 deletions(-) create mode 100644 corpora/proc/tier0/kallsyms-write.meta.toml create mode 100644 corpora/proc/tier0/kallsyms-write.txt create mode 100644 corpora/source/pinned/MAINTAINERS.excerpt create mode 100644 corpora/source/pinned/MAINTAINERS.meta.toml create mode 100644 corpora/source/pinned/README.md create mode 100644 corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.meta.toml create mode 100644 corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.tbl create mode 100644 corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.meta.toml create mode 100644 corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.tbl create mode 100644 corpora/source/pinned/fs/read_write.c create mode 100644 corpora/source/pinned/fs/read_write.meta.toml create mode 100644 corpora/source/pinned/kernel/Kconfig.meta.toml create mode 100644 corpora/source/pinned/kernel/Kconfig.preempt create mode 100644 kxray/source/__init__.py create mode 100644 kxray/source/citations.py create mode 100644 kxray/source/kconfig.py create mode 100644 kxray/source/maintainers.py create mode 100644 kxray/source/symbols.py create mode 100644 kxray/source/syscalls.py create mode 100644 kxray/source/tree.py create mode 100644 tests/test_source.py diff --git a/LAYOUT.md b/LAYOUT.md index 8f665c4..e04e79a 100644 --- a/LAYOUT.md +++ b/LAYOUT.md @@ -8,7 +8,7 @@ linux-kernel-internals/ │ ├── btf/ # BTF reader: types, fields, offsets, holes, type tags │ ├── trace/ # ftrace function_graph, function and trace_event parsers │ ├── proc/ # /proc and /sys snapshot parsers, and the ABI stability ledger -│ ├── source/ # kernel tree navigation, Kconfig, MAINTAINERS +│ ├── source/ # the kernel's own files: MAINTAINERS, Kconfig, syscall tables │ ├── models/ # the shared model everything else renders │ ├── replay/ # recorded Tier 1 session playback │ └── corpus/ # pinned artefacts and the diff normaliser @@ -81,6 +81,16 @@ Three of the five ride along in BTF as a `type_tag` record. `__iomem` does not, `kxray/proc/pidstat.py` is a file of its own for one reason, and the reason is a real capture. The kernel prints the command name in brackets and does not escape it, so a process whose executable is called `od) d ma` prints as `37 (od) d ma) R 1 0 ...`, and a whitespace split puts the state two fields to the left of where it belongs and reports a process as being in a state that does not exist. `corpora/proc/tier0/odd-comm-stat.txt` is that line off the pinned kernel. The parser takes the first opening bracket and the last closing bracket, which is what `procps` has done for decades, and keeps what the naive split would have said so that a lesson can show both answers rather than assert that the trap is real. +`kxray/source/` reads the kernel's own text files, which is the half of this toolkit that never touches a running machine. `tree.py` is the handle, and the first thing it answers is how much kernel is actually here: the full 1.6 GB that `./kxbox/kernel/tree.sh` unpacks, or the five files committed under `corpora/source/pinned/`, or nothing at all. That distinction is load bearing rather than tidy. On the partial tree a lookup that finds nothing has to say the file is not in the corpus, never that it is not in the kernel, and those are different sentences with only one of them true. + +`kxray/source/maintainers.py` is the one with a trap in it worth stating on its own. The `F:` patterns in that file look like shell globs and are not, because the kernel's own header block says a single star stops at a slash. `fnmatch.fnmatch("fs/proc/base.c", "fs/*")` is True, the answer the file gives is False, and FILESYSTEMS (VFS and infrastructure) carries exactly that pattern. So a tool built on `fnmatch` mails every patch under `fs/` to the VFS maintainers, and `fs/proc/` has its own section that said so by writing one star instead of a trailing slash. The `K:` tag has a second one: `scripts/get_maintainer.pl` applies those regexes with perl's `/x`, which is why the audit pattern `\baudit_[a-z_0-9]\+\b` contains a literal plus sign and does not match `audit_log_start`. Both are confirmed against the script at lines 575 and 622 rather than reasoned about. + +`kxray/source/syscalls.py` reads both `.tbl` files so that write being 4 on i386 and 1 on x86-64 can be a table rather than a claim. The pinned box is 32 bit and the reader's laptop is not, so every syscall number a lesson prints comes from a different table than the one the reader knows. `number_of` returns None when a name appears under two abis rather than picking one, because `rt_sigaction` is 13 under `64` and 512 under `x32` and quietly returning the first is how a tool gets it wrong on the machine where it matters. + +`kxray/source/kconfig.py` reads the kernel's Kconfig files, which is the other half of `tools/kconfig.py`: not what this project asked for but what the kernel does about it. It answers one question, which is why a symbol is on. `CONFIG_PREEMPT=y` gives a `.config` with `CONFIG_PREEMPT_BUILD=y` and `CONFIG_PREEMPTION=y` in it that nobody chose, and six of the fifteen symbols in `kernel/Kconfig.preempt` have no prompt at all, so they cannot be set by hand and in a `.config` they look exactly like something a person picked. What it does not do is evaluate a condition, because doing that properly means being `scripts/kconfig`. + +`kxray/source/citations.py` is what `tools/refcheck` hashes with. A citation here is anchored on text and never on a line number, and that already survived files moving. What it did not survive is the anchor holding still while the code under it changes, which is the common case, because an anchor is usually a signature and the body underneath is the part people edit. So a confirmed citation carries a hash of the seven lines around its anchor, taken over whitespace normalised text, so that reindentation and tab churn do not fire and a renamed variable does. All 73 citations in this repository carry one. + `kxray/layout.py` is the arithmetic that turns a tree of frames into rectangles. It is in `kxray` for the same reason. A widget and an animation of the same trace call it and get the same answer, so the wide box is in the same place in both. `kxshapes/` is the next step up from that. It is the nine shapes every picture in this book is built out of, held as plain data rather than as drawing: a frame card, a layer band, an object box, a pointer thread, an ops plug, a trace cell, a CPU lane, a context badge and a memory slot. A test asserts there are exactly nine, because a closed set is the point. Each shape works out its own rows, its own labels and its own alt text, and neither renderer is allowed to work any of that out again. It is a package of its own rather than a module inside either renderer, and that is the whole reason it exists. If the arithmetic lived in `kxwidgets` then `kxmanim` would have to redo it, and two renderers doing their own arithmetic are two renderers that can disagree, in the worst possible way, which is that both pictures look fine and one of them is wrong. diff --git a/blueprints/page-fault.refs.toml b/blueprints/page-fault.refs.toml index 2524b30..e569038 100644 --- a/blueprints/page-fault.refs.toml +++ b/blueprints/page-fault.refs.toml @@ -32,6 +32,7 @@ anchor = "DEFINE_IDTENTRY_RAW_ERRORCODE(exc_page_fault)" kernel = "7.2.2" confirmed = true line = 1492 +context = "8a120ab634bd" note = "The architecture entry point. Everything in this blueprint starts here on x86-64, and the macro is what wires it to vector 14 in the interrupt descriptor table." [[references]] @@ -41,6 +42,7 @@ anchor = "enum x86_pf_error_code" kernel = "7.2.2" confirmed = true line = 20 +context = "d843617f4149" note = "The bits the hardware puts in the error code. Present, write, user, reserved, instruction fetch, protection key and SGX. Everything the handler decides in its first few branches comes out of this word." [[references]] @@ -50,6 +52,7 @@ anchor = "bool fault_in_kernel_space(unsigned long address)" kernel = "7.2.2" confirmed = true line = 1124 +context = "10397ec59d21" note = "The split between a kernel address and a user address, which is the first fork in the road and the one that decides whether any of the rest of this applies." [[references]] @@ -59,6 +62,7 @@ anchor = "do_user_addr_fault(struct pt_regs *regs," kernel = "7.2.2" confirmed = true line = 1216 +context = "47152630a3dd" note = "The user address path. Finds the VMA, checks permissions against it, and calls into the architecture independent code." [[references]] @@ -68,6 +72,7 @@ anchor = "struct vm_area_struct *lock_vma_under_rcu(struct mm_struct *mm," kernel = "7.2.2" confirmed = true line = 296 +context = "3324c917d071" note = "The per VMA lock lookup, tried before the mmap lock. This is the fast path that lets faults on different VMAs in the same process run at the same time. It lives in mm/mmap_lock.c, not in mm/memory.c where most write ups still put it." [[references]] @@ -77,6 +82,7 @@ anchor = "need to extend the vma, which helps the VM layer a lot." kernel = "7.2.2" confirmed = true line = 494 +context = "7018912fe88b" note = "The slow path, taking the mmap lock for read and looking the VMA up in the tree. Also where a stack that needs growing gets grown. The anchor is the last line of the comment above lock_mm_and_find_vma rather than the function itself, because this file defines the function twice, once for CONFIG_MMU and once for the architectures without it, and the two first lines are word for word the same." [[references]] @@ -86,6 +92,7 @@ anchor = "struct vm_area_struct {" kernel = "7.2.2" confirmed = true line = 920 +context = "71116c302bff" note = "One mapped range in one address space. Start, end, flags, the file it is backed by if any, and the operations table." [[references]] @@ -95,6 +102,7 @@ anchor = "struct mm_struct {" kernel = "7.2.2" confirmed = true line = 1160 +context = "d3a4bccee54d" note = "The address space itself. Holds the page table root, the VMA tree, the mmap lock and the fault counters this blueprint says move." [[references]] @@ -104,6 +112,7 @@ anchor = "struct vm_fault {" kernel = "7.2.2" confirmed = true line = 730 +context = "a1bc13d968e1" note = "The working state of one fault, passed down every level of the handler. The address, the flags, the VMA, and whichever page table entry the walk has reached." [[references]] @@ -113,6 +122,7 @@ anchor = "@FAULT_FLAG_ALLOW_RETRY: Allow to retry the fault if blocked." kernel = "7.2.2" confirmed = true line = 1760 +context = "43a4c2c06510" note = "The flag that lets the handler drop the mmap lock and start over rather than sleeping while holding it. The retry protocol in section 3 is built on this one bit." [[references]] @@ -122,6 +132,7 @@ anchor = "vm_fault_t handle_mm_fault" kernel = "7.2.2" confirmed = true line = 6651 +context = "ecdad5c556d7" note = "The architecture independent entry point. Every architecture funnels into this, which is why the blueprint splits at exactly this line." [[references]] @@ -131,6 +142,7 @@ anchor = "static vm_fault_t __handle_mm_fault(struct vm_area_struct *vma," kernel = "7.2.2" confirmed = true line = 6417 +context = "b958e5ee976c" note = "The page table walk. Allocates each level that is missing on the way down, from the top level entry to the pmd." [[references]] @@ -140,6 +152,7 @@ anchor = "static vm_fault_t handle_pte_fault(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 6335 +context = "52e48429d069" note = "The dispatch. Looks at the pte and picks anonymous, file backed, swap, numa or write protect from what it finds." [[references]] @@ -149,6 +162,7 @@ anchor = "Use the zero-page for reads" kernel = "7.2.2" confirmed = true line = 5307 +context = "c8da8d82366d" note = "A fault on a mapping with nothing behind it. Allocates a folio, or maps the shared zero page when the access is a read. The anchor is the comment on the zero page branch inside do_anonymous_page, because the forward declaration near the top of the file is character for character the same as the definition and an anchor cannot tell them apart." [[references]] @@ -158,6 +172,7 @@ anchor = "vm_fault_t do_swap_page(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 4747 +context = "20447b7a0ecb" note = "A fault on a page that was swapped out. This is the path that turns into a major fault and the one that sleeps on real hardware." [[references]] @@ -167,6 +182,7 @@ anchor = "static vm_fault_t do_cow_fault(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 5872 +context = "fc02aa08ebef" note = "A write to a private file backed mapping. Reads the original, copies it, and maps the copy, which is where a private mapping stops sharing with the file." [[references]] @@ -176,6 +192,7 @@ anchor = "vm_fault_t finish_fault(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 5617 +context = "ea2c4ad842fb" note = "Where a file backed fault installs its pte, under the pte lock, after checking nothing changed while the lock was dropped." [[references]] @@ -185,6 +202,7 @@ anchor = "vm_fault_t filemap_fault(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 3546 +context = "7fc4b42b47eb" note = "The default fault operation for a file backed mapping. Looks in the page cache, and starts readahead or a read when the folio is not there." [[references]] @@ -194,6 +212,7 @@ anchor = "struct vm_operations_struct {" kernel = "7.2.2" confirmed = true line = 783 +context = "0751587ad1d7" note = "The operations a mapping can override. The fault slot is the one that matters here, and a mapping that leaves it empty is anonymous by definition." [[references]] @@ -203,6 +222,7 @@ anchor = "enum vm_fault_reason {" kernel = "7.2.2" confirmed = true line = 1681 +context = "60e794995a66" note = "The whole set of vm_fault_t values, including VM_FAULT_RETRY, which means nothing was fixed and the caller has to fault again. The anchor is the head of the enum rather than the one value, because the values are laid out with tabs and an anchor with a tab in it is a thing nobody can retype." [[references]] @@ -212,6 +232,7 @@ anchor = "unable to handle page fault for address" kernel = "7.2.2" confirmed = true line = 545 +context = "b2b4fd0bd33d" note = "The oops header for a fault in kernel mode that no fixup handles. This is the string a person searches for at three in the morning, which is why it is in the blueprint by its exact text." [[references]] @@ -221,6 +242,7 @@ anchor = "BUG: Bad page map in process" kernel = "7.2.2" confirmed = true line = 599 +context = "646c4e33c524" note = "What print_bad_pte reports when a pte points at something that is not a valid page. A corrupt page table found from inside the fault path prints this rather than crashing at the access." [[references]] @@ -230,6 +252,7 @@ anchor = "kernelmode_fixup_or_oops(struct pt_regs *regs, unsigned long error_cod kernel = "7.2.2" confirmed = true line = 728 +context = "5e5a6b05bb61" note = "The exception table lookup. A fault inside copy_from_user is expected and gets fixed up here, and one anywhere else becomes the oops above." [[references]] @@ -239,6 +262,7 @@ anchor = "DEFINE_EVENT(exceptions, page_fault_user," kernel = "7.2.2" confirmed = true line = 33 +context = "d9e3139d801d" note = "The tracepoints the architecture entry point fires, one for a user fault and one for a kernel fault. Section 5 lists these as the first thing an observer sees. This used to be written down as arch/x86/include/asm/trace/exceptions.h, which is where it lived for years and is not where it lives now. Confirming the citations is what caught that." [[references]] @@ -248,6 +272,7 @@ anchor = "vm_fault_t do_huge_pmd_anonymous_page(struct vm_fault *vmf)" kernel = "7.2.2" confirmed = true line = 1519 +context = "b7dbe02db325" note = "The transparent huge page path, taken from the pmd level before the walk ever reaches a pte. A build without huge pages never gets here." [[references]] @@ -257,6 +282,7 @@ anchor = "bool out_of_memory(struct oom_control *oc)" kernel = "7.2.2" confirmed = true line = 1103 +context = "9ed38fa6c83b" note = "Where a fault that could not allocate memory ends up, by way of VM_FAULT_OOM and pagefault_out_of_memory. The fault does not return an error to userspace, something gets killed instead." [[references]] @@ -266,6 +292,7 @@ anchor = "min_flt number of minor faults" kernel = "7.2.2" confirmed = true line = 335 +context = "e4044e65723e" note = "The minor and major fault counters as userspace sees them, in the documented layout of /proc/PID/stat. Section 5 says these move and this is where their meaning is written down. The spacing in the anchor is load bearing, because the line below is cmin_flt with the same words after it." [[references]] @@ -275,4 +302,5 @@ anchor = "PERF_COUNT_SW_PAGE_FAULTS" kernel = "7.2.2" confirmed = true line = 1290 +context = "4538dbc6ccd0" note = "The perf software events the entry point emits, including the split between minor and major that lands in the counters above." diff --git a/blueprints/write-path.refs.toml b/blueprints/write-path.refs.toml index efe3015..9495ecb 100644 --- a/blueprints/write-path.refs.toml +++ b/blueprints/write-path.refs.toml @@ -34,6 +34,7 @@ anchor = "SYSCALL_DEFINE3(write, unsigned int, fd," kernel = "7.2.2" confirmed = true line = 747 +context = "537170aed093" note = "The system call itself. Three arguments, and the return value is a count of bytes rather than a success flag, which is the fact that makes short writes possible and makes ignoring the return value a bug." [[references]] @@ -43,6 +44,7 @@ anchor = "ssize_t ksys_write(unsigned int fd" kernel = "7.2.2" confirmed = true line = 728 +context = "9a573838c189" note = "The descriptor lookup and the file position handling, which is everything the system call does that is about the process rather than about the file." [[references]] @@ -52,6 +54,7 @@ anchor = "ssize_t vfs_write(struct file *file, const char __user *buf" kernel = "7.2.2" confirmed = true line = 667 +context = "b52d46f97544" note = "The VFS entry point and the top of everything this blueprint describes. Four refusals, a dispatch and two counters, in about thirty lines." [[references]] @@ -61,6 +64,7 @@ anchor = "int rw_verify_area(int read_write, struct file *file" kernel = "7.2.2" confirmed = true line = 453 +context = "062b4c7ed3f7" note = "The position and count checks, plus the two hooks that let something outside the filesystem refuse a write: the security modules and the file notification machinery." # There is no entry for the line that applies the cap, and that is worth explaining rather than @@ -77,6 +81,7 @@ anchor = "#define MAX_RW_COUNT (INT_MAX & PAGE_MASK)" kernel = "7.2.2" confirmed = true line = 2424 +context = "4a34c6cbd112" note = "What that limit is. A little under two gigabytes on a 32-bit build, rounded down to a page, and it is the same number on 64-bit because the return type is signed and 32 bits wide in the ABI." [[references]] @@ -86,6 +91,7 @@ anchor = "static ssize_t new_sync_write(struct file *filp" kernel = "7.2.2" confirmed = true line = 585 +context = "8fe3e9a33540" note = "The adapter that turns the old three argument interface into the iterator one. It builds a kiocb and an iov_iter on the stack, which is why neither of them is allocated anywhere in this path." [[references]] @@ -95,6 +101,7 @@ anchor = "ret = filp->f_op->write_iter(&kiocb, &iter);" kernel = "7.2.2" confirmed = true line = 595 +context = "c7e77a2e5b18" note = "The call into the filesystem, and the line after it is a BUG_ON, because a synchronous write is not allowed to come back saying it will finish later." [[references]] @@ -104,6 +111,7 @@ anchor = "static inline void file_start_write(struct file *file)" kernel = "7.2.2" confirmed = true line = 2729 +context = "f8f931d5a68e" note = "Taking the filesystem's freeze protection for the duration of the write. This is a lock, it is held across everything below, and it is the reason a frozen filesystem makes writers wait rather than fail." [[references]] @@ -113,6 +121,7 @@ anchor = "static ssize_t shmem_file_write_iter(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 3408 +context = "dd01c9306283" note = "The filesystem half of the path on the machine this project boots. Five calls, in an order that is nearly the same as the generic one, which is what makes it a fair thing to read as an example." [[references]] @@ -122,6 +131,7 @@ anchor = "ssize_t generic_file_write_iter(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 4498 +context = "160148f683b8" note = "The generic version most filesystems use. Take the inode lock, check, write, drop the lock, and then handle O_SYNC outside it." [[references]] @@ -131,6 +141,7 @@ anchor = "ssize_t __generic_file_write_iter(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 4451 +context = "e5845795c4e6" note = "The part inside the inode lock, and the split between direct and buffered writes. Direct writes are a different mechanism from this one and it is worth seeing where they leave." [[references]] @@ -140,6 +151,7 @@ anchor = "ssize_t generic_write_checks(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 1767 +context = "678b6447f4ae" note = "The checks every buffered write goes through, and the place the iterator gets truncated when the write has to be shortened rather than refused." [[references]] @@ -149,6 +161,7 @@ anchor = "int generic_write_checks_count(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 1737 +context = "d09f62040b76" note = "Refusing a write to a swap file, applying O_APPEND, and refusing a non blocking write that the filesystem cannot do without blocking." [[references]] @@ -158,6 +171,7 @@ anchor = "iocb->ki_pos = i_size_read(inode);" kernel = "7.2.2" confirmed = true line = 1749 +context = "14b8ed69cd5d" note = "O_APPEND, in one line. The position is read here rather than by the caller, and it is read with the inode lock held, which is the whole reason appends from two processes do not overwrite each other." [[references]] @@ -167,6 +181,7 @@ anchor = "int generic_write_check_limits(struct file *file" kernel = "7.2.2" confirmed = true line = 1710 +context = "897e7b8a8711" note = "The file size limits. Two of them, the filesystem's own maximum and the process resource limit, and going past the second one sends SIGXFSZ as well as returning an error." [[references]] @@ -176,6 +191,7 @@ anchor = "int file_remove_privs(struct file *file)" kernel = "7.2.2" confirmed = true line = 2411 +context = "4a44c90c1ea5" note = "Dropping setuid and setgid because the file has been modified. It runs on every write to every file, and on the pinned machine it is the most expensive thing in the trace after the allocation." [[references]] @@ -185,6 +201,7 @@ anchor = "int file_update_time(struct file *file)" kernel = "7.2.2" confirmed = true line = 2506 +context = "d447b07b7e1a" note = "The modification timestamp. It is skipped when the clock has not moved on since the last update, which makes its presence in a trace a fact about when somebody ran it." [[references]] @@ -194,6 +211,7 @@ anchor = "ssize_t generic_perform_write(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 4336 +context = "8a5fcbf7839b" note = "The loop that does the actual work, and the function this whole blueprint is arranged around. Everything above it is checking and everything below it is the filesystem." [[references]] @@ -203,6 +221,7 @@ anchor = "size_t chunk = mapping_max_folio_size(mapping);" kernel = "7.2.2" confirmed = true line = 4342 +context = "98884df03f6e" note = "How much of the write the loop tries to do in one go. It is the largest folio this mapping supports rather than a page, which is why the loop can run once for a write far larger than a page." [[references]] @@ -212,6 +231,7 @@ anchor = "status = a_ops->write_begin(iocb, mapping, pos, bytes," kernel = "7.2.2" confirmed = true line = 4364 +context = "d87d749d2c77" note = "The first of the two calls into the filesystem. It comes back with a folio that is locked and belongs to the caller until write_end." [[references]] @@ -221,6 +241,7 @@ anchor = "copied = copy_folio_from_iter_atomic(folio, offset, bytes, i);" kernel = "7.2.2" confirmed = true line = 4382 +context = "a1440b133b09" note = "The copy, and the one line in this path that is not allowed to sleep. The comment above it says why: a fault taken here can re-enter the filesystem that is holding the folio lock." [[references]] @@ -230,6 +251,7 @@ anchor = "if (mapping_writably_mapped(mapping))" kernel = "7.2.2" confirmed = true line = 4373 +context = "597a4d662d5d" note = "The cache flush that only happens when the same file is also mapped into somebody's address space. On architectures with virtually indexed caches, skipping it would let a writer and a mapper see different bytes." [[references]] @@ -239,6 +261,7 @@ anchor = "status = a_ops->write_end(iocb, mapping, pos, bytes, copied," kernel = "7.2.2" confirmed = true line = 4385 +context = "ec5646c53958" note = "The second call into the filesystem, and the one that decides how much of the copy counted. Returning less than was copied makes the loop revert the iterator and try again." [[references]] @@ -248,6 +271,7 @@ anchor = "if (fatal_signal_pending(current)) {" kernel = "7.2.2" confirmed = true line = 4359 +context = "0873f8dd3835" note = "The only place in the loop a signal is looked at, checked once per folio and only for fatal signals. An ordinary signal does not interrupt a write in progress." [[references]] @@ -257,6 +281,7 @@ anchor = "void balance_dirty_pages_ratelimited(struct address_space *mapping)" kernel = "7.2.2" confirmed = true line = 2111 +context = "a800f648046a" note = "The throttle, called once per folio at the top of the loop. Most calls do nothing, and the one that does not can sleep for as long as it takes writeback to catch up." [[references]] @@ -266,6 +291,7 @@ anchor = "int balance_dirty_pages_ratelimited_flags(struct address_space *mappin kernel = "7.2.2" confirmed = true line = 2041 +context = "387daff39a48" note = "Where the decision is actually made, from a per CPU counter of how many pages this CPU has dirtied since it last looked." [[references]] @@ -275,6 +301,7 @@ anchor = "shmem_write_begin(const struct kiocb *iocb, struct address_space *mapp kernel = "7.2.2" confirmed = true line = 3226 +context = "05c7d5b115b0" note = "One filesystem's write_begin. It checks the seals, gets a folio, and refuses a folio with a hardware memory error in it." [[references]] @@ -284,6 +311,7 @@ anchor = "int shmem_get_folio(struct inode *inode, pgoff_t index, loff_t write_e kernel = "7.2.2" confirmed = true line = 2615 +context = "856b4f8392c1" note = "Finding the folio in the page cache or allocating one. On a first write to a fresh file this is where the allocation happens, and it is most of the time in the capture in section 5." [[references]] @@ -293,6 +321,7 @@ anchor = "if (unlikely(info->seals & (F_SEAL_GROW |" kernel = "7.2.2" confirmed = true line = 3237 +context = "6470052105c2" note = "Seals, which are the one way a write to a file that is open for writing can be refused with EPERM at this depth. Only tmpfs and memfd have them." [[references]] @@ -302,6 +331,7 @@ anchor = "shmem_write_end(const struct kiocb *iocb, struct address_space *mappin kernel = "7.2.2" confirmed = true line = 3264 +context = "69114b775059" note = "Growing the file, zeroing the rest of a partly written folio, marking it dirty and unlocking it. The order of those four is the order the rest of the kernel depends on." [[references]] @@ -311,6 +341,7 @@ anchor = "bool folio_mark_dirty(struct folio *folio)" kernel = "7.2.2" confirmed = true line = 2778 +context = "6c048781f542" note = "The last thing that happens to the data inside the system call. After this the folio is somebody else's problem and the write returns." [[references]] @@ -320,6 +351,7 @@ anchor = "static const struct address_space_operations shmem_aops = {" kernel = "7.2.2" confirmed = true line = 5136 +context = "fa5a5d0fa30b" note = "The ops table the loop calls through, filled in for tmpfs. Its dirty_folio is noop_dirty_folio, which is the single line that explains why tmpfs never writes anything back." [[references]] @@ -329,6 +361,7 @@ anchor = "if (WARN_ON_ONCE(folio_test_dirty(folio) &&" kernel = "7.2.2" confirmed = true line = 211 +context = "4f67432acb9f" note = "The warning that fires when a folio leaves the page cache still marked dirty. It is the one message in this area a reader is likely to meet, and it means data was thrown away rather than written." [[references]] @@ -338,6 +371,7 @@ anchor = "static inline ssize_t generic_write_sync(struct kiocb *iocb" kernel = "7.2.2" confirmed = true line = 2663 +context = "8bba97b1910f" note = "O_SYNC and O_DSYNC, handled after the inode lock is dropped. This is the only path in this blueprint that reaches storage before returning." [[references]] @@ -347,6 +381,7 @@ anchor = "DEFINE_EVENT(writeback_folio_template, writeback_dirty_folio," kernel = "7.2.2" confirmed = true line = 90 +context = "a156fc5e2adb" note = "The tracepoint that fires when a folio is dirtied, which is the observable event closest to the end of this path." [[references]] @@ -356,6 +391,7 @@ anchor = "TRACE_EVENT(balance_dirty_pages," kernel = "7.2.2" confirmed = true line = 657 +context = "790619ef11a4" note = "The tracepoint for the throttle, and the one that says how long a writer was made to wait and why. Nothing else in the kernel reports that number." [[references]] @@ -365,4 +401,5 @@ anchor = "static int do_io_accounting(struct task_struct *task" kernel = "7.2.2" confirmed = true line = 3002 +context = "14b7bac656cb" note = "Where the per process byte and call counters vfs_write bumps are read back out. It is behind CONFIG_TASK_IO_ACCOUNTING, and section 8 says what that means for the pinned build." diff --git a/corpora/BASELINE.toml b/corpora/BASELINE.toml index 52d0562..10f70c5 100644 --- a/corpora/BASELINE.toml +++ b/corpora/BASELINE.toml @@ -12,10 +12,10 @@ schema = 1 [totals] -artefacts = 32 -lines = 1982 -read = 1604 -skipped = 209 +artefacts = 38 +lines = 5224 +read = 2779 +skipped = 454 unparsed = 0 [[artefact]] @@ -114,6 +114,15 @@ read = 7 skipped = 1 unparsed = 0 +[[artefact]] +path = "corpora/proc/tier0/kallsyms-write.txt" +reader = "kallsyms" +lines = 8 +found = 8 +read = 8 +skipped = 0 +unparsed = 0 + [[artefact]] path = "corpora/proc/tier0/lockdep-stats-after.txt" reader = "lockdep-stats" @@ -204,6 +213,48 @@ read = 1 skipped = 0 unparsed = 0 +[[artefact]] +path = "corpora/source/pinned/MAINTAINERS.excerpt" +reader = "maintainers" +lines = 296 +found = 13 +read = 216 +skipped = 80 +unparsed = 0 + +[[artefact]] +path = "corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.tbl" +reader = "syscall-table" +lines = 479 +found = 461 +read = 461 +skipped = 18 +unparsed = 0 + +[[artefact]] +path = "corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.tbl" +reader = "syscall-table" +lines = 443 +found = 421 +read = 421 +skipped = 22 +unparsed = 0 + +[[artefact]] +path = "corpora/source/pinned/fs/read_write.c" +reader = "none" +lines = 1822 +found = 0 + +[[artefact]] +path = "corpora/source/pinned/kernel/Kconfig.preempt" +reader = "kconfig-source" +lines = 194 +found = 15 +read = 69 +skipped = 125 +unparsed = 0 + [[artefact]] path = "corpora/traces/handwritten/page-fault.txt" reader = "function_graph" diff --git a/corpora/README.md b/corpora/README.md index 992719f..ff39842 100644 --- a/corpora/README.md +++ b/corpora/README.md @@ -16,6 +16,7 @@ corpora/ ├── experiments/ # measurements rather than recordings, so far all Tier 1 ├── oops/ # crash and lockdep text used by the debugging lessons, tier0 and handwritten ├── litmus/ # herd7 and klitmus7 output for the memory model lessons +├── source/pinned/ # five files out of the pinned tarball, not off a running kernel └── tier0/ # recipes.toml, the list of what a Tier 0 session can replay ``` @@ -27,6 +28,14 @@ Tier 0 has exactly two limits that no amount of work removes. It has one CPU, so `traces/tier1/` and `experiments/tier1/` are for captures that hit one of those two and nothing else. A Tier 1 capture came off whatever machine somebody had, so its metadata carries the kernel version, the distribution, the architecture and the CPU count in full, and the one in here at the moment is arm64 running 6.8. That is a fine place to demonstrate that the trace file interleaves CPUs, which is true everywhere Linux runs. It would be a terrible place to demonstrate which functions a write calls. Knowing which of those you are doing is the whole job. +## Source files + +`source/pinned/` is the one directory here that did not come off a running kernel. It is five files out of the `linux-7.2.2` tarball, checksum verified, so that the parsers in `kxray.source` have something real to run against in CI and in a notebook that has downloaded nothing. `source/pinned/README.md` says what each one is there to show. + +The whole tree is 1.6 GB and `./kxbox/kernel/tree.sh` unpacks it, which is what a reader who wants complete answers should do. This is a kernel tree in shape and almost none of one in content, and `Tree.complete` is False on it so that a lookup which misses says the file is not in the corpus rather than saying it is not in the kernel. + +One of the five is an excerpt, because MAINTAINERS is 29847 lines. The `.excerpt` on its name is the mechanism rather than a label: a read of `MAINTAINERS` finds it and marks what it hands back as partial, so being a slice travels with the content. + ## Experiments A capture says what the kernel did. An experiment says what it cost, which needs a clock, which is why `experiments/` is Tier 1 and likely to stay that way. Same rules otherwise: output committed exactly as it came out, metadata beside it, nothing edited to match the prose. diff --git a/corpora/proc/tier0/kallsyms-write.meta.toml b/corpora/proc/tier0/kallsyms-write.meta.toml new file mode 100644 index 0000000..ebe4d72 --- /dev/null +++ b/corpora/proc/tier0/kallsyms-write.meta.toml @@ -0,0 +1,41 @@ +source = "tier0" +evidence = true + +describes = "the symbols behind a write, off the running pinned kernel" +path = "/proc/kallsyms" +stability = "not-abi" + +kernel = "7.2.2" +arch = "i386" +profile = "A-full" +uniprocessor = true +preempt = true +captured = "2026-09-06" +tier = 0 + +command = "grep -E \" (__ia32_sys_write|ksys_write|vfs_write|__ia32_sys_read|ksys_read|vfs_read|sys_call_table|ia32_sys_call_table|handle_mm_fault|do_page_fault)$\" /proc/kallsyms" +setup = [] + +# Eight lines out of the 50787 the running kernel had, filtered by the command above rather than +# edited afterwards. The filter is in the metadata so anybody can take it again. +# +# This is the other end of kxray.source.symbols. The source says SYSCALL_DEFINE3(write, ...), the +# syscall table says the entry point is sys_write, and the running kernel calls it +# __ia32_sys_write. Grepping the source for the name in a stack trace finds nothing. +# +# Two names asked for are not in the answer. There is no ia32_sys_call_table, because a 32 bit +# kernel has one table and calls it sys_call_table. There is no do_page_fault either, which is a +# thing several lessons had to learn the hard way in a different context. +# +# The stability is not-abi rather than undocumented. The closing section of Documentation/ABI/README +# names kernel symbols as something that must not under any circumstances be considered stable, so +# this is the strongest warning the ledger has and it is on the file behind every stack trace. +timings_are_real = false + +unparsed_lines = 0 +symbols = 8 +total_symbols_on_the_box = 50787 +addresses_hidden = false +text_symbols = 7 +data_symbols = 1 +asked_for_and_absent = ["ia32_sys_call_table", "do_page_fault"] diff --git a/corpora/proc/tier0/kallsyms-write.txt b/corpora/proc/tier0/kallsyms-write.txt new file mode 100644 index 0000000..603402a --- /dev/null +++ b/corpora/proc/tier0/kallsyms-write.txt @@ -0,0 +1,8 @@ +c12354b0 T handle_mm_fault +c1276650 T vfs_read +c1276ce0 T vfs_write +c1277090 T ksys_read +c1277160 T __ia32_sys_read +c1277180 T ksys_write +c1277250 T __ia32_sys_write +c14b5900 D sys_call_table diff --git a/corpora/source/pinned/MAINTAINERS.excerpt b/corpora/source/pinned/MAINTAINERS.excerpt new file mode 100644 index 0000000..d518ce7 --- /dev/null +++ b/corpora/source/pinned/MAINTAINERS.excerpt @@ -0,0 +1,296 @@ +List of maintainers +=================== + +Descriptions of section entries and preferred order +--------------------------------------------------- + + M: *Mail* patches to: FullName + R: Designated *Reviewer*: FullName + These reviewers should be CCed on patches. + L: *Mailing list* that is relevant to this area + S: *Status*, one of the following: + Supported: Someone is actually paid to look after this. + Maintained: Someone actually looks after it. + Odd Fixes: It has a maintainer but they don't have time to do + much other than throw the odd patch in. See below.. + Orphan: No current maintainer [but maybe you could take the + role as you write your new code]. + Obsolete: Old code. Something tagged obsolete generally means + it has been replaced by a better system and you + should be using that. + W: *Web-page* with status/info + Q: *Patchwork* web based patch tracking system site + B: URI for where to file *bugs*. A web-page with detailed bug + filing info, a direct bug tracker link, or a mailto: URI. + C: URI for *chat* protocol, server and channel where developers + usually hang out, for example irc://server/channel. + P: *Subsystem Profile* document for more details submitting + patches to the given subsystem. This is either an in-tree .rst file + inside Documentation/, or a URI. + See Documentation/maintainer/maintainer-entry-profile.rst for details. + T: *SCM* tree type and location. + Type is one of: git, hg, quilt, stgit, topgit + F: *Files* and directories wildcard patterns. + A trailing slash includes all files and subdirectory files. + F: drivers/net/ all files in and below drivers/net + F: drivers/net/* all files in drivers/net, but not below + F: */net/* all files in "any top level directory"/net + F: fs/**/*foo*.c all *foo*.c files in any subdirectory of fs + One pattern per line. Multiple F: lines acceptable. + X: *Excluded* files and directories that are NOT maintained, same + rules as F:. Files exclusions are tested before file matches. + Can be useful for excluding a specific subdirectory, for instance: + F: net/ + X: net/ipv6/ + matches all files in and below net excluding net/ipv6/ + N: Files and directories *Regex* patterns. + N: [^a-z]tegra all files whose path contains tegra + (not including files like integrator) + One pattern per line. Multiple N: lines acceptable. + scripts/get_maintainer.pl has different behavior for files that + match F: pattern and matches of N: patterns. By default, + get_maintainer will not look at git log history when an F: pattern + match occurs. When an N: match occurs, git log history is used + to also notify the people that have git commit signatures. + K: *Content regex* (perl extended) pattern match in a patch or file. + For instance: + K: of_get_profile + matches patches or files that contain "of_get_profile" + K: \b(printk|pr_(info|err))\b + matches patches or files that contain one or more of the words + printk, pr_info or pr_err + One regex pattern per line. Multiple K: lines acceptable. + +Maintainers List +---------------- + +.. note:: This file is an excerpt. See pinned.meta.toml beside it. + +ABI/API +L: linux-api@vger.kernel.org +F: include/linux/syscalls.h +F: kernel/sys_ni.c +X: arch/*/include/uapi/ +X: include/uapi/ + +AUDIT SUBSYSTEM +M: Paul Moore +M: Eric Paris +L: audit@vger.kernel.org +S: Supported +W: https://github.com/linux-audit +Q: https://patchwork.kernel.org/project/audit/list +B: mailto:audit@vger.kernel.org +P: https://github.com/linux-audit/audit-kernel/blob/main/README.md +T: git https://git.kernel.org/pub/scm/linux/kernel/git/pcmoore/audit.git +F: include/asm-generic/audit_*.h +F: include/linux/audit.h +F: include/linux/audit_arch.h +F: include/uapi/linux/audit.h +F: kernel/audit* +F: lib/*audit.c +K: \baudit_[a-z_0-9]\+\b + +DOCUMENTATION +M: Jonathan Corbet +R: Shuah Khan +L: linux-doc@vger.kernel.org +S: Maintained +P: Documentation/doc-guide/maintainer-profile.rst +T: git git://git.lwn.net/linux.git docs-next +F: Documentation/ +F: tools/lib/python/ +F: tools/docs/ +F: tools/net/ynl/pyynl/lib/doc_generator.py +X: Documentation/ABI/ +X: Documentation/admin-guide/media/ +X: Documentation/devicetree/ +X: Documentation/driver-api/media/ +X: Documentation/firmware-guide/acpi/ +X: Documentation/i2c/ +X: Documentation/netlink/ +X: Documentation/power/ +X: Documentation/spi/ +X: Documentation/userspace-api/media/ + +FILESYSTEMS (VFS and infrastructure) +M: Alexander Viro +M: Christian Brauner +R: Jan Kara +L: linux-fsdevel@vger.kernel.org +S: Maintained +T: git https://git.kernel.org/pub/scm/linux/kernel/git/vfs/vfs.git +F: fs/* +F: include/linux/fs.h +F: include/linux/fs_types.h +F: include/uapi/linux/fs.h +F: include/uapi/linux/openat2.h +F: rust/kernel/fs.rs +F: rust/kernel/fs/ +F: rust/kernel/seq_file.rs +F: rust/kernel/sync/poll.rs +F: Documentation/driver-api/early-userspace/buffer-format.rst +F: init/do_mounts* +F: init/*initramfs* + +LOCKING PRIMITIVES +M: Peter Zijlstra +M: Ingo Molnar +M: Will Deacon +M: Boqun Feng (LOCKDEP & RUST) +R: Waiman Long +L: linux-kernel@vger.kernel.org +S: Maintained +P: Documentation/process/maintainer-tip.rst +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git locking/core +F: Documentation/locking/ +F: arch/*/include/asm/spinlock*.h +F: include/linux/local_lock*.h +F: include/linux/lockdep*.h +F: include/linux/mutex*.h +F: include/linux/rwlock*.h +F: include/linux/rwsem*.h +F: include/linux/seqlock.h +F: include/linux/spinlock*.h +F: kernel/locking/ +F: lib/locking*.[ch] +F: rust/helpers/mutex.c +F: rust/helpers/spinlock.c +F: rust/kernel/sync/lock.rs +F: rust/kernel/sync/lock/ +F: rust/kernel/sync/locked_by.rs +X: kernel/locking/locktorture.c + +MEMORY MANAGEMENT +M: Andrew Morton +L: linux-mm@kvack.org +S: Maintained +W: http://www.linux-mm.org +T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +T: quilt git://git.kernel.org/pub/scm/linux/kernel/git/akpm/25-new +F: mm/ +F: tools/mm/ + +MEMORY MANAGEMENT - CORE +M: Andrew Morton +M: David Hildenbrand +R: Lorenzo Stoakes +R: Liam R. Howlett +R: Vlastimil Babka +R: Mike Rapoport +R: Suren Baghdasaryan +R: Michal Hocko +L: linux-mm@kvack.org +S: Maintained +W: http://www.linux-mm.org +T: git git://git.kernel.org/pub/scm/linux/kernel/git/akpm/mm +F: Documentation/admin-guide/sysctl/vm.rst +F: include/linux/folio_batch.h +F: include/linux/gfp.h +F: include/linux/gfp_types.h +F: include/linux/highmem.h +F: include/linux/leafops.h +F: include/linux/memory.h +F: include/linux/mm.h +F: include/linux/mm_*.h +F: include/linux/mmzone.h +F: include/linux/mmdebug.h +F: include/linux/mmu_notifier.h +F: include/linux/pagewalk.h +F: include/linux/pgalloc.h +F: include/linux/pgtable.h +F: include/linux/ptdump.h +F: include/linux/vmpressure.h +F: include/linux/vmstat.h +F: fs/proc/meminfo.c +F: kernel/fork.c +F: mm/Kconfig +F: mm/debug.c +F: mm/folio-compat.c +F: mm/highmem.c +F: mm/init-mm.c +F: mm/internal.h +F: mm/maccess.c +F: mm/memory.c +F: mm/mmu_notifier.c +F: mm/mmzone.c +F: mm/pagewalk.c +F: mm/pgtable-generic.c +F: mm/ptdump.c +F: mm/sparse-vmemmap.c +F: mm/sparse.c +F: mm/util.c +F: mm/vmpressure.c +F: mm/vmstat.c +N: include\/linux\/page[-_][a-zA-Z]* + +PROC FILESYSTEM +L: linux-kernel@vger.kernel.org +L: linux-fsdevel@vger.kernel.org +S: Maintained +F: Documentation/filesystems/proc.rst +F: fs/proc/ +F: include/linux/proc_fs.h +F: tools/testing/selftests/proc/ + +TEGRA ARCHITECTURE SUPPORT +M: Thierry Reding +M: Jonathan Hunter +L: linux-tegra@vger.kernel.org +S: Supported +Q: http://patchwork.ozlabs.org/project/linux-tegra/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tegra/linux.git +N: [^a-z]tegra + +TRACING +M: Steven Rostedt +M: Masami Hiramatsu +R: Mathieu Desnoyers +L: linux-kernel@vger.kernel.org +L: linux-trace-kernel@vger.kernel.org +S: Maintained +Q: https://patchwork.kernel.org/project/linux-trace-kernel/list/ +T: git git://git.kernel.org/pub/scm/linux/kernel/git/trace/linux-trace.git +F: Documentation/core-api/tracepoint.rst +F: Documentation/trace/* +F: fs/tracefs/ +F: include/linux/trace*.h +F: include/trace/ +F: kernel/trace/ +F: kernel/tracepoint.c +F: scripts/tracing/ +F: scripts/tracepoint-update.c +F: tools/testing/selftests/ftrace/ + +X86 ARCHITECTURE (32-BIT AND 64-BIT) +M: Thomas Gleixner +M: Ingo Molnar +M: Borislav Petkov +M: Dave Hansen +M: x86@kernel.org +R: "H. Peter Anvin" +L: linux-kernel@vger.kernel.org +S: Maintained +P: Documentation/process/maintainer-tip.rst +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86/core +F: Documentation/arch/x86/ +F: Documentation/devicetree/bindings/x86/ +F: arch/x86/ +F: tools/testing/selftests/x86 + +X86 MM +M: Dave Hansen +M: Andy Lutomirski +M: Peter Zijlstra +L: linux-kernel@vger.kernel.org +S: Maintained +T: git git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip.git x86/mm +F: arch/x86/mm/ + +THE REST +M: Linus Torvalds +L: linux-kernel@vger.kernel.org +S: Buried alive in reporters +T: git git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git +F: * +F: */ diff --git a/corpora/source/pinned/MAINTAINERS.meta.toml b/corpora/source/pinned/MAINTAINERS.meta.toml new file mode 100644 index 0000000..96b97aa --- /dev/null +++ b/corpora/source/pinned/MAINTAINERS.meta.toml @@ -0,0 +1,44 @@ +source = "pinned" +evidence = true + +describes = "the format block from MAINTAINERS, and thirteen of its sections" +path = "MAINTAINERS" +excerpt = true + +kernel = "7.2.2" +tarball_sha256 = "7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83" +taken = "2026-09-06" + +# This is the one file in corpora/source/pinned that is not whole. The real MAINTAINERS on 7.2.2 is +# 29847 lines and 916 KB, which is nine times the rest of this corpus put together, so what is here +# is the header block that documents the format, then the sections that cover the paths this +# repository cites, then three sections chosen for the tags they carry. +# +# ABI/API is here because it is the shortest section in the file with an X: line on it, and X: is +# tested before F:. AUDIT SUBSYSTEM is here for its K: line, which matches on the content of a +# patch rather than on a path. TEGRA ARCHITECTURE SUPPORT is here for its N: line, which is the +# same regex the header block uses as its worked example. +# +# The `.excerpt` on the filename is load bearing. kxray.source.tree finds it when asked for +# MAINTAINERS and sets `partial` on what it hands back, so anything that reads it knows a lookup +# that misses might be missing because of this file rather than because of the kernel. + +# The full file, counted at the version above. +full_lines = 29847 +full_sections = 3293 +full_file_patterns = 9738 +full_excluded_patterns = 100 +full_path_patterns = 105 +full_content_patterns = 66 + +# This excerpt. +sections = 13 +catch_all = "THE REST" + +# What the lookups come out as, against this excerpt. +memory_c_sections = ["MEMORY MANAGEMENT", "MEMORY MANAGEMENT - CORE", "THE REST"] +proc_base_c_sections = ["PROC FILESYSTEM", "THE REST"] +read_write_c_sections = ["FILESYSTEMS (VFS and infrastructure)", "THE REST"] +sections_with_no_maintainer = ["ABI/API", "PROC FILESYSTEM"] +sections_with_no_status = ["ABI/API"] +audit_content_pattern_matches_audit_log_start = false diff --git a/corpora/source/pinned/README.md b/corpora/source/pinned/README.md new file mode 100644 index 0000000..bdf659f --- /dev/null +++ b/corpora/source/pinned/README.md @@ -0,0 +1,74 @@ +# Five files out of the pinned kernel + +Everything else in `corpora/` came off a running kernel. This came out of the tarball, so it is a different kind of evidence and it lives in a directory of its own rather than being filed next to the traces. + +The tarball is `linux-7.2.2.tar.xz`, sha256 `7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83`, which is the same checksum `kxbox/kernel/build.sh` and `kxbox/kernel/tree.sh` verify before either of them will do anything. The checksum is in every metadata file beside every capture here, so a file in this directory can be traced back to a kernel without going through anything this repository wrote. + +## Why not the whole tree + +The whole tree is 1.6 GB. `./kxbox/kernel/tree.sh` unpacks it and `kxray.source.tree.find()` uses it when it is there. This directory is what a reader has when it is not there, which is every reader opening a lesson in Colab and every CI run. + +So this is a kernel tree in shape and almost none of one in content, and that difference has to survive being read. `Tree.complete` is False here, and a lookup that misses says the file is not in the corpus rather than saying it is not in the kernel. Those are different sentences and only one of them is true. + +## What is here + +``` +corpora/source/pinned/ +├── MAINTAINERS.excerpt # the format block and thirteen sections +├── arch/x86/entry/syscalls/syscall_32.tbl # the numbers the pinned box answers to +├── arch/x86/entry/syscalls/syscall_64.tbl # the numbers the reader's laptop answers to +├── fs/read_write.c # one real source file, whole +└── kernel/Kconfig.preempt # why a .config has symbols nobody chose +``` + +Four of the five are verbatim. The fifth is an excerpt, and the `.excerpt` on its name is the mechanism rather than a label: `Tree.read("MAINTAINERS")` finds it, returns it, and sets `partial` on what it hands back, so the fact that it is a slice travels with the content. + +## MAINTAINERS.excerpt + +The real file is 29847 lines and 916 KB, which is nine times the rest of this directory put together. + +What is kept is the header block that documents the format, then the thirteen sections that cover the paths this repository cites, then three sections chosen for the tags they carry. ABI/API is the shortest section with an `X:` line on it, and exclusions are tested before matches. AUDIT SUBSYSTEM has a `K:` line, which matches on the content of a patch rather than on a path. TEGRA ARCHITECTURE SUPPORT has an `N:` line, and it is the same regex the header block uses as its own worked example. + +Three things this file is here to show. + +A path has more than one maintainer. `mm/memory.c` matches MEMORY MANAGEMENT, MEMORY MANAGEMENT - CORE and THE REST, and THE REST is the last section in the file with `F: *` and `F: */` on it, so it matches everything in Linux. Any lookup that takes the first hit and stops reports Linus for every file in the kernel. + +The globs are not fnmatch. FILESYSTEMS (VFS and infrastructure) carries `F: fs/*`, and `fnmatch.fnmatch("fs/proc/base.c", "fs/*")` is True while the answer the file gives is False, because a single star stops at a slash. `fs/proc/` belongs to PROC FILESYSTEM, and the VFS section said so by writing one star instead of a trailing slash. + +A section can name nobody. PROC FILESYSTEM has a status of Maintained, two mailing lists and no `M:` line at all, so the honest answer to who to mail is the lists. + +## The two syscall tables + +Write is 4 on i386 and 1 on x86-64. Read is 3 and 0. Both tables are here so that sentence can be a table rather than a claim. + +The pinned box is 32 bit, so every syscall number any lesson prints comes out of `syscall_32.tbl`, and a reader comparing against the numbers they know from their own machine will find they do not line up. That is not a mistake in either place. A syscall number with no architecture attached is not an identifier. + +Nineteen rows in the 32 bit table have a name and no entry point: `break`, `stty`, `gtty`, `ftime`, `prof` and the rest. Those numbers are reserved forever, because a binary from 1994 making one of those calls has to get ENOSYS rather than somebody else's system call. + +The 64 bit table has three abis in it where the 32 bit one has one, and the same name appears under more than one of them. `rt_sigaction` is 13 under `64` and 512 under `x32`. + +## fs/read_write.c + +One real kernel source file, whole, so that symbol lookup and citation anchors have something to resolve against with nothing downloaded. + +Nothing in this file is named `sys_write`, which is the name the syscall table uses. What is here is `SYSCALL_DEFINE3(write, ...)` on line 747, and the symbol the running kernel ends up with is `__ia32_sys_write`, which is in `corpora/proc/tier0/kallsyms-write.txt` off the box. Three names for one call and no two of them equal. + +Grepping this file for `sys_write` is the part worth trying. It finds lines 728 and 750, and both are `ksys_write`, a different function that the real entry point calls. The reader lands three lines from what they wanted, on something close enough to be believed, and the thing they were actually looking for is spelled `write`. + +`vfs_write` is defined on line 667 and is not exported, so a module cannot call it. `rw_verify_area` is exported. `__kernel_write` is exported to exactly one named module, `autofs4`. Being in `kallsyms` and being callable from a module are different questions. + +## kernel/Kconfig.preempt + +The pinned kernel is built with `CONFIG_PREEMPT=y`. Its `.config` also has `CONFIG_PREEMPT_BUILD=y` and `CONFIG_PREEMPTION=y` in it, which nobody asked for, and this file is the whole explanation: `PREEMPT` selects `PREEMPT_BUILD`, and `PREEMPT_BUILD` selects `PREEMPTION`. + +Six of its fifteen symbols have a type and no prompt. A symbol with no prompt never appears in `menuconfig` and cannot be set by hand at all, and in a `.config` it looks exactly like something a person chose. + +## Taking these again + +```sh +./kxbox/kernel/tree.sh +``` + +That unpacks the pinned tarball into `kxbox/kernel/build/tree/linux-7.2.2/`, checksum verified. The four verbatim files are copies out of that. The excerpt is the header block plus the sections named in `MAINTAINERS.excerpt.meta.toml`, in the order the real file lists them, with the tabs left alone. + +Nothing in here has been edited to match any prose. If a number in a metadata file disagrees with the file beside it, the file is right. diff --git a/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.meta.toml b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.meta.toml new file mode 100644 index 0000000..bfccd9d --- /dev/null +++ b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.meta.toml @@ -0,0 +1,26 @@ +source = "pinned" +evidence = true + +describes = "every system call number the pinned 32 bit kernel answers to" +path = "arch/x86/entry/syscalls/syscall_32.tbl" +excerpt = false + +kernel = "7.2.2" +arch = "i386" +tarball_sha256 = "7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83" +taken = "2026-09-06" + +# Verbatim out of the tarball, because this is the table the pinned box actually uses and every +# syscall number any lesson prints comes from here. The comparison against the 64 bit table beside +# it is the point: write is 4 here and 1 there. +# +# Nineteen rows have a name and no entry point. Those numbers are reserved forever so that a very +# old binary making a very old call gets ENOSYS rather than somebody else's system call. + +rows = 461 +abis = ["i386"] +reserved_rows = 19 +holes = 11 +write_number = 4 +read_number = 3 +write_entry = "sys_write" diff --git a/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.tbl b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.tbl new file mode 100644 index 0000000..f832ebd --- /dev/null +++ b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_32.tbl @@ -0,0 +1,479 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# 32-bit system call numbers and entry vectors +# +# The format is: +# [ [noreturn]] +# +# The __ia32_sys and __ia32_compat_sys stubs are created on-the-fly for +# sys_*() system calls and compat_sys_*() compat system calls if +# IA32_EMULATION is defined, and expect struct pt_regs *regs as their only +# parameter. +# +# The abi is always "i386" for this file. +# +0 i386 restart_syscall sys_restart_syscall +1 i386 exit sys_exit - noreturn +2 i386 fork sys_fork +3 i386 read sys_read +4 i386 write sys_write +5 i386 open sys_open compat_sys_open +6 i386 close sys_close +7 i386 waitpid sys_waitpid +8 i386 creat sys_creat +9 i386 link sys_link +10 i386 unlink sys_unlink +11 i386 execve sys_execve compat_sys_execve +12 i386 chdir sys_chdir +13 i386 time sys_time32 +14 i386 mknod sys_mknod +15 i386 chmod sys_chmod +16 i386 lchown sys_lchown16 +17 i386 break +18 i386 oldstat sys_stat +19 i386 lseek sys_lseek compat_sys_lseek +20 i386 getpid sys_getpid +21 i386 mount sys_mount +22 i386 umount sys_oldumount +23 i386 setuid sys_setuid16 +24 i386 getuid sys_getuid16 +25 i386 stime sys_stime32 +26 i386 ptrace sys_ptrace compat_sys_ptrace +27 i386 alarm sys_alarm +28 i386 oldfstat sys_fstat +29 i386 pause sys_pause +30 i386 utime sys_utime32 +31 i386 stty +32 i386 gtty +33 i386 access sys_access +34 i386 nice sys_nice +35 i386 ftime +36 i386 sync sys_sync +37 i386 kill sys_kill +38 i386 rename sys_rename +39 i386 mkdir sys_mkdir +40 i386 rmdir sys_rmdir +41 i386 dup sys_dup +42 i386 pipe sys_pipe +43 i386 times sys_times compat_sys_times +44 i386 prof +45 i386 brk sys_brk +46 i386 setgid sys_setgid16 +47 i386 getgid sys_getgid16 +48 i386 signal sys_signal +49 i386 geteuid sys_geteuid16 +50 i386 getegid sys_getegid16 +51 i386 acct sys_acct +52 i386 umount2 sys_umount +53 i386 lock +54 i386 ioctl sys_ioctl compat_sys_ioctl +55 i386 fcntl sys_fcntl compat_sys_fcntl64 +56 i386 mpx +57 i386 setpgid sys_setpgid +58 i386 ulimit +59 i386 oldolduname sys_olduname +60 i386 umask sys_umask +61 i386 chroot sys_chroot +62 i386 ustat sys_ustat compat_sys_ustat +63 i386 dup2 sys_dup2 +64 i386 getppid sys_getppid +65 i386 getpgrp sys_getpgrp +66 i386 setsid sys_setsid +67 i386 sigaction sys_sigaction compat_sys_sigaction +68 i386 sgetmask sys_sgetmask +69 i386 ssetmask sys_ssetmask +70 i386 setreuid sys_setreuid16 +71 i386 setregid sys_setregid16 +72 i386 sigsuspend sys_sigsuspend +73 i386 sigpending sys_sigpending compat_sys_sigpending +74 i386 sethostname sys_sethostname +75 i386 setrlimit sys_setrlimit compat_sys_setrlimit +76 i386 getrlimit sys_old_getrlimit compat_sys_old_getrlimit +77 i386 getrusage sys_getrusage compat_sys_getrusage +78 i386 gettimeofday sys_gettimeofday compat_sys_gettimeofday +79 i386 settimeofday sys_settimeofday compat_sys_settimeofday +80 i386 getgroups sys_getgroups16 +81 i386 setgroups sys_setgroups16 +82 i386 select sys_old_select compat_sys_old_select +83 i386 symlink sys_symlink +84 i386 oldlstat sys_lstat +85 i386 readlink sys_readlink +86 i386 uselib sys_uselib +87 i386 swapon sys_swapon +88 i386 reboot sys_reboot +89 i386 readdir sys_old_readdir compat_sys_old_readdir +90 i386 mmap sys_old_mmap compat_sys_ia32_mmap +91 i386 munmap sys_munmap +92 i386 truncate sys_truncate compat_sys_truncate +93 i386 ftruncate sys_ftruncate compat_sys_ftruncate +94 i386 fchmod sys_fchmod +95 i386 fchown sys_fchown16 +96 i386 getpriority sys_getpriority +97 i386 setpriority sys_setpriority +98 i386 profil +99 i386 statfs sys_statfs compat_sys_statfs +100 i386 fstatfs sys_fstatfs compat_sys_fstatfs +101 i386 ioperm sys_ioperm +102 i386 socketcall sys_socketcall compat_sys_socketcall +103 i386 syslog sys_syslog +104 i386 setitimer sys_setitimer compat_sys_setitimer +105 i386 getitimer sys_getitimer compat_sys_getitimer +106 i386 stat sys_newstat compat_sys_newstat +107 i386 lstat sys_newlstat compat_sys_newlstat +108 i386 fstat sys_newfstat compat_sys_newfstat +109 i386 olduname sys_uname +110 i386 iopl sys_iopl +111 i386 vhangup sys_vhangup +112 i386 idle +113 i386 vm86old sys_vm86old sys_ni_syscall +114 i386 wait4 sys_wait4 compat_sys_wait4 +115 i386 swapoff sys_swapoff +116 i386 sysinfo sys_sysinfo compat_sys_sysinfo +117 i386 ipc sys_ipc compat_sys_ipc +118 i386 fsync sys_fsync +119 i386 sigreturn sys_sigreturn compat_sys_sigreturn +120 i386 clone sys_clone compat_sys_ia32_clone +121 i386 setdomainname sys_setdomainname +122 i386 uname sys_newuname +123 i386 modify_ldt sys_modify_ldt +124 i386 adjtimex sys_adjtimex_time32 +125 i386 mprotect sys_mprotect +126 i386 sigprocmask sys_sigprocmask compat_sys_sigprocmask +127 i386 create_module +128 i386 init_module sys_init_module +129 i386 delete_module sys_delete_module +130 i386 get_kernel_syms +131 i386 quotactl sys_quotactl +132 i386 getpgid sys_getpgid +133 i386 fchdir sys_fchdir +134 i386 bdflush sys_ni_syscall +135 i386 sysfs sys_sysfs +136 i386 personality sys_personality +137 i386 afs_syscall +138 i386 setfsuid sys_setfsuid16 +139 i386 setfsgid sys_setfsgid16 +140 i386 _llseek sys_llseek +141 i386 getdents sys_getdents compat_sys_getdents +142 i386 _newselect sys_select compat_sys_select +143 i386 flock sys_flock +144 i386 msync sys_msync +145 i386 readv sys_readv +146 i386 writev sys_writev +147 i386 getsid sys_getsid +148 i386 fdatasync sys_fdatasync +149 i386 _sysctl sys_ni_syscall +150 i386 mlock sys_mlock +151 i386 munlock sys_munlock +152 i386 mlockall sys_mlockall +153 i386 munlockall sys_munlockall +154 i386 sched_setparam sys_sched_setparam +155 i386 sched_getparam sys_sched_getparam +156 i386 sched_setscheduler sys_sched_setscheduler +157 i386 sched_getscheduler sys_sched_getscheduler +158 i386 sched_yield sys_sched_yield +159 i386 sched_get_priority_max sys_sched_get_priority_max +160 i386 sched_get_priority_min sys_sched_get_priority_min +161 i386 sched_rr_get_interval sys_sched_rr_get_interval_time32 +162 i386 nanosleep sys_nanosleep_time32 +163 i386 mremap sys_mremap +164 i386 setresuid sys_setresuid16 +165 i386 getresuid sys_getresuid16 +166 i386 vm86 sys_vm86 sys_ni_syscall +167 i386 query_module +168 i386 poll sys_poll +169 i386 nfsservctl +170 i386 setresgid sys_setresgid16 +171 i386 getresgid sys_getresgid16 +172 i386 prctl sys_prctl +173 i386 rt_sigreturn sys_rt_sigreturn compat_sys_rt_sigreturn +174 i386 rt_sigaction sys_rt_sigaction compat_sys_rt_sigaction +175 i386 rt_sigprocmask sys_rt_sigprocmask compat_sys_rt_sigprocmask +176 i386 rt_sigpending sys_rt_sigpending compat_sys_rt_sigpending +177 i386 rt_sigtimedwait sys_rt_sigtimedwait_time32 compat_sys_rt_sigtimedwait_time32 +178 i386 rt_sigqueueinfo sys_rt_sigqueueinfo compat_sys_rt_sigqueueinfo +179 i386 rt_sigsuspend sys_rt_sigsuspend compat_sys_rt_sigsuspend +180 i386 pread64 sys_ia32_pread64 +181 i386 pwrite64 sys_ia32_pwrite64 +182 i386 chown sys_chown16 +183 i386 getcwd sys_getcwd +184 i386 capget sys_capget +185 i386 capset sys_capset +186 i386 sigaltstack sys_sigaltstack compat_sys_sigaltstack +187 i386 sendfile sys_sendfile compat_sys_sendfile +188 i386 getpmsg +189 i386 putpmsg +190 i386 vfork sys_vfork +191 i386 ugetrlimit sys_getrlimit compat_sys_getrlimit +192 i386 mmap2 sys_mmap_pgoff +193 i386 truncate64 sys_ia32_truncate64 +194 i386 ftruncate64 sys_ia32_ftruncate64 +195 i386 stat64 sys_stat64 compat_sys_ia32_stat64 +196 i386 lstat64 sys_lstat64 compat_sys_ia32_lstat64 +197 i386 fstat64 sys_fstat64 compat_sys_ia32_fstat64 +198 i386 lchown32 sys_lchown +199 i386 getuid32 sys_getuid +200 i386 getgid32 sys_getgid +201 i386 geteuid32 sys_geteuid +202 i386 getegid32 sys_getegid +203 i386 setreuid32 sys_setreuid +204 i386 setregid32 sys_setregid +205 i386 getgroups32 sys_getgroups +206 i386 setgroups32 sys_setgroups +207 i386 fchown32 sys_fchown +208 i386 setresuid32 sys_setresuid +209 i386 getresuid32 sys_getresuid +210 i386 setresgid32 sys_setresgid +211 i386 getresgid32 sys_getresgid +212 i386 chown32 sys_chown +213 i386 setuid32 sys_setuid +214 i386 setgid32 sys_setgid +215 i386 setfsuid32 sys_setfsuid +216 i386 setfsgid32 sys_setfsgid +217 i386 pivot_root sys_pivot_root +218 i386 mincore sys_mincore +219 i386 madvise sys_madvise +220 i386 getdents64 sys_getdents64 +221 i386 fcntl64 sys_fcntl64 compat_sys_fcntl64 +# 222 is unused +# 223 is unused +224 i386 gettid sys_gettid +225 i386 readahead sys_ia32_readahead +226 i386 setxattr sys_setxattr +227 i386 lsetxattr sys_lsetxattr +228 i386 fsetxattr sys_fsetxattr +229 i386 getxattr sys_getxattr +230 i386 lgetxattr sys_lgetxattr +231 i386 fgetxattr sys_fgetxattr +232 i386 listxattr sys_listxattr +233 i386 llistxattr sys_llistxattr +234 i386 flistxattr sys_flistxattr +235 i386 removexattr sys_removexattr +236 i386 lremovexattr sys_lremovexattr +237 i386 fremovexattr sys_fremovexattr +238 i386 tkill sys_tkill +239 i386 sendfile64 sys_sendfile64 +240 i386 futex sys_futex_time32 +241 i386 sched_setaffinity sys_sched_setaffinity compat_sys_sched_setaffinity +242 i386 sched_getaffinity sys_sched_getaffinity compat_sys_sched_getaffinity +243 i386 set_thread_area sys_set_thread_area +244 i386 get_thread_area sys_get_thread_area +245 i386 io_setup sys_io_setup compat_sys_io_setup +246 i386 io_destroy sys_io_destroy +247 i386 io_getevents sys_io_getevents_time32 +248 i386 io_submit sys_io_submit compat_sys_io_submit +249 i386 io_cancel sys_io_cancel +250 i386 fadvise64 sys_ia32_fadvise64 +# 251 is available for reuse (was briefly sys_set_zone_reclaim) +252 i386 exit_group sys_exit_group - noreturn +253 i386 lookup_dcookie +254 i386 epoll_create sys_epoll_create +255 i386 epoll_ctl sys_epoll_ctl +256 i386 epoll_wait sys_epoll_wait +257 i386 remap_file_pages sys_remap_file_pages +258 i386 set_tid_address sys_set_tid_address +259 i386 timer_create sys_timer_create compat_sys_timer_create +260 i386 timer_settime sys_timer_settime32 +261 i386 timer_gettime sys_timer_gettime32 +262 i386 timer_getoverrun sys_timer_getoverrun +263 i386 timer_delete sys_timer_delete +264 i386 clock_settime sys_clock_settime32 +265 i386 clock_gettime sys_clock_gettime32 +266 i386 clock_getres sys_clock_getres_time32 +267 i386 clock_nanosleep sys_clock_nanosleep_time32 +268 i386 statfs64 sys_statfs64 compat_sys_statfs64 +269 i386 fstatfs64 sys_fstatfs64 compat_sys_fstatfs64 +270 i386 tgkill sys_tgkill +271 i386 utimes sys_utimes_time32 +272 i386 fadvise64_64 sys_ia32_fadvise64_64 +273 i386 vserver +274 i386 mbind sys_mbind +275 i386 get_mempolicy sys_get_mempolicy +276 i386 set_mempolicy sys_set_mempolicy +277 i386 mq_open sys_mq_open compat_sys_mq_open +278 i386 mq_unlink sys_mq_unlink +279 i386 mq_timedsend sys_mq_timedsend_time32 +280 i386 mq_timedreceive sys_mq_timedreceive_time32 +281 i386 mq_notify sys_mq_notify compat_sys_mq_notify +282 i386 mq_getsetattr sys_mq_getsetattr compat_sys_mq_getsetattr +283 i386 kexec_load sys_kexec_load compat_sys_kexec_load +284 i386 waitid sys_waitid compat_sys_waitid +# 285 sys_setaltroot +286 i386 add_key sys_add_key +287 i386 request_key sys_request_key +288 i386 keyctl sys_keyctl compat_sys_keyctl +289 i386 ioprio_set sys_ioprio_set +290 i386 ioprio_get sys_ioprio_get +291 i386 inotify_init sys_inotify_init +292 i386 inotify_add_watch sys_inotify_add_watch +293 i386 inotify_rm_watch sys_inotify_rm_watch +294 i386 migrate_pages sys_migrate_pages +295 i386 openat sys_openat compat_sys_openat +296 i386 mkdirat sys_mkdirat +297 i386 mknodat sys_mknodat +298 i386 fchownat sys_fchownat +299 i386 futimesat sys_futimesat_time32 +300 i386 fstatat64 sys_fstatat64 compat_sys_ia32_fstatat64 +301 i386 unlinkat sys_unlinkat +302 i386 renameat sys_renameat +303 i386 linkat sys_linkat +304 i386 symlinkat sys_symlinkat +305 i386 readlinkat sys_readlinkat +306 i386 fchmodat sys_fchmodat +307 i386 faccessat sys_faccessat +308 i386 pselect6 sys_pselect6_time32 compat_sys_pselect6_time32 +309 i386 ppoll sys_ppoll_time32 compat_sys_ppoll_time32 +310 i386 unshare sys_unshare +311 i386 set_robust_list sys_set_robust_list compat_sys_set_robust_list +312 i386 get_robust_list sys_get_robust_list compat_sys_get_robust_list +313 i386 splice sys_splice +314 i386 sync_file_range sys_ia32_sync_file_range +315 i386 tee sys_tee +316 i386 vmsplice sys_vmsplice +317 i386 move_pages sys_move_pages +318 i386 getcpu sys_getcpu +319 i386 epoll_pwait sys_epoll_pwait +320 i386 utimensat sys_utimensat_time32 +321 i386 signalfd sys_signalfd compat_sys_signalfd +322 i386 timerfd_create sys_timerfd_create +323 i386 eventfd sys_eventfd +324 i386 fallocate sys_ia32_fallocate +325 i386 timerfd_settime sys_timerfd_settime32 +326 i386 timerfd_gettime sys_timerfd_gettime32 +327 i386 signalfd4 sys_signalfd4 compat_sys_signalfd4 +328 i386 eventfd2 sys_eventfd2 +329 i386 epoll_create1 sys_epoll_create1 +330 i386 dup3 sys_dup3 +331 i386 pipe2 sys_pipe2 +332 i386 inotify_init1 sys_inotify_init1 +333 i386 preadv sys_preadv compat_sys_preadv +334 i386 pwritev sys_pwritev compat_sys_pwritev +335 i386 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +336 i386 perf_event_open sys_perf_event_open +337 i386 recvmmsg sys_recvmmsg_time32 compat_sys_recvmmsg_time32 +338 i386 fanotify_init sys_fanotify_init +339 i386 fanotify_mark sys_fanotify_mark compat_sys_fanotify_mark +340 i386 prlimit64 sys_prlimit64 +341 i386 name_to_handle_at sys_name_to_handle_at +342 i386 open_by_handle_at sys_open_by_handle_at compat_sys_open_by_handle_at +343 i386 clock_adjtime sys_clock_adjtime32 +344 i386 syncfs sys_syncfs +345 i386 sendmmsg sys_sendmmsg compat_sys_sendmmsg +346 i386 setns sys_setns +347 i386 process_vm_readv sys_process_vm_readv +348 i386 process_vm_writev sys_process_vm_writev +349 i386 kcmp sys_kcmp +350 i386 finit_module sys_finit_module +351 i386 sched_setattr sys_sched_setattr +352 i386 sched_getattr sys_sched_getattr +353 i386 renameat2 sys_renameat2 +354 i386 seccomp sys_seccomp +355 i386 getrandom sys_getrandom +356 i386 memfd_create sys_memfd_create +357 i386 bpf sys_bpf +358 i386 execveat sys_execveat compat_sys_execveat +359 i386 socket sys_socket +360 i386 socketpair sys_socketpair +361 i386 bind sys_bind +362 i386 connect sys_connect +363 i386 listen sys_listen +364 i386 accept4 sys_accept4 +365 i386 getsockopt sys_getsockopt sys_getsockopt +366 i386 setsockopt sys_setsockopt sys_setsockopt +367 i386 getsockname sys_getsockname +368 i386 getpeername sys_getpeername +369 i386 sendto sys_sendto +370 i386 sendmsg sys_sendmsg compat_sys_sendmsg +371 i386 recvfrom sys_recvfrom compat_sys_recvfrom +372 i386 recvmsg sys_recvmsg compat_sys_recvmsg +373 i386 shutdown sys_shutdown +374 i386 userfaultfd sys_userfaultfd +375 i386 membarrier sys_membarrier +376 i386 mlock2 sys_mlock2 +377 i386 copy_file_range sys_copy_file_range +378 i386 preadv2 sys_preadv2 compat_sys_preadv2 +379 i386 pwritev2 sys_pwritev2 compat_sys_pwritev2 +380 i386 pkey_mprotect sys_pkey_mprotect +381 i386 pkey_alloc sys_pkey_alloc +382 i386 pkey_free sys_pkey_free +383 i386 statx sys_statx +384 i386 arch_prctl sys_arch_prctl +385 i386 io_pgetevents sys_io_pgetevents_time32 compat_sys_io_pgetevents +386 i386 rseq sys_rseq +393 i386 semget sys_semget +394 i386 semctl sys_semctl compat_sys_semctl +395 i386 shmget sys_shmget +396 i386 shmctl sys_shmctl compat_sys_shmctl +397 i386 shmat sys_shmat compat_sys_shmat +398 i386 shmdt sys_shmdt +399 i386 msgget sys_msgget +400 i386 msgsnd sys_msgsnd compat_sys_msgsnd +401 i386 msgrcv sys_msgrcv compat_sys_msgrcv +402 i386 msgctl sys_msgctl compat_sys_msgctl +403 i386 clock_gettime64 sys_clock_gettime +404 i386 clock_settime64 sys_clock_settime +405 i386 clock_adjtime64 sys_clock_adjtime +406 i386 clock_getres_time64 sys_clock_getres +407 i386 clock_nanosleep_time64 sys_clock_nanosleep +408 i386 timer_gettime64 sys_timer_gettime +409 i386 timer_settime64 sys_timer_settime +410 i386 timerfd_gettime64 sys_timerfd_gettime +411 i386 timerfd_settime64 sys_timerfd_settime +412 i386 utimensat_time64 sys_utimensat +413 i386 pselect6_time64 sys_pselect6 compat_sys_pselect6_time64 +414 i386 ppoll_time64 sys_ppoll compat_sys_ppoll_time64 +416 i386 io_pgetevents_time64 sys_io_pgetevents compat_sys_io_pgetevents_time64 +417 i386 recvmmsg_time64 sys_recvmmsg compat_sys_recvmmsg_time64 +418 i386 mq_timedsend_time64 sys_mq_timedsend +419 i386 mq_timedreceive_time64 sys_mq_timedreceive +420 i386 semtimedop_time64 sys_semtimedop +421 i386 rt_sigtimedwait_time64 sys_rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +422 i386 futex_time64 sys_futex +423 i386 sched_rr_get_interval_time64 sys_sched_rr_get_interval +424 i386 pidfd_send_signal sys_pidfd_send_signal +425 i386 io_uring_setup sys_io_uring_setup +426 i386 io_uring_enter sys_io_uring_enter +427 i386 io_uring_register sys_io_uring_register +428 i386 open_tree sys_open_tree +429 i386 move_mount sys_move_mount +430 i386 fsopen sys_fsopen +431 i386 fsconfig sys_fsconfig +432 i386 fsmount sys_fsmount +433 i386 fspick sys_fspick +434 i386 pidfd_open sys_pidfd_open +435 i386 clone3 sys_clone3 +436 i386 close_range sys_close_range +437 i386 openat2 sys_openat2 +438 i386 pidfd_getfd sys_pidfd_getfd +439 i386 faccessat2 sys_faccessat2 +440 i386 process_madvise sys_process_madvise +441 i386 epoll_pwait2 sys_epoll_pwait2 compat_sys_epoll_pwait2 +442 i386 mount_setattr sys_mount_setattr +443 i386 quotactl_fd sys_quotactl_fd +444 i386 landlock_create_ruleset sys_landlock_create_ruleset +445 i386 landlock_add_rule sys_landlock_add_rule +446 i386 landlock_restrict_self sys_landlock_restrict_self +447 i386 memfd_secret sys_memfd_secret +448 i386 process_mrelease sys_process_mrelease +449 i386 futex_waitv sys_futex_waitv +450 i386 set_mempolicy_home_node sys_set_mempolicy_home_node +451 i386 cachestat sys_cachestat +452 i386 fchmodat2 sys_fchmodat2 +453 i386 map_shadow_stack sys_map_shadow_stack +454 i386 futex_wake sys_futex_wake +455 i386 futex_wait sys_futex_wait +456 i386 futex_requeue sys_futex_requeue +457 i386 statmount sys_statmount +458 i386 listmount sys_listmount +459 i386 lsm_get_self_attr sys_lsm_get_self_attr +460 i386 lsm_set_self_attr sys_lsm_set_self_attr +461 i386 lsm_list_modules sys_lsm_list_modules +462 i386 mseal sys_mseal +463 i386 setxattrat sys_setxattrat +464 i386 getxattrat sys_getxattrat +465 i386 listxattrat sys_listxattrat +466 i386 removexattrat sys_removexattrat +467 i386 open_tree_attr sys_open_tree_attr +468 i386 file_getattr sys_file_getattr +469 i386 file_setattr sys_file_setattr +470 i386 listns sys_listns +471 i386 rseq_slice_yield sys_rseq_slice_yield diff --git a/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.meta.toml b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.meta.toml new file mode 100644 index 0000000..4d4fe0c --- /dev/null +++ b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.meta.toml @@ -0,0 +1,26 @@ +source = "pinned" +evidence = true + +describes = "the x86-64 system call numbers, here so the i386 ones can be compared against them" +path = "arch/x86/entry/syscalls/syscall_64.tbl" +excerpt = false + +kernel = "7.2.2" +arch = "x86_64" +tarball_sha256 = "7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83" +taken = "2026-09-06" + +# The pinned box does not run this table. It is committed anyway, because the reader's laptop +# almost certainly does, and a number that differs between the two is the fastest way to make the +# point that a syscall number with no architecture attached is not an identifier. +# +# This file has three abis in it where the 32 bit one has a single abi, and the same name turns up +# under more than one of them. rt_sigaction is 13 under `64` and 512 under `x32`. + +rows = 421 +abis = ["common", "64", "x32"] +reserved_rows = 16 +holes = 127 +write_number = 1 +read_number = 0 +duplicate_names = ["rt_sigaction", "rt_sigreturn", "ioctl", "readv", "writev"] diff --git a/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.tbl b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.tbl new file mode 100644 index 0000000..524155d --- /dev/null +++ b/corpora/source/pinned/arch/x86/entry/syscalls/syscall_64.tbl @@ -0,0 +1,443 @@ +# SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +# +# 64-bit system call numbers and entry vectors +# +# The format is: +# [ [noreturn]] +# +# The __x64_sys_*() stubs are created on-the-fly for sys_*() system calls +# +# The abi is "common", "64" or "x32" for this file. +# +0 common read sys_read +1 common write sys_write +2 common open sys_open +3 common close sys_close +4 common stat sys_newstat +5 common fstat sys_newfstat +6 common lstat sys_newlstat +7 common poll sys_poll +8 common lseek sys_lseek +9 common mmap sys_mmap +10 common mprotect sys_mprotect +11 common munmap sys_munmap +12 common brk sys_brk +13 64 rt_sigaction sys_rt_sigaction +14 common rt_sigprocmask sys_rt_sigprocmask +15 64 rt_sigreturn sys_rt_sigreturn +16 64 ioctl sys_ioctl +17 common pread64 sys_pread64 +18 common pwrite64 sys_pwrite64 +19 64 readv sys_readv +20 64 writev sys_writev +21 common access sys_access +22 common pipe sys_pipe +23 common select sys_select +24 common sched_yield sys_sched_yield +25 common mremap sys_mremap +26 common msync sys_msync +27 common mincore sys_mincore +28 common madvise sys_madvise +29 common shmget sys_shmget +30 common shmat sys_shmat +31 common shmctl sys_shmctl +32 common dup sys_dup +33 common dup2 sys_dup2 +34 common pause sys_pause +35 common nanosleep sys_nanosleep +36 common getitimer sys_getitimer +37 common alarm sys_alarm +38 common setitimer sys_setitimer +39 common getpid sys_getpid +40 common sendfile sys_sendfile64 +41 common socket sys_socket +42 common connect sys_connect +43 common accept sys_accept +44 common sendto sys_sendto +45 64 recvfrom sys_recvfrom +46 64 sendmsg sys_sendmsg +47 64 recvmsg sys_recvmsg +48 common shutdown sys_shutdown +49 common bind sys_bind +50 common listen sys_listen +51 common getsockname sys_getsockname +52 common getpeername sys_getpeername +53 common socketpair sys_socketpair +54 64 setsockopt sys_setsockopt +55 64 getsockopt sys_getsockopt +56 common clone sys_clone +57 common fork sys_fork +58 common vfork sys_vfork +59 64 execve sys_execve +60 common exit sys_exit - noreturn +61 common wait4 sys_wait4 +62 common kill sys_kill +63 common uname sys_newuname +64 common semget sys_semget +65 common semop sys_semop +66 common semctl sys_semctl +67 common shmdt sys_shmdt +68 common msgget sys_msgget +69 common msgsnd sys_msgsnd +70 common msgrcv sys_msgrcv +71 common msgctl sys_msgctl +72 common fcntl sys_fcntl +73 common flock sys_flock +74 common fsync sys_fsync +75 common fdatasync sys_fdatasync +76 common truncate sys_truncate +77 common ftruncate sys_ftruncate +78 common getdents sys_getdents +79 common getcwd sys_getcwd +80 common chdir sys_chdir +81 common fchdir sys_fchdir +82 common rename sys_rename +83 common mkdir sys_mkdir +84 common rmdir sys_rmdir +85 common creat sys_creat +86 common link sys_link +87 common unlink sys_unlink +88 common symlink sys_symlink +89 common readlink sys_readlink +90 common chmod sys_chmod +91 common fchmod sys_fchmod +92 common chown sys_chown +93 common fchown sys_fchown +94 common lchown sys_lchown +95 common umask sys_umask +96 common gettimeofday sys_gettimeofday +97 common getrlimit sys_getrlimit +98 common getrusage sys_getrusage +99 common sysinfo sys_sysinfo +100 common times sys_times +101 64 ptrace sys_ptrace +102 common getuid sys_getuid +103 common syslog sys_syslog +104 common getgid sys_getgid +105 common setuid sys_setuid +106 common setgid sys_setgid +107 common geteuid sys_geteuid +108 common getegid sys_getegid +109 common setpgid sys_setpgid +110 common getppid sys_getppid +111 common getpgrp sys_getpgrp +112 common setsid sys_setsid +113 common setreuid sys_setreuid +114 common setregid sys_setregid +115 common getgroups sys_getgroups +116 common setgroups sys_setgroups +117 common setresuid sys_setresuid +118 common getresuid sys_getresuid +119 common setresgid sys_setresgid +120 common getresgid sys_getresgid +121 common getpgid sys_getpgid +122 common setfsuid sys_setfsuid +123 common setfsgid sys_setfsgid +124 common getsid sys_getsid +125 common capget sys_capget +126 common capset sys_capset +127 64 rt_sigpending sys_rt_sigpending +128 64 rt_sigtimedwait sys_rt_sigtimedwait +129 64 rt_sigqueueinfo sys_rt_sigqueueinfo +130 common rt_sigsuspend sys_rt_sigsuspend +131 64 sigaltstack sys_sigaltstack +132 common utime sys_utime +133 common mknod sys_mknod +134 64 uselib +135 common personality sys_personality +136 common ustat sys_ustat +137 common statfs sys_statfs +138 common fstatfs sys_fstatfs +139 common sysfs sys_sysfs +140 common getpriority sys_getpriority +141 common setpriority sys_setpriority +142 common sched_setparam sys_sched_setparam +143 common sched_getparam sys_sched_getparam +144 common sched_setscheduler sys_sched_setscheduler +145 common sched_getscheduler sys_sched_getscheduler +146 common sched_get_priority_max sys_sched_get_priority_max +147 common sched_get_priority_min sys_sched_get_priority_min +148 common sched_rr_get_interval sys_sched_rr_get_interval +149 common mlock sys_mlock +150 common munlock sys_munlock +151 common mlockall sys_mlockall +152 common munlockall sys_munlockall +153 common vhangup sys_vhangup +154 common modify_ldt sys_modify_ldt +155 common pivot_root sys_pivot_root +156 64 _sysctl sys_ni_syscall +157 common prctl sys_prctl +158 common arch_prctl sys_arch_prctl +159 common adjtimex sys_adjtimex +160 common setrlimit sys_setrlimit +161 common chroot sys_chroot +162 common sync sys_sync +163 common acct sys_acct +164 common settimeofday sys_settimeofday +165 common mount sys_mount +166 common umount2 sys_umount +167 common swapon sys_swapon +168 common swapoff sys_swapoff +169 common reboot sys_reboot +170 common sethostname sys_sethostname +171 common setdomainname sys_setdomainname +172 common iopl sys_iopl +173 common ioperm sys_ioperm +174 64 create_module +175 common init_module sys_init_module +176 common delete_module sys_delete_module +177 64 get_kernel_syms +178 64 query_module +179 common quotactl sys_quotactl +180 64 nfsservctl +181 common getpmsg +182 common putpmsg +183 common afs_syscall +184 common tuxcall +185 common security +186 common gettid sys_gettid +187 common readahead sys_readahead +188 common setxattr sys_setxattr +189 common lsetxattr sys_lsetxattr +190 common fsetxattr sys_fsetxattr +191 common getxattr sys_getxattr +192 common lgetxattr sys_lgetxattr +193 common fgetxattr sys_fgetxattr +194 common listxattr sys_listxattr +195 common llistxattr sys_llistxattr +196 common flistxattr sys_flistxattr +197 common removexattr sys_removexattr +198 common lremovexattr sys_lremovexattr +199 common fremovexattr sys_fremovexattr +200 common tkill sys_tkill +201 common time sys_time +202 common futex sys_futex +203 common sched_setaffinity sys_sched_setaffinity +204 common sched_getaffinity sys_sched_getaffinity +205 64 set_thread_area +206 64 io_setup sys_io_setup +207 common io_destroy sys_io_destroy +208 common io_getevents sys_io_getevents +209 64 io_submit sys_io_submit +210 common io_cancel sys_io_cancel +211 64 get_thread_area +212 common lookup_dcookie +213 common epoll_create sys_epoll_create +214 64 epoll_ctl_old +215 64 epoll_wait_old +216 common remap_file_pages sys_remap_file_pages +217 common getdents64 sys_getdents64 +218 common set_tid_address sys_set_tid_address +219 common restart_syscall sys_restart_syscall +220 common semtimedop sys_semtimedop +221 common fadvise64 sys_fadvise64 +222 64 timer_create sys_timer_create +223 common timer_settime sys_timer_settime +224 common timer_gettime sys_timer_gettime +225 common timer_getoverrun sys_timer_getoverrun +226 common timer_delete sys_timer_delete +227 common clock_settime sys_clock_settime +228 common clock_gettime sys_clock_gettime +229 common clock_getres sys_clock_getres +230 common clock_nanosleep sys_clock_nanosleep +231 common exit_group sys_exit_group - noreturn +232 common epoll_wait sys_epoll_wait +233 common epoll_ctl sys_epoll_ctl +234 common tgkill sys_tgkill +235 common utimes sys_utimes +236 64 vserver +237 common mbind sys_mbind +238 common set_mempolicy sys_set_mempolicy +239 common get_mempolicy sys_get_mempolicy +240 common mq_open sys_mq_open +241 common mq_unlink sys_mq_unlink +242 common mq_timedsend sys_mq_timedsend +243 common mq_timedreceive sys_mq_timedreceive +244 64 mq_notify sys_mq_notify +245 common mq_getsetattr sys_mq_getsetattr +246 64 kexec_load sys_kexec_load +247 64 waitid sys_waitid +248 common add_key sys_add_key +249 common request_key sys_request_key +250 common keyctl sys_keyctl +251 common ioprio_set sys_ioprio_set +252 common ioprio_get sys_ioprio_get +253 common inotify_init sys_inotify_init +254 common inotify_add_watch sys_inotify_add_watch +255 common inotify_rm_watch sys_inotify_rm_watch +256 common migrate_pages sys_migrate_pages +257 common openat sys_openat +258 common mkdirat sys_mkdirat +259 common mknodat sys_mknodat +260 common fchownat sys_fchownat +261 common futimesat sys_futimesat +262 common newfstatat sys_newfstatat +263 common unlinkat sys_unlinkat +264 common renameat sys_renameat +265 common linkat sys_linkat +266 common symlinkat sys_symlinkat +267 common readlinkat sys_readlinkat +268 common fchmodat sys_fchmodat +269 common faccessat sys_faccessat +270 common pselect6 sys_pselect6 +271 common ppoll sys_ppoll +272 common unshare sys_unshare +273 64 set_robust_list sys_set_robust_list +274 64 get_robust_list sys_get_robust_list +275 common splice sys_splice +276 common tee sys_tee +277 common sync_file_range sys_sync_file_range +278 64 vmsplice sys_vmsplice +279 64 move_pages sys_move_pages +280 common utimensat sys_utimensat +281 common epoll_pwait sys_epoll_pwait +282 common signalfd sys_signalfd +283 common timerfd_create sys_timerfd_create +284 common eventfd sys_eventfd +285 common fallocate sys_fallocate +286 common timerfd_settime sys_timerfd_settime +287 common timerfd_gettime sys_timerfd_gettime +288 common accept4 sys_accept4 +289 common signalfd4 sys_signalfd4 +290 common eventfd2 sys_eventfd2 +291 common epoll_create1 sys_epoll_create1 +292 common dup3 sys_dup3 +293 common pipe2 sys_pipe2 +294 common inotify_init1 sys_inotify_init1 +295 64 preadv sys_preadv +296 64 pwritev sys_pwritev +297 64 rt_tgsigqueueinfo sys_rt_tgsigqueueinfo +298 common perf_event_open sys_perf_event_open +299 64 recvmmsg sys_recvmmsg +300 common fanotify_init sys_fanotify_init +301 common fanotify_mark sys_fanotify_mark +302 common prlimit64 sys_prlimit64 +303 common name_to_handle_at sys_name_to_handle_at +304 common open_by_handle_at sys_open_by_handle_at +305 common clock_adjtime sys_clock_adjtime +306 common syncfs sys_syncfs +307 64 sendmmsg sys_sendmmsg +308 common setns sys_setns +309 common getcpu sys_getcpu +310 64 process_vm_readv sys_process_vm_readv +311 64 process_vm_writev sys_process_vm_writev +312 common kcmp sys_kcmp +313 common finit_module sys_finit_module +314 common sched_setattr sys_sched_setattr +315 common sched_getattr sys_sched_getattr +316 common renameat2 sys_renameat2 +317 common seccomp sys_seccomp +318 common getrandom sys_getrandom +319 common memfd_create sys_memfd_create +320 common kexec_file_load sys_kexec_file_load +321 common bpf sys_bpf +322 64 execveat sys_execveat +323 common userfaultfd sys_userfaultfd +324 common membarrier sys_membarrier +325 common mlock2 sys_mlock2 +326 common copy_file_range sys_copy_file_range +327 64 preadv2 sys_preadv2 +328 64 pwritev2 sys_pwritev2 +329 common pkey_mprotect sys_pkey_mprotect +330 common pkey_alloc sys_pkey_alloc +331 common pkey_free sys_pkey_free +332 common statx sys_statx +333 common io_pgetevents sys_io_pgetevents +334 common rseq sys_rseq +335 common uretprobe sys_uretprobe +336 common uprobe sys_uprobe +# don't use numbers 387 through 423, add new calls after the last +# 'common' entry +424 common pidfd_send_signal sys_pidfd_send_signal +425 common io_uring_setup sys_io_uring_setup +426 common io_uring_enter sys_io_uring_enter +427 common io_uring_register sys_io_uring_register +428 common open_tree sys_open_tree +429 common move_mount sys_move_mount +430 common fsopen sys_fsopen +431 common fsconfig sys_fsconfig +432 common fsmount sys_fsmount +433 common fspick sys_fspick +434 common pidfd_open sys_pidfd_open +435 common clone3 sys_clone3 +436 common close_range sys_close_range +437 common openat2 sys_openat2 +438 common pidfd_getfd sys_pidfd_getfd +439 common faccessat2 sys_faccessat2 +440 common process_madvise sys_process_madvise +441 common epoll_pwait2 sys_epoll_pwait2 +442 common mount_setattr sys_mount_setattr +443 common quotactl_fd sys_quotactl_fd +444 common landlock_create_ruleset sys_landlock_create_ruleset +445 common landlock_add_rule sys_landlock_add_rule +446 common landlock_restrict_self sys_landlock_restrict_self +447 common memfd_secret sys_memfd_secret +448 common process_mrelease sys_process_mrelease +449 common futex_waitv sys_futex_waitv +450 common set_mempolicy_home_node sys_set_mempolicy_home_node +451 common cachestat sys_cachestat +452 common fchmodat2 sys_fchmodat2 +453 common map_shadow_stack sys_map_shadow_stack +454 common futex_wake sys_futex_wake +455 common futex_wait sys_futex_wait +456 common futex_requeue sys_futex_requeue +457 common statmount sys_statmount +458 common listmount sys_listmount +459 common lsm_get_self_attr sys_lsm_get_self_attr +460 common lsm_set_self_attr sys_lsm_set_self_attr +461 common lsm_list_modules sys_lsm_list_modules +462 common mseal sys_mseal +463 common setxattrat sys_setxattrat +464 common getxattrat sys_getxattrat +465 common listxattrat sys_listxattrat +466 common removexattrat sys_removexattrat +467 common open_tree_attr sys_open_tree_attr +468 common file_getattr sys_file_getattr +469 common file_setattr sys_file_setattr +470 common listns sys_listns +471 common rseq_slice_yield sys_rseq_slice_yield + +# +# Due to a historical design error, certain syscalls are numbered differently +# in x32 as compared to native x86_64. These syscalls have numbers 512-547. +# Do not add new syscalls to this range. Numbers 548 and above are available +# for non-x32 use. +# +512 x32 rt_sigaction compat_sys_rt_sigaction +513 x32 rt_sigreturn compat_sys_x32_rt_sigreturn +514 x32 ioctl compat_sys_ioctl +515 x32 readv sys_readv +516 x32 writev sys_writev +517 x32 recvfrom compat_sys_recvfrom +518 x32 sendmsg compat_sys_sendmsg +519 x32 recvmsg compat_sys_recvmsg +520 x32 execve compat_sys_execve +521 x32 ptrace compat_sys_ptrace +522 x32 rt_sigpending compat_sys_rt_sigpending +523 x32 rt_sigtimedwait compat_sys_rt_sigtimedwait_time64 +524 x32 rt_sigqueueinfo compat_sys_rt_sigqueueinfo +525 x32 sigaltstack compat_sys_sigaltstack +526 x32 timer_create compat_sys_timer_create +527 x32 mq_notify compat_sys_mq_notify +528 x32 kexec_load compat_sys_kexec_load +529 x32 waitid compat_sys_waitid +530 x32 set_robust_list compat_sys_set_robust_list +531 x32 get_robust_list compat_sys_get_robust_list +532 x32 vmsplice sys_vmsplice +533 x32 move_pages sys_move_pages +534 x32 preadv compat_sys_preadv64 +535 x32 pwritev compat_sys_pwritev64 +536 x32 rt_tgsigqueueinfo compat_sys_rt_tgsigqueueinfo +537 x32 recvmmsg compat_sys_recvmmsg_time64 +538 x32 sendmmsg compat_sys_sendmmsg +539 x32 process_vm_readv sys_process_vm_readv +540 x32 process_vm_writev sys_process_vm_writev +541 x32 setsockopt sys_setsockopt +542 x32 getsockopt sys_getsockopt +543 x32 io_setup compat_sys_io_setup +544 x32 io_submit compat_sys_io_submit +545 x32 execveat compat_sys_execveat +546 x32 preadv2 compat_sys_preadv64v2 +547 x32 pwritev2 compat_sys_pwritev64v2 +# This is the end of the legacy x32 range. Numbers 548 and above are +# not special and are not to be used for x32-specific syscalls. diff --git a/corpora/source/pinned/fs/read_write.c b/corpora/source/pinned/fs/read_write.c new file mode 100644 index 0000000..e8c14e2 --- /dev/null +++ b/corpora/source/pinned/fs/read_write.c @@ -0,0 +1,1822 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * linux/fs/read_write.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "internal.h" + +#include +#include + +const struct file_operations generic_ro_fops = { + .llseek = generic_file_llseek, + .read_iter = generic_file_read_iter, + .mmap_prepare = generic_file_readonly_mmap_prepare, + .splice_read = filemap_splice_read, + .setlease = generic_setlease, +}; + +EXPORT_SYMBOL(generic_ro_fops); + +static inline bool unsigned_offsets(struct file *file) +{ + return file->f_op->fop_flags & FOP_UNSIGNED_OFFSET; +} + +/** + * vfs_setpos_cookie - update the file offset for lseek and reset cookie + * @file: file structure in question + * @offset: file offset to seek to + * @maxsize: maximum file size + * @cookie: cookie to reset + * + * Update the file offset to the value specified by @offset if the given + * offset is valid and it is not equal to the current file offset and + * reset the specified cookie to indicate that a seek happened. + * + * Return the specified offset on success and -EINVAL on invalid offset. + */ +static loff_t vfs_setpos_cookie(struct file *file, loff_t offset, + loff_t maxsize, u64 *cookie) +{ + if (offset < 0 && !unsigned_offsets(file)) + return -EINVAL; + if (offset > maxsize) + return -EINVAL; + + if (offset != file->f_pos) { + file->f_pos = offset; + if (cookie) + *cookie = 0; + } + return offset; +} + +/** + * vfs_setpos - update the file offset for lseek + * @file: file structure in question + * @offset: file offset to seek to + * @maxsize: maximum file size + * + * This is a low-level filesystem helper for updating the file offset to + * the value specified by @offset if the given offset is valid and it is + * not equal to the current file offset. + * + * Return the specified offset on success and -EINVAL on invalid offset. + */ +loff_t vfs_setpos(struct file *file, loff_t offset, loff_t maxsize) +{ + return vfs_setpos_cookie(file, offset, maxsize, NULL); +} +EXPORT_SYMBOL(vfs_setpos); + +/** + * must_set_pos - check whether f_pos has to be updated + * @file: file to seek on + * @offset: offset to use + * @whence: type of seek operation + * @eof: end of file + * + * Check whether f_pos needs to be updated and update @offset according + * to @whence. + * + * Return: 0 if f_pos doesn't need to be updated, 1 if f_pos has to be + * updated, and negative error code on failure. + */ +static int must_set_pos(struct file *file, loff_t *offset, int whence, loff_t eof) +{ + switch (whence) { + case SEEK_END: + *offset += eof; + break; + case SEEK_CUR: + /* + * Here we special-case the lseek(fd, 0, SEEK_CUR) + * position-querying operation. Avoid rewriting the "same" + * f_pos value back to the file because a concurrent read(), + * write() or lseek() might have altered it + */ + if (*offset == 0) { + *offset = file->f_pos; + return 0; + } + break; + case SEEK_DATA: + /* + * In the generic case the entire file is data, so as long as + * offset isn't at the end of the file then the offset is data. + */ + if ((unsigned long long)*offset >= eof) + return -ENXIO; + break; + case SEEK_HOLE: + /* + * There is a virtual hole at the end of the file, so as long as + * offset isn't i_size or larger, return i_size. + */ + if ((unsigned long long)*offset >= eof) + return -ENXIO; + *offset = eof; + break; + } + + return 1; +} + +/** + * generic_file_llseek_size - generic llseek implementation for regular files + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @maxsize: max size of this file in file system + * @eof: offset used for SEEK_END position + * + * This is a variant of generic_file_llseek that allows passing in a custom + * maximum file size and a custom EOF position, for e.g. hashed directories + * + * Synchronization: + * SEEK_SET and SEEK_END are unsynchronized (but atomic on 64bit platforms) + * SEEK_CUR is synchronized against other SEEK_CURs, but not read/writes. + * read/writes behave like SEEK_SET against seeks. + */ +loff_t +generic_file_llseek_size(struct file *file, loff_t offset, int whence, + loff_t maxsize, loff_t eof) +{ + int ret; + + ret = must_set_pos(file, &offset, whence, eof); + if (ret < 0) + return ret; + if (ret == 0) + return offset; + + if (whence == SEEK_CUR) { + /* + * If the file requires locking via f_pos_lock we know + * that mutual exclusion for SEEK_CUR on the same file + * is guaranteed. If the file isn't locked, we take + * f_lock to protect against f_pos races with other + * SEEK_CURs. + */ + if (file_seek_cur_needs_f_lock(file)) { + guard(spinlock)(&file->f_lock); + return vfs_setpos(file, file->f_pos + offset, maxsize); + } + return vfs_setpos(file, file->f_pos + offset, maxsize); + } + + return vfs_setpos(file, offset, maxsize); +} +EXPORT_SYMBOL(generic_file_llseek_size); + +/** + * generic_llseek_cookie - versioned llseek implementation + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @cookie: cookie to update + * + * See generic_file_llseek for a general description and locking assumptions. + * + * In contrast to generic_file_llseek, this function also resets a + * specified cookie to indicate a seek took place. + */ +loff_t generic_llseek_cookie(struct file *file, loff_t offset, int whence, + u64 *cookie) +{ + struct inode *inode = file->f_mapping->host; + loff_t maxsize = inode->i_sb->s_maxbytes; + loff_t eof = i_size_read(inode); + int ret; + + if (WARN_ON_ONCE(!cookie)) + return -EINVAL; + + /* + * Require that this is only used for directories that guarantee + * synchronization between readdir and seek so that an update to + * @cookie is correctly synchronized with concurrent readdir. + */ + if (WARN_ON_ONCE(!(file->f_mode & FMODE_ATOMIC_POS))) + return -EINVAL; + + ret = must_set_pos(file, &offset, whence, eof); + if (ret < 0) + return ret; + if (ret == 0) + return offset; + + /* No need to hold f_lock because we know that f_pos_lock is held. */ + if (whence == SEEK_CUR) + return vfs_setpos_cookie(file, file->f_pos + offset, maxsize, cookie); + + return vfs_setpos_cookie(file, offset, maxsize, cookie); +} +EXPORT_SYMBOL(generic_llseek_cookie); + +/** + * generic_file_llseek - generic llseek implementation for regular files + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + * This is a generic implementation of ->llseek useable for all normal local + * filesystems. It just updates the file offset to the value specified by + * @offset and @whence. + */ +loff_t generic_file_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file->f_mapping->host; + + return generic_file_llseek_size(file, offset, whence, + inode->i_sb->s_maxbytes, + i_size_read(inode)); +} +EXPORT_SYMBOL(generic_file_llseek); + +/** + * fixed_size_llseek - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @size: size of the file + * + */ +loff_t fixed_size_llseek(struct file *file, loff_t offset, int whence, loff_t size) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: case SEEK_END: + return generic_file_llseek_size(file, offset, whence, + size, size); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(fixed_size_llseek); + +/** + * no_seek_end_llseek - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + */ +loff_t no_seek_end_llseek(struct file *file, loff_t offset, int whence) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: + return generic_file_llseek_size(file, offset, whence, + OFFSET_MAX, 0); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(no_seek_end_llseek); + +/** + * no_seek_end_llseek_size - llseek implementation for fixed-sized devices + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * @size: maximal offset allowed + * + */ +loff_t no_seek_end_llseek_size(struct file *file, loff_t offset, int whence, loff_t size) +{ + switch (whence) { + case SEEK_SET: case SEEK_CUR: + return generic_file_llseek_size(file, offset, whence, + size, 0); + default: + return -EINVAL; + } +} +EXPORT_SYMBOL(no_seek_end_llseek_size); + +/** + * noop_llseek - No Operation Performed llseek implementation + * @file: file structure to seek on + * @offset: file offset to seek to + * @whence: type of seek + * + * This is an implementation of ->llseek useable for the rare special case when + * userspace expects the seek to succeed but the (device) file is actually not + * able to perform the seek. In this case you use noop_llseek() instead of + * falling back to the default implementation of ->llseek. + */ +loff_t noop_llseek(struct file *file, loff_t offset, int whence) +{ + return file->f_pos; +} +EXPORT_SYMBOL(noop_llseek); + +loff_t default_llseek(struct file *file, loff_t offset, int whence) +{ + struct inode *inode = file_inode(file); + loff_t retval; + + retval = inode_lock_killable(inode); + if (retval) + return retval; + switch (whence) { + case SEEK_END: + offset += i_size_read(inode); + break; + case SEEK_CUR: + if (offset == 0) { + retval = file->f_pos; + goto out; + } + offset += file->f_pos; + break; + case SEEK_DATA: + /* + * In the generic case the entire file is data, so as + * long as offset isn't at the end of the file then the + * offset is data. + */ + if (offset >= inode->i_size) { + retval = -ENXIO; + goto out; + } + break; + case SEEK_HOLE: + /* + * There is a virtual hole at the end of the file, so + * as long as offset isn't i_size or larger, return + * i_size. + */ + if (offset >= inode->i_size) { + retval = -ENXIO; + goto out; + } + offset = inode->i_size; + break; + } + retval = -EINVAL; + if (offset >= 0 || unsigned_offsets(file)) { + if (offset != file->f_pos) + file->f_pos = offset; + retval = offset; + } +out: + inode_unlock(inode); + return retval; +} +EXPORT_SYMBOL(default_llseek); + +loff_t vfs_llseek(struct file *file, loff_t offset, int whence) +{ + if (!(file->f_mode & FMODE_LSEEK)) + return -ESPIPE; + return file->f_op->llseek(file, offset, whence); +} +EXPORT_SYMBOL(vfs_llseek); + +static off_t ksys_lseek(unsigned int fd, off_t offset, unsigned int whence) +{ + off_t retval; + CLASS(fd_pos, f)(fd); + if (fd_empty(f)) + return -EBADF; + + retval = -EINVAL; + if (whence <= SEEK_MAX) { + loff_t res = vfs_llseek(fd_file(f), offset, whence); + retval = res; + if (res != (loff_t)retval) + retval = -EOVERFLOW; /* LFS: should only happen on 32 bit platforms */ + } + return retval; +} + +SYSCALL_DEFINE3(lseek, unsigned int, fd, off_t, offset, unsigned int, whence) +{ + return ksys_lseek(fd, offset, whence); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE3(lseek, unsigned int, fd, compat_off_t, offset, unsigned int, whence) +{ + return ksys_lseek(fd, offset, whence); +} +#endif + +#if !defined(CONFIG_64BIT) || defined(CONFIG_COMPAT) || \ + defined(__ARCH_WANT_SYS_LLSEEK) +SYSCALL_DEFINE5(llseek, unsigned int, fd, unsigned long, offset_high, + unsigned long, offset_low, loff_t __user *, result, + unsigned int, whence) +{ + int retval; + CLASS(fd_pos, f)(fd); + loff_t offset; + + if (fd_empty(f)) + return -EBADF; + + if (whence > SEEK_MAX) + return -EINVAL; + + offset = vfs_llseek(fd_file(f), ((loff_t) offset_high << 32) | offset_low, + whence); + + retval = (int)offset; + if (offset >= 0) { + retval = -EFAULT; + if (!copy_to_user(result, &offset, sizeof(offset))) + retval = 0; + } + return retval; +} +#endif + +int rw_verify_area(int read_write, struct file *file, const loff_t *ppos, size_t count) +{ + int mask = read_write == READ ? MAY_READ : MAY_WRITE; + int ret; + + if (unlikely((ssize_t) count < 0)) + return -EINVAL; + + if (ppos) { + loff_t pos = *ppos; + + if (unlikely(pos < 0)) { + if (!unsigned_offsets(file)) + return -EINVAL; + if (count >= -pos) /* both values are in 0..LLONG_MAX */ + return -EOVERFLOW; + } else if (unlikely((loff_t) (pos + count) < 0)) { + if (!unsigned_offsets(file)) + return -EINVAL; + } + } + + ret = security_file_permission(file, mask); + if (ret) + return ret; + + return fsnotify_file_area_perm(file, mask, ppos, count); +} +EXPORT_SYMBOL(rw_verify_area); + +static ssize_t new_sync_read(struct file *filp, char __user *buf, size_t len, loff_t *ppos) +{ + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + kiocb.ki_pos = (ppos ? *ppos : 0); + iov_iter_ubuf(&iter, ITER_DEST, buf, len); + + ret = filp->f_op->read_iter(&kiocb, &iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +static int warn_unsupported(struct file *file, const char *op) +{ + pr_warn_ratelimited( + "kernel %s not supported for file %pD4 (pid: %d comm: %.20s)\n", + op, file, current->pid, current->comm); + return -EINVAL; +} + +ssize_t __kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) +{ + struct kvec iov = { + .iov_base = buf, + .iov_len = min_t(size_t, count, MAX_RW_COUNT), + }; + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_READ))) + return -EINVAL; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + /* + * Also fail if ->read_iter and ->read are both wired up as that + * implies very convoluted semantics. + */ + if (unlikely(!file->f_op->read_iter || file->f_op->read)) + return warn_unsupported(file, "read"); + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos ? *pos : 0; + iov_iter_kvec(&iter, ITER_DEST, &iov, 1, iov.iov_len); + ret = file->f_op->read_iter(&kiocb, &iter); + if (ret > 0) { + if (pos) + *pos = kiocb.ki_pos; + fsnotify_access(file); + add_rchar(current, ret); + } + inc_syscr(current); + return ret; +} + +ssize_t kernel_read(struct file *file, void *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + ret = rw_verify_area(READ, file, pos, count); + if (ret) + return ret; + return __kernel_read(file, buf, count, pos); +} +EXPORT_SYMBOL(kernel_read); + +ssize_t vfs_read(struct file *file, char __user *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + if (unlikely(!access_ok(buf, count))) + return -EFAULT; + + ret = rw_verify_area(READ, file, pos, count); + if (ret) + return ret; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + + if (file->f_op->read) + ret = file->f_op->read(file, buf, count, pos); + else if (file->f_op->read_iter) + ret = new_sync_read(file, buf, count, pos); + else + ret = -EINVAL; + if (ret > 0) { + fsnotify_access(file); + add_rchar(current, ret); + } + inc_syscr(current); + return ret; +} + +static ssize_t new_sync_write(struct file *filp, const char __user *buf, size_t len, loff_t *ppos) +{ + struct kiocb kiocb; + struct iov_iter iter; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + kiocb.ki_pos = (ppos ? *ppos : 0); + iov_iter_ubuf(&iter, ITER_SOURCE, (void __user *)buf, len); + + ret = filp->f_op->write_iter(&kiocb, &iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ret > 0 && ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +/* caller is responsible for file_start_write/file_end_write */ +ssize_t __kernel_write_iter(struct file *file, struct iov_iter *from, loff_t *pos) +{ + struct kiocb kiocb; + ssize_t ret; + + if (WARN_ON_ONCE(!(file->f_mode & FMODE_WRITE))) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + /* + * Also fail if ->write_iter and ->write are both wired up as that + * implies very convoluted semantics. + */ + if (unlikely(!file->f_op->write_iter || file->f_op->write)) + return warn_unsupported(file, "write"); + + init_sync_kiocb(&kiocb, file); + kiocb.ki_pos = pos ? *pos : 0; + ret = file->f_op->write_iter(&kiocb, from); + if (ret > 0) { + if (pos) + *pos = kiocb.ki_pos; + fsnotify_modify(file); + add_wchar(current, ret); + } + inc_syscw(current); + return ret; +} + +/* caller is responsible for file_start_write/file_end_write */ +ssize_t __kernel_write(struct file *file, const void *buf, size_t count, loff_t *pos) +{ + struct kvec iov = { + .iov_base = (void *)buf, + .iov_len = min_t(size_t, count, MAX_RW_COUNT), + }; + struct iov_iter iter; + iov_iter_kvec(&iter, ITER_SOURCE, &iov, 1, iov.iov_len); + return __kernel_write_iter(file, &iter, pos); +} +/* + * autofs is one of the few internal kernel users that actually + * wants this _and_ can be built as a module. So we need to export + * this symbol for autofs, even though it really isn't appropriate + * for any other kernel modules. + */ +EXPORT_SYMBOL_FOR_MODULES(__kernel_write, "autofs4"); + +ssize_t kernel_write(struct file *file, const void *buf, size_t count, + loff_t *pos) +{ + ssize_t ret; + + ret = rw_verify_area(WRITE, file, pos, count); + if (ret) + return ret; + + file_start_write(file); + ret = __kernel_write(file, buf, count, pos); + file_end_write(file); + return ret; +} +EXPORT_SYMBOL(kernel_write); + +ssize_t vfs_write(struct file *file, const char __user *buf, size_t count, loff_t *pos) +{ + ssize_t ret; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + if (unlikely(!access_ok(buf, count))) + return -EFAULT; + + ret = rw_verify_area(WRITE, file, pos, count); + if (ret) + return ret; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + file_start_write(file); + if (file->f_op->write) + ret = file->f_op->write(file, buf, count, pos); + else if (file->f_op->write_iter) + ret = new_sync_write(file, buf, count, pos); + else + ret = -EINVAL; + if (ret > 0) { + fsnotify_modify(file); + add_wchar(current, ret); + } + inc_syscw(current); + file_end_write(file); + return ret; +} + +/* file_ppos returns &file->f_pos or NULL if file is stream */ +static inline loff_t *file_ppos(struct file *file) +{ + return file->f_mode & FMODE_STREAM ? NULL : &file->f_pos; +} + +ssize_t ksys_read(unsigned int fd, char __user *buf, size_t count) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_read(fd_file(f), buf, count, ppos); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + return ret; +} + +SYSCALL_DEFINE3(read, unsigned int, fd, char __user *, buf, size_t, count) +{ + return ksys_read(fd, buf, count); +} + +ssize_t ksys_write(unsigned int fd, const char __user *buf, size_t count) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_write(fd_file(f), buf, count, ppos); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + return ret; +} + +SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf, + size_t, count) +{ + return ksys_write(fd, buf, count); +} + +ssize_t ksys_pread64(unsigned int fd, char __user *buf, size_t count, + loff_t pos) +{ + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_mode & FMODE_PREAD) + return vfs_read(fd_file(f), buf, count, &pos); + + return -ESPIPE; +} + +SYSCALL_DEFINE4(pread64, unsigned int, fd, char __user *, buf, + size_t, count, loff_t, pos) +{ + return ksys_pread64(fd, buf, count, pos); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PREAD64) +COMPAT_SYSCALL_DEFINE5(pread64, unsigned int, fd, char __user *, buf, + size_t, count, compat_arg_u64_dual(pos)) +{ + return ksys_pread64(fd, buf, count, compat_arg_u64_glue(pos)); +} +#endif + +ssize_t ksys_pwrite64(unsigned int fd, const char __user *buf, + size_t count, loff_t pos) +{ + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (fd_empty(f)) + return -EBADF; + + if (fd_file(f)->f_mode & FMODE_PWRITE) + return vfs_write(fd_file(f), buf, count, &pos); + + return -ESPIPE; +} + +SYSCALL_DEFINE4(pwrite64, unsigned int, fd, const char __user *, buf, + size_t, count, loff_t, pos) +{ + return ksys_pwrite64(fd, buf, count, pos); +} + +#if defined(CONFIG_COMPAT) && defined(__ARCH_WANT_COMPAT_PWRITE64) +COMPAT_SYSCALL_DEFINE5(pwrite64, unsigned int, fd, const char __user *, buf, + size_t, count, compat_arg_u64_dual(pos)) +{ + return ksys_pwrite64(fd, buf, count, compat_arg_u64_glue(pos)); +} +#endif + +static ssize_t do_iter_readv_writev(struct file *filp, struct iov_iter *iter, + loff_t *ppos, int type, rwf_t flags) +{ + struct kiocb kiocb; + ssize_t ret; + + init_sync_kiocb(&kiocb, filp); + ret = kiocb_set_rw_flags(&kiocb, flags, type); + if (ret) + return ret; + kiocb.ki_pos = (ppos ? *ppos : 0); + + if (type == READ) + ret = filp->f_op->read_iter(&kiocb, iter); + else + ret = filp->f_op->write_iter(&kiocb, iter); + BUG_ON(ret == -EIOCBQUEUED); + if (ppos) + *ppos = kiocb.ki_pos; + return ret; +} + +/* Do it by hand, with file-ops */ +static ssize_t do_loop_readv_writev(struct file *filp, struct iov_iter *iter, + loff_t *ppos, int type, rwf_t flags) +{ + ssize_t ret = 0; + + if (flags & ~RWF_HIPRI) + return -EOPNOTSUPP; + + while (iov_iter_count(iter)) { + ssize_t nr; + + if (type == READ) { + nr = filp->f_op->read(filp, iter_iov_addr(iter), + iter_iov_len(iter), ppos); + } else { + nr = filp->f_op->write(filp, iter_iov_addr(iter), + iter_iov_len(iter), ppos); + } + + if (nr < 0) { + if (!ret) + ret = nr; + break; + } + ret += nr; + if (nr != iter_iov_len(iter)) + break; + iov_iter_advance(iter, nr); + } + + return ret; +} + +ssize_t vfs_iocb_iter_read(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->read_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + goto out; + ret = rw_verify_area(READ, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + ret = file->f_op->read_iter(iocb, iter); +out: + if (ret >= 0) + fsnotify_access(file); + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_read); + +ssize_t vfs_iter_read(struct file *file, struct iov_iter *iter, loff_t *ppos, + rwf_t flags) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->read_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + goto out; + ret = rw_verify_area(READ, file, ppos, tot_len); + if (ret < 0) + return ret; + + ret = do_iter_readv_writev(file, iter, ppos, READ, flags); +out: + if (ret >= 0) + fsnotify_access(file); + return ret; +} +EXPORT_SYMBOL(vfs_iter_read); + +/* + * Caller is responsible for calling kiocb_end_write() on completion + * if async iocb was queued. + */ +ssize_t vfs_iocb_iter_write(struct file *file, struct kiocb *iocb, + struct iov_iter *iter) +{ + size_t tot_len; + ssize_t ret = 0; + + if (!file->f_op->write_iter) + return -EINVAL; + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + return 0; + ret = rw_verify_area(WRITE, file, &iocb->ki_pos, tot_len); + if (ret < 0) + return ret; + + kiocb_start_write(iocb); + ret = file->f_op->write_iter(iocb, iter); + if (ret != -EIOCBQUEUED) + kiocb_end_write(iocb); + if (ret > 0) + fsnotify_modify(file); + + return ret; +} +EXPORT_SYMBOL(vfs_iocb_iter_write); + +ssize_t vfs_iter_write(struct file *file, struct iov_iter *iter, loff_t *ppos, + rwf_t flags) +{ + size_t tot_len; + ssize_t ret; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + if (!file->f_op->write_iter) + return -EINVAL; + + tot_len = iov_iter_count(iter); + if (!tot_len) + return 0; + + ret = rw_verify_area(WRITE, file, ppos, tot_len); + if (ret < 0) + return ret; + + file_start_write(file); + ret = do_iter_readv_writev(file, iter, ppos, WRITE, flags); + if (ret > 0) + fsnotify_modify(file); + file_end_write(file); + + return ret; +} +EXPORT_SYMBOL(vfs_iter_write); + +static ssize_t vfs_readv(struct file *file, const struct iovec __user *vec, + unsigned long vlen, loff_t *pos, rwf_t flags) +{ + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + size_t tot_len; + ssize_t ret = 0; + + if (!(file->f_mode & FMODE_READ)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_READ)) + return -EINVAL; + + ret = import_iovec(ITER_DEST, vec, vlen, ARRAY_SIZE(iovstack), &iov, + &iter); + if (ret < 0) + return ret; + + tot_len = iov_iter_count(&iter); + if (!tot_len) + goto out; + + ret = rw_verify_area(READ, file, pos, tot_len); + if (ret < 0) + goto out; + + if (file->f_op->read_iter) + ret = do_iter_readv_writev(file, &iter, pos, READ, flags); + else + ret = do_loop_readv_writev(file, &iter, pos, READ, flags); +out: + if (ret >= 0) + fsnotify_access(file); + kfree(iov); + return ret; +} + +static ssize_t vfs_writev(struct file *file, const struct iovec __user *vec, + unsigned long vlen, loff_t *pos, rwf_t flags) +{ + struct iovec iovstack[UIO_FASTIOV]; + struct iovec *iov = iovstack; + struct iov_iter iter; + size_t tot_len; + ssize_t ret = 0; + + if (!(file->f_mode & FMODE_WRITE)) + return -EBADF; + if (!(file->f_mode & FMODE_CAN_WRITE)) + return -EINVAL; + + ret = import_iovec(ITER_SOURCE, vec, vlen, ARRAY_SIZE(iovstack), &iov, + &iter); + if (ret < 0) + return ret; + + tot_len = iov_iter_count(&iter); + if (!tot_len) + goto out; + + ret = rw_verify_area(WRITE, file, pos, tot_len); + if (ret < 0) + goto out; + + file_start_write(file); + if (file->f_op->write_iter) + ret = do_iter_readv_writev(file, &iter, pos, WRITE, flags); + else + ret = do_loop_readv_writev(file, &iter, pos, WRITE, flags); + if (ret > 0) + fsnotify_modify(file); + file_end_write(file); +out: + kfree(iov); + return ret; +} + +static ssize_t do_readv(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, rwf_t flags) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_readv(fd_file(f), vec, vlen, ppos, flags); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + return ret; +} + +static ssize_t do_writev(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, rwf_t flags) +{ + CLASS(fd_pos, f)(fd); + ssize_t ret = -EBADF; + + if (!fd_empty(f)) { + loff_t pos, *ppos = file_ppos(fd_file(f)); + if (ppos) { + pos = *ppos; + ppos = &pos; + } + ret = vfs_writev(fd_file(f), vec, vlen, ppos, flags); + if (ret >= 0 && ppos) + fd_file(f)->f_pos = pos; + } + + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + return ret; +} + +static inline loff_t pos_from_hilo(unsigned long high, unsigned long low) +{ +#define HALF_LONG_BITS (BITS_PER_LONG / 2) + return (((loff_t)high << HALF_LONG_BITS) << HALF_LONG_BITS) | low; +} + +static ssize_t do_preadv(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, loff_t pos, rwf_t flags) +{ + ssize_t ret = -EBADF; + + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (!fd_empty(f)) { + ret = -ESPIPE; + if (fd_file(f)->f_mode & FMODE_PREAD) + ret = vfs_readv(fd_file(f), vec, vlen, &pos, flags); + } + + if (ret > 0) + add_rchar(current, ret); + inc_syscr(current); + return ret; +} + +static ssize_t do_pwritev(unsigned long fd, const struct iovec __user *vec, + unsigned long vlen, loff_t pos, rwf_t flags) +{ + ssize_t ret = -EBADF; + + if (pos < 0) + return -EINVAL; + + CLASS(fd, f)(fd); + if (!fd_empty(f)) { + ret = -ESPIPE; + if (fd_file(f)->f_mode & FMODE_PWRITE) + ret = vfs_writev(fd_file(f), vec, vlen, &pos, flags); + } + + if (ret > 0) + add_wchar(current, ret); + inc_syscw(current); + return ret; +} + +SYSCALL_DEFINE3(readv, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen) +{ + return do_readv(fd, vec, vlen, 0); +} + +SYSCALL_DEFINE3(writev, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen) +{ + return do_writev(fd, vec, vlen, 0); +} + +SYSCALL_DEFINE5(preadv, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + return do_preadv(fd, vec, vlen, pos, 0); +} + +SYSCALL_DEFINE6(preadv2, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, + rwf_t, flags) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + + return do_preadv(fd, vec, vlen, pos, flags); +} + +SYSCALL_DEFINE5(pwritev, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + return do_pwritev(fd, vec, vlen, pos, 0); +} + +SYSCALL_DEFINE6(pwritev2, unsigned long, fd, const struct iovec __user *, vec, + unsigned long, vlen, unsigned long, pos_l, unsigned long, pos_h, + rwf_t, flags) +{ + loff_t pos = pos_from_hilo(pos_h, pos_l); + + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + + return do_pwritev(fd, vec, vlen, pos, flags); +} + +/* + * Various compat syscalls. Note that they all pretend to take a native + * iovec - import_iovec will properly treat those as compat_iovecs based on + * in_compat_syscall(). + */ +#ifdef CONFIG_COMPAT +#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64 +COMPAT_SYSCALL_DEFINE4(preadv64, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos) +{ + return do_preadv(fd, vec, vlen, pos, 0); +} +#endif + +COMPAT_SYSCALL_DEFINE5(preadv, compat_ulong_t, fd, + const struct iovec __user *, vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + return do_preadv(fd, vec, vlen, pos, 0); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PREADV64V2 +COMPAT_SYSCALL_DEFINE5(preadv64v2, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos, rwf_t, flags) +{ + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + return do_preadv(fd, vec, vlen, pos, flags); +} +#endif + +COMPAT_SYSCALL_DEFINE6(preadv2, compat_ulong_t, fd, + const struct iovec __user *, vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high, + rwf_t, flags) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + if (pos == -1) + return do_readv(fd, vec, vlen, flags); + return do_preadv(fd, vec, vlen, pos, flags); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64 +COMPAT_SYSCALL_DEFINE4(pwritev64, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos) +{ + return do_pwritev(fd, vec, vlen, pos, 0); +} +#endif + +COMPAT_SYSCALL_DEFINE5(pwritev, compat_ulong_t, fd, + const struct iovec __user *,vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + return do_pwritev(fd, vec, vlen, pos, 0); +} + +#ifdef __ARCH_WANT_COMPAT_SYS_PWRITEV64V2 +COMPAT_SYSCALL_DEFINE5(pwritev64v2, unsigned long, fd, + const struct iovec __user *, vec, + unsigned long, vlen, loff_t, pos, rwf_t, flags) +{ + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + return do_pwritev(fd, vec, vlen, pos, flags); +} +#endif + +COMPAT_SYSCALL_DEFINE6(pwritev2, compat_ulong_t, fd, + const struct iovec __user *,vec, + compat_ulong_t, vlen, u32, pos_low, u32, pos_high, rwf_t, flags) +{ + loff_t pos = ((loff_t)pos_high << 32) | pos_low; + + if (pos == -1) + return do_writev(fd, vec, vlen, flags); + return do_pwritev(fd, vec, vlen, pos, flags); +} +#endif /* CONFIG_COMPAT */ + +static ssize_t do_sendfile(int out_fd, int in_fd, loff_t *ppos, + size_t count, loff_t max) +{ + struct inode *in_inode, *out_inode; + struct pipe_inode_info *opipe; + loff_t pos; + loff_t out_pos; + ssize_t retval; + int fl; + + /* + * Get input file, and verify that it is ok.. + */ + CLASS(fd, in)(in_fd); + if (fd_empty(in)) + return -EBADF; + if (!(fd_file(in)->f_mode & FMODE_READ)) + return -EBADF; + if (!ppos) { + pos = fd_file(in)->f_pos; + } else { + pos = *ppos; + if (!(fd_file(in)->f_mode & FMODE_PREAD)) + return -ESPIPE; + } + retval = rw_verify_area(READ, fd_file(in), &pos, count); + if (retval < 0) + return retval; + if (count > MAX_RW_COUNT) + count = MAX_RW_COUNT; + + /* + * Get output file, and verify that it is ok.. + */ + CLASS(fd, out)(out_fd); + if (fd_empty(out)) + return -EBADF; + if (!(fd_file(out)->f_mode & FMODE_WRITE)) + return -EBADF; + in_inode = file_inode(fd_file(in)); + out_inode = file_inode(fd_file(out)); + out_pos = fd_file(out)->f_pos; + + if (!max) + max = min(in_inode->i_sb->s_maxbytes, out_inode->i_sb->s_maxbytes); + + if (unlikely(pos + count > max)) { + if (pos >= max) + return -EOVERFLOW; + count = max - pos; + } + + fl = 0; +#if 0 + /* + * We need to debate whether we can enable this or not. The + * man page documents EAGAIN return for the output at least, + * and the application is arguably buggy if it doesn't expect + * EAGAIN on a non-blocking file descriptor. + */ + if (fd_file(in)->f_flags & O_NONBLOCK) + fl = SPLICE_F_NONBLOCK; +#endif + opipe = get_pipe_info(fd_file(out), true); + if (!opipe) { + retval = rw_verify_area(WRITE, fd_file(out), &out_pos, count); + if (retval < 0) + return retval; + retval = do_splice_direct(fd_file(in), &pos, fd_file(out), &out_pos, + count, fl); + } else { + if (fd_file(out)->f_flags & O_NONBLOCK) + fl |= SPLICE_F_NONBLOCK; + + retval = splice_file_to_pipe(fd_file(in), opipe, &pos, count, fl); + } + + if (retval > 0) { + add_rchar(current, retval); + add_wchar(current, retval); + fsnotify_access(fd_file(in)); + fsnotify_modify(fd_file(out)); + fd_file(out)->f_pos = out_pos; + if (ppos) + *ppos = pos; + else + fd_file(in)->f_pos = pos; + } + + inc_syscr(current); + inc_syscw(current); + if (pos > max) + retval = -EOVERFLOW; + return retval; +} + +SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, off_t __user *, offset, size_t, count) +{ + loff_t pos; + off_t off; + ssize_t ret; + + if (offset) { + if (unlikely(get_user(off, offset))) + return -EFAULT; + pos = off; + ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, loff_t __user *, offset, size_t, count) +{ + loff_t pos; + ssize_t ret; + + if (offset) { + if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) + return -EFAULT; + ret = do_sendfile(out_fd, in_fd, &pos, count, 0); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +#ifdef CONFIG_COMPAT +COMPAT_SYSCALL_DEFINE4(sendfile, int, out_fd, int, in_fd, + compat_off_t __user *, offset, compat_size_t, count) +{ + loff_t pos; + off_t off; + ssize_t ret; + + if (offset) { + if (unlikely(get_user(off, offset))) + return -EFAULT; + pos = off; + ret = do_sendfile(out_fd, in_fd, &pos, count, MAX_NON_LFS); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} + +COMPAT_SYSCALL_DEFINE4(sendfile64, int, out_fd, int, in_fd, + compat_loff_t __user *, offset, compat_size_t, count) +{ + loff_t pos; + ssize_t ret; + + if (offset) { + if (unlikely(copy_from_user(&pos, offset, sizeof(loff_t)))) + return -EFAULT; + ret = do_sendfile(out_fd, in_fd, &pos, count, 0); + if (unlikely(put_user(pos, offset))) + return -EFAULT; + return ret; + } + + return do_sendfile(out_fd, in_fd, NULL, count, 0); +} +#endif + +/* + * Performs necessary checks before doing a file copy + * + * Can adjust amount of bytes to copy via @req_count argument. + * Returns appropriate error code that caller should return or + * zero in case the copy should be allowed. + */ +static int generic_copy_file_checks(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t *req_count, unsigned int flags) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + uint64_t count = *req_count; + loff_t size_in; + int ret; + + ret = generic_file_rw_checks(file_in, file_out); + if (ret) + return ret; + + /* + * We allow some filesystems to handle cross sb copy, but passing + * a file of the wrong filesystem type to filesystem driver can result + * in an attempt to dereference the wrong type of ->private_data, so + * avoid doing that until we really have a good reason. + * + * nfs and cifs define several different file_system_type structures + * and several different sets of file_operations, but they all end up + * using the same ->copy_file_range() function pointer. + */ + if (flags & COPY_FILE_SPLICE) { + /* cross sb splice is allowed */ + } else if (file_out->f_op->copy_file_range) { + if (file_in->f_op->copy_file_range != + file_out->f_op->copy_file_range) + return -EXDEV; + } else if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) { + return -EXDEV; + } + + /* Don't touch certain kinds of inodes */ + if (IS_IMMUTABLE(inode_out)) + return -EPERM; + + if (IS_SWAPFILE(inode_in) || IS_SWAPFILE(inode_out)) + return -ETXTBSY; + + /* Ensure offsets don't wrap. */ + if (pos_in + count < pos_in || pos_out + count < pos_out) + return -EOVERFLOW; + + /* Shorten the copy to EOF */ + size_in = i_size_read(inode_in); + if (pos_in >= size_in) + count = 0; + else + count = min(count, size_in - (uint64_t)pos_in); + + ret = generic_write_check_limits(file_out, pos_out, &count); + if (ret) + return ret; + + /* Don't allow overlapped copying within the same file. */ + if (inode_in == inode_out && + pos_out + count > pos_in && + pos_out < pos_in + count) + return -EINVAL; + + *req_count = count; + return 0; +} + +/* + * copy_file_range() differs from regular file read and write in that it + * specifically allows return partial success. When it does so is up to + * the copy_file_range method. + */ +ssize_t vfs_copy_file_range(struct file *file_in, loff_t pos_in, + struct file *file_out, loff_t pos_out, + size_t len, unsigned int flags) +{ + ssize_t ret; + bool splice = flags & COPY_FILE_SPLICE; + bool samesb = file_inode(file_in)->i_sb == file_inode(file_out)->i_sb; + + if (flags & ~COPY_FILE_SPLICE) + return -EINVAL; + + ret = generic_copy_file_checks(file_in, pos_in, file_out, pos_out, &len, + flags); + if (unlikely(ret)) + return ret; + + ret = rw_verify_area(READ, file_in, &pos_in, len); + if (unlikely(ret)) + return ret; + + ret = rw_verify_area(WRITE, file_out, &pos_out, len); + if (unlikely(ret)) + return ret; + + if (len == 0) + return 0; + + /* + * Make sure return value doesn't overflow in 32bit compat mode. Also + * limit the size for all cases except when calling ->copy_file_range(). + */ + if (splice || !file_out->f_op->copy_file_range || in_compat_syscall()) + len = min_t(size_t, MAX_RW_COUNT, len); + + file_start_write(file_out); + + /* + * Cloning is supported by more file systems, so we implement copy on + * same sb using clone, but for filesystems where both clone and copy + * are supported (e.g. nfs,cifs), we only call the copy method. + */ + if (!splice && file_out->f_op->copy_file_range) { + ret = file_out->f_op->copy_file_range(file_in, pos_in, + file_out, pos_out, + len, flags); + } else if (!splice && file_in->f_op->remap_file_range && samesb) { + ret = file_in->f_op->remap_file_range(file_in, pos_in, + file_out, pos_out, len, REMAP_FILE_CAN_SHORTEN); + /* fallback to splice */ + if (ret <= 0) + splice = true; + } else if (samesb) { + /* Fallback to splice for same sb copy for backward compat */ + splice = true; + } + + file_end_write(file_out); + + if (!splice) + goto done; + + /* + * We can get here for same sb copy of filesystems that do not implement + * ->copy_file_range() in case filesystem does not support clone or in + * case filesystem supports clone but rejected the clone request (e.g. + * because it was not block aligned). + * + * In both cases, fall back to kernel copy so we are able to maintain a + * consistent story about which filesystems support copy_file_range() + * and which filesystems do not, that will allow userspace tools to + * make consistent desicions w.r.t using copy_file_range(). + * + * We also get here if caller (e.g. nfsd) requested COPY_FILE_SPLICE + * for server-side-copy between any two sb. + * + * In any case, we call do_splice_direct() and not splice_file_range(), + * without file_start_write() held, to avoid possible deadlocks related + * to splicing from input file, while file_start_write() is held on + * the output file on a different sb. + */ + ret = do_splice_direct(file_in, &pos_in, file_out, &pos_out, len, 0); +done: + if (ret > 0) { + fsnotify_access(file_in); + add_rchar(current, ret); + fsnotify_modify(file_out); + add_wchar(current, ret); + } + + inc_syscr(current); + inc_syscw(current); + + return ret; +} +EXPORT_SYMBOL(vfs_copy_file_range); + +SYSCALL_DEFINE6(copy_file_range, int, fd_in, loff_t __user *, off_in, + int, fd_out, loff_t __user *, off_out, + size_t, len, unsigned int, flags) +{ + loff_t pos_in; + loff_t pos_out; + ssize_t ret = -EBADF; + + CLASS(fd, f_in)(fd_in); + if (fd_empty(f_in)) + return -EBADF; + + CLASS(fd, f_out)(fd_out); + if (fd_empty(f_out)) + return -EBADF; + + if (off_in) { + if (copy_from_user(&pos_in, off_in, sizeof(loff_t))) + return -EFAULT; + } else { + pos_in = fd_file(f_in)->f_pos; + } + + if (off_out) { + if (copy_from_user(&pos_out, off_out, sizeof(loff_t))) + return -EFAULT; + } else { + pos_out = fd_file(f_out)->f_pos; + } + + if (flags != 0) + return -EINVAL; + + ret = vfs_copy_file_range(fd_file(f_in), pos_in, fd_file(f_out), pos_out, len, + flags); + if (ret > 0) { + pos_in += ret; + pos_out += ret; + + if (off_in) { + if (copy_to_user(off_in, &pos_in, sizeof(loff_t))) + ret = -EFAULT; + } else { + fd_file(f_in)->f_pos = pos_in; + } + + if (off_out) { + if (copy_to_user(off_out, &pos_out, sizeof(loff_t))) + ret = -EFAULT; + } else { + fd_file(f_out)->f_pos = pos_out; + } + } + return ret; +} + +/* + * Don't operate on ranges the page cache doesn't support, and don't exceed the + * LFS limits. If pos is under the limit it becomes a short access. If it + * exceeds the limit we return -EFBIG. + */ +int generic_write_check_limits(struct file *file, loff_t pos, loff_t *count) +{ + struct inode *inode = file->f_mapping->host; + loff_t max_size = inode->i_sb->s_maxbytes; + loff_t limit = rlimit(RLIMIT_FSIZE); + + if (limit != RLIM_INFINITY) { + if (pos >= limit) { + send_sig(SIGXFSZ, current, 0); + return -EFBIG; + } + *count = min(*count, limit - pos); + } + + if (!(file->f_flags & O_LARGEFILE)) + max_size = MAX_NON_LFS; + + if (unlikely(pos >= max_size)) + return -EFBIG; + + *count = min(*count, max_size - pos); + + return 0; +} +EXPORT_SYMBOL_GPL(generic_write_check_limits); + +/* Like generic_write_checks(), but takes size of write instead of iter. */ +int generic_write_checks_count(struct kiocb *iocb, loff_t *count) +{ + struct file *file = iocb->ki_filp; + struct inode *inode = file->f_mapping->host; + + if (IS_SWAPFILE(inode)) + return -ETXTBSY; + + if (!*count) + return 0; + + if (iocb->ki_flags & IOCB_APPEND) + iocb->ki_pos = i_size_read(inode); + + if ((iocb->ki_flags & IOCB_NOWAIT) && + !((iocb->ki_flags & IOCB_DIRECT) || + (file->f_op->fop_flags & FOP_BUFFER_WASYNC))) + return -EINVAL; + + return generic_write_check_limits(iocb->ki_filp, iocb->ki_pos, count); +} +EXPORT_SYMBOL(generic_write_checks_count); + +/* + * Performs necessary checks before doing a write + * + * Can adjust writing position or amount of bytes to write. + * Returns appropriate error code that caller should return or + * zero in case that write should be allowed. + */ +ssize_t generic_write_checks(struct kiocb *iocb, struct iov_iter *from) +{ + loff_t count = iov_iter_count(from); + int ret; + + ret = generic_write_checks_count(iocb, &count); + if (ret) + return ret; + + iov_iter_truncate(from, count); + return iov_iter_count(from); +} +EXPORT_SYMBOL(generic_write_checks); + +/* + * Performs common checks before doing a file copy/clone + * from @file_in to @file_out. + */ +int generic_file_rw_checks(struct file *file_in, struct file *file_out) +{ + struct inode *inode_in = file_inode(file_in); + struct inode *inode_out = file_inode(file_out); + + /* Don't copy dirs, pipes, sockets... */ + if (S_ISDIR(inode_in->i_mode) || S_ISDIR(inode_out->i_mode)) + return -EISDIR; + if (!S_ISREG(inode_in->i_mode) || !S_ISREG(inode_out->i_mode)) + return -EINVAL; + + if (!(file_in->f_mode & FMODE_READ) || + !(file_out->f_mode & FMODE_WRITE) || + (file_out->f_flags & O_APPEND)) + return -EBADF; + + return 0; +} + +int generic_atomic_write_valid(struct kiocb *iocb, struct iov_iter *iter) +{ + size_t len = iov_iter_count(iter); + + if (!iter_is_ubuf(iter)) + return -EINVAL; + + if (!is_power_of_2(len)) + return -EINVAL; + + if (!IS_ALIGNED(iocb->ki_pos, len)) + return -EINVAL; + + if (!(iocb->ki_flags & IOCB_DIRECT)) + return -EOPNOTSUPP; + + return 0; +} +EXPORT_SYMBOL_GPL(generic_atomic_write_valid); diff --git a/corpora/source/pinned/fs/read_write.meta.toml b/corpora/source/pinned/fs/read_write.meta.toml new file mode 100644 index 0000000..df03803 --- /dev/null +++ b/corpora/source/pinned/fs/read_write.meta.toml @@ -0,0 +1,32 @@ +source = "pinned" +evidence = true + +describes = "the file the write path goes through, and where three names for one call live" +path = "fs/read_write.c" +excerpt = false + +kernel = "7.2.2" +tarball_sha256 = "7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83" +taken = "2026-09-06" + +# One real kernel source file, whole, so that symbol lookup and citation anchors have something to +# resolve against in CI and in a notebook that has downloaded nothing. This one rather than another +# because the write path blueprint already cites it and because it holds the gap between what a +# symbol is called and what the source calls it. +# +# The string sys_write does not appear in this file. SYSCALL_DEFINE3(write, ...) does, and the +# symbol the running kernel ends up with is __ia32_sys_write on this architecture. Three names. +# +# vfs_write is defined here and not exported, so a module cannot call it. rw_verify_area is +# exported. __kernel_write is exported to exactly one named module. All three are worth seeing +# together, because "it is in kallsyms" and "a module can call it" are different questions. + +lines = 1822 +syscall_definitions = 28 +export_lines = 24 +vfs_write_line = 667 +vfs_write_exported = false +ksys_write_line = 728 +write_syscall_line = 747 +rw_verify_area_line = 453 +rw_verify_area_exported = true diff --git a/corpora/source/pinned/kernel/Kconfig.meta.toml b/corpora/source/pinned/kernel/Kconfig.meta.toml new file mode 100644 index 0000000..1bd00d2 --- /dev/null +++ b/corpora/source/pinned/kernel/Kconfig.meta.toml @@ -0,0 +1,25 @@ +source = "pinned" +evidence = true + +describes = "the four preemption models, and how choosing one turns on two more symbols" +path = "kernel/Kconfig.preempt" +excerpt = false + +kernel = "7.2.2" +tarball_sha256 = "7d0e7ce14f98c43efe880cffbf354a59be45928fdf7170d7333c374ae91c0d83" +taken = "2026-09-06" + +# The pinned kernel is built with CONFIG_PREEMPT=y, and its .config also has CONFIG_PREEMPT_BUILD=y +# and CONFIG_PREEMPTION=y in it, which nobody asked for. This file is the whole explanation. +# PREEMPT selects PREEMPT_BUILD, and PREEMPT_BUILD selects PREEMPTION. +# +# Six of the fifteen symbols here have a type and no prompt, which means they never appear in +# menuconfig and cannot be set by hand at all. In a .config they look exactly like something a +# person chose. + +symbols = 15 +hidden_symbols = 6 +choice_prompt = "Preemption Model" +choice_members = ["PREEMPT_NONE", "PREEMPT_VOLUNTARY", "PREEMPT", "PREEMPT_LAZY"] +preempt_selects = ["PREEMPT_BUILD"] +preempt_build_selects = ["PREEMPTION", "UNINLINE_SPIN_UNLOCK"] diff --git a/corpora/source/pinned/kernel/Kconfig.preempt b/corpora/source/pinned/kernel/Kconfig.preempt new file mode 100644 index 0000000..88c594c --- /dev/null +++ b/corpora/source/pinned/kernel/Kconfig.preempt @@ -0,0 +1,194 @@ +# SPDX-License-Identifier: GPL-2.0-only + +config PREEMPT_NONE_BUILD + bool + +config PREEMPT_VOLUNTARY_BUILD + bool + +config PREEMPT_BUILD + bool + select PREEMPTION + select UNINLINE_SPIN_UNLOCK if !ARCH_INLINE_SPIN_UNLOCK + +config ARCH_HAS_PREEMPT_LAZY + bool + +choice + prompt "Preemption Model" + default PREEMPT_LAZY if ARCH_HAS_PREEMPT_LAZY + default PREEMPT_NONE + +config PREEMPT_NONE + bool "No Forced Preemption (Server)" + depends on !PREEMPT_RT + depends on ARCH_NO_PREEMPT + select PREEMPT_NONE_BUILD if !PREEMPT_DYNAMIC + help + This is the traditional Linux preemption model, geared towards + throughput. It will still provide good latencies most of the + time, but there are no guarantees and occasional longer delays + are possible. + + Select this option if you are building a kernel for a server or + scientific/computation system, or if you want to maximize the + raw processing power of the kernel, irrespective of scheduling + latencies. + +config PREEMPT_VOLUNTARY + bool "Voluntary Kernel Preemption (Desktop)" + depends on !ARCH_HAS_PREEMPT_LAZY + depends on !ARCH_NO_PREEMPT + depends on !PREEMPT_RT + select PREEMPT_VOLUNTARY_BUILD if !PREEMPT_DYNAMIC + help + This option reduces the latency of the kernel by adding more + "explicit preemption points" to the kernel code. These new + preemption points have been selected to reduce the maximum + latency of rescheduling, providing faster application reactions, + at the cost of slightly lower throughput. + + This allows reaction to interactive events by allowing a + low priority process to voluntarily preempt itself even if it + is in kernel mode executing a system call. This allows + applications to run more 'smoothly' even when the system is + under load. + + Select this if you are building a kernel for a desktop system. + +config PREEMPT + bool "Preemptible Kernel (Low-Latency Desktop)" + depends on !ARCH_NO_PREEMPT + select PREEMPT_BUILD if !PREEMPT_DYNAMIC + help + This option reduces the latency of the kernel by making + all kernel code (that is not executing in a critical section) + preemptible. This allows reaction to interactive events by + permitting a low priority process to be preempted involuntarily + even if it is in kernel mode executing a system call and would + otherwise not be about to reach a natural preemption point. + This allows applications to run more 'smoothly' even when the + system is under load, at the cost of slightly lower throughput + and a slight runtime overhead to kernel code. + + Select this if you are building a kernel for a desktop or + embedded system with latency requirements in the milliseconds + range. + +config PREEMPT_LAZY + bool "Scheduler controlled preemption model" + depends on !ARCH_NO_PREEMPT + depends on ARCH_HAS_PREEMPT_LAZY + select PREEMPT_BUILD if !PREEMPT_DYNAMIC + help + This option provides a scheduler driven preemption model that + is fundamentally similar to full preemption, but is less + eager to preempt SCHED_NORMAL tasks in an attempt to + reduce lock holder preemption and recover some of the performance + gains seen from using Voluntary preemption. + +endchoice + +config PREEMPT_RT + bool "Fully Preemptible Kernel (Real-Time)" + depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST + select PREEMPTION + help + This option turns the kernel into a real-time kernel by replacing + various locking primitives (spinlocks, rwlocks, etc.) with + preemptible priority-inheritance aware variants, enforcing + interrupt threading and introducing mechanisms to break up long + non-preemptible sections. This makes the kernel, except for very + low level and critical code paths (entry code, scheduler, low + level interrupt handling) fully preemptible and brings most + execution contexts under scheduler control. + + Select this if you are building a kernel for systems which + require real-time guarantees. + +config PREEMPT_RT_NEEDS_BH_LOCK + bool "Enforce softirq synchronisation on PREEMPT_RT" + depends on PREEMPT_RT + help + Enforce synchronisation across the softirqs context. On PREEMPT_RT + the softirq is preemptible. This enforces the same per-CPU BLK + semantic non-PREEMPT_RT builds have. This should not be needed + because per-CPU locks were added to avoid the per-CPU BKL. + + This switch provides the old behaviour for testing reasons. Select + this if you suspect an error with preemptible softirq and want test + the old synchronized behaviour. + +config PREEMPT_COUNT + bool + +config PREEMPTION + bool + select PREEMPT_COUNT + +config PREEMPT_DYNAMIC + bool "Preemption behaviour defined on boot" + depends on HAVE_PREEMPT_DYNAMIC + select JUMP_LABEL if HAVE_PREEMPT_DYNAMIC_KEY + select PREEMPT_BUILD + default y if HAVE_PREEMPT_DYNAMIC_CALL + help + This option allows to define the preemption model on the kernel + command line parameter and thus override the default preemption + model defined during compile time. + + The feature is primarily interesting for Linux distributions which + provide a pre-built kernel binary to reduce the number of kernel + flavors they offer while still offering different usecases. + + The runtime overhead is negligible with HAVE_STATIC_CALL_INLINE enabled + but if runtime patching is not available for the specific architecture + then the potential overhead should be considered. + + Interesting if you want the same pre-built kernel should be used for + both Server and Desktop workloads. + +config SCHED_CORE + bool "Core Scheduling for SMT" + depends on SCHED_SMT + help + This option permits Core Scheduling, a means of coordinated task + selection across SMT siblings. When enabled -- see + prctl(PR_SCHED_CORE) -- task selection ensures that all SMT siblings + will execute a task from the same 'core group', forcing idle when no + matching task is found. + + Use of this feature includes: + - mitigation of some (not all) SMT side channels; + - limiting SMT interference to improve determinism and/or performance. + + SCHED_CORE is default disabled. When it is enabled and unused, + which is the likely usage by Linux distributions, there should + be no measurable impact on performance. + +config SCHED_CLASS_EXT + bool "Extensible Scheduling Class" + depends on BPF_SYSCALL && BPF_JIT && DEBUG_INFO_BTF + select STACKTRACE if STACKTRACE_SUPPORT + help + This option enables a new scheduler class sched_ext (SCX), which + allows scheduling policies to be implemented as BPF programs to + achieve the following: + + - Ease of experimentation and exploration: Enabling rapid + iteration of new scheduling policies. + - Customization: Building application-specific schedulers which + implement policies that are not applicable to general-purpose + schedulers. + - Rapid scheduler deployments: Non-disruptive swap outs of + scheduling policies in production environments. + + sched_ext leverages BPF struct_ops feature to define a structure + which exports function callbacks and flags to BPF programs that + wish to implement scheduling policies. The struct_ops structure + exported by sched_ext is struct sched_ext_ops, and is conceptually + similar to struct sched_class. + + For more information: + Documentation/scheduler/sched-ext.rst + https://github.com/sched-ext/scx diff --git a/kxray/source/__init__.py b/kxray/source/__init__.py new file mode 100644 index 0000000..1faaa36 --- /dev/null +++ b/kxray/source/__init__.py @@ -0,0 +1,47 @@ +"""Asking the kernel source a question, from a notebook with almost none of it on disk. + + from kxray import source + + tree = source.tree.find() + print(tree.describe()) + print(source.maintainers.load(tree).table("mm/memory.c")) + print(source.syscalls.load(tree, "i386").by_number(4)) + +Every other package in `kxray` reads something a running kernel produced. This one reads the kernel +itself, which is a different problem, because the thing being read is 1.6 GB and a reader opening a +lesson in Colab has none of it. + +So everything here goes through `tree.find()`, which returns the full unpacked source when +`./kxbox/kernel/tree.sh` has been run and the five files in `corpora/source/pinned/` when it has +not. The object says which it is, and a lookup that misses in the partial tree says "not in the +corpus" rather than "not in the kernel". Those are different sentences and only one of them is true. + +Five modules, and each one exists because a name means something different depending on which of +them you asked. + +`maintainers` reads MAINTAINERS, which is a glob engine wearing an address book. `syscalls` reads +the `.tbl` files, where write is 4 on i386 and 1 on x86-64. `kconfig` reads the kernel's own Kconfig +files, which is where a symbol with no prompt turns out to be on because something else selected it. +`symbols` bridges a name in a stack trace to the line that defines it, which for a system call means +going through a macro. `citations` is the anchor and the context hash that `tools/refcheck` uses, so +that a citation whose text still matches but whose surroundings have been rewritten gets noticed. + +The models live in the modules rather than in `kxray/models.py`, which is the opposite of what +`kxray.proc` does. The difference is real: the `/proc` readers hand their results to widgets, to the +baseline and to each other, so those types are shared and belong in the shared file. A syscall table +row is only ever a syscall table row, the same way a BTF header is only ever a BTF header, and +`kxray/btf/format.py` has kept its own dataclasses for that reason since the beginning. +""" + +from __future__ import annotations + +from kxray.source import citations, kconfig, maintainers, symbols, syscalls, tree + +__all__ = [ + "citations", + "kconfig", + "maintainers", + "symbols", + "syscalls", + "tree", +] diff --git a/kxray/source/citations.py b/kxray/source/citations.py new file mode 100644 index 0000000..e3d3462 --- /dev/null +++ b/kxray/source/citations.py @@ -0,0 +1,148 @@ +"""Anchors into the kernel source, and a hash of what was around them. + + from kxray.source import citations + + hit = citations.resolve(text, "ssize_t vfs_write(struct file *file") + print(hit.line, hit.context) + +Every citation in this repository names a file and a piece of text to find in it, never a line +number, because a line number is wrong the moment somebody adds a line above it. `tools/refcheck` +resolves those anchors against a real tree and writes the line it found, and the next run finds the +same text wherever it has moved to. That much already worked before this module existed. + +What did not work is the case where the anchor still matches and the answer has changed anyway. An +anchor is usually a function signature. The signature is the most stable line in a function and the +body underneath it is the part people edit, so a citation supporting a sentence about what a +function does can go stale without the anchor moving at all. Nothing notices. The checker is green +and the lesson is wrong. + +So a confirmed citation also records a hash of the lines around the anchor. When the anchor +resolves and the hash still matches, nothing near it has changed. When the anchor resolves and the +hash does not, the citation is still valid as a pointer and the text it points at has been edited, +which is the moment to go and read it rather than six months later. + +The hash is over normalised lines: leading and trailing space removed, runs of whitespace collapsed +to one. That is a deliberate choice about what counts as a change. Reindenting a block, or the tab +to space churn that happens when a file gets reformatted, does not fire. Renaming a variable, +adding a branch, or changing an argument does. The alternative, hashing the bytes, gives a checker +that cries every release and gets ignored, which is worse than not having one. + +`RADIUS` is three lines either side, so seven lines in all. Wide enough to cover the signature and +the top of a body, narrow enough that an unrelated edit forty lines away does not drag it in. + +Twelve hex characters of sha256. This detects change, it does not defend against anybody. Nobody is +trying to forge a kernel source file that collides with a hash in a TOML file in this repository, +and a full digest in every citation would be sixty four characters of noise in a file people read. +""" + +from __future__ import annotations + +import hashlib +import re +from dataclasses import dataclass + +# Lines either side of the anchor that go into the hash. +RADIUS = 3 + +# Characters of the digest kept. See the module docstring. +WIDTH = 12 + +SPACE_RE = re.compile(r"\s+") + + +def normalise(line: str) -> str: + """One line reduced to what a change to it would have to survive.""" + return SPACE_RE.sub(" ", line).strip() + + +def window(lines: list[str], index: int, radius: int = RADIUS) -> list[str]: + """The lines around a hit, clipped at both ends of the file. + + Clipped rather than padded. A citation three lines from the top of a file gets a shorter window + and a hash over it, which is correct, where padding would make two different short windows at + two different ends of a file hash the same. + """ + start = max(0, index - radius) + return lines[start : index + radius + 1] + + +def context_hash(lines: list[str], index: int, radius: int = RADIUS) -> str: + """The digest of the normalised window around one line.""" + body = "\n".join(normalise(line) for line in window(lines, index, radius)) + return hashlib.sha256(body.encode("utf-8")).hexdigest()[:WIDTH] + + +def hash_text(text: str, anchor: str, radius: int = RADIUS) -> str: + """The context hash for the first place an anchor appears, or empty when it does not.""" + lines = text.splitlines() + index = next((n for n, line in enumerate(lines) if anchor in line), None) + return "" if index is None else context_hash(lines, index, radius) + + +@dataclass(frozen=True) +class Hit: + """What resolving one anchor against one file found.""" + + anchor: str + line: int = 0 + context: str = "" + count: int = 0 + problem: str = "" + + @property + def found(self) -> bool: + return self.line > 0 + + @property + def unique(self) -> bool: + return self.count == 1 + + def __str__(self) -> str: + if not self.found: + return f"{self.anchor!r}: {self.problem or 'not found'}" + return f"line {self.line}, context {self.context}" + + +def resolve(text: str, anchor: str, radius: int = RADIUS) -> Hit: + """Find an anchor in a file and hash what is around it. + + An anchor that matches more than once still returns the first hit, with the count and a + sentence saying to pick a longer one. Returning nothing would hide a citation that is nearly + right, and returning the first quietly is how a citation ends up pointing at a call site rather + than at the function it thought it named. + """ + lines = text.splitlines() + hits = [number for number, line in enumerate(lines) if anchor in line] + if not hits: + return Hit(anchor=anchor, problem="not in the file any more") + problem = "" + if len(hits) > 1: + problem = f"appears {len(hits)} times, so pick a longer anchor" + return Hit( + anchor=anchor, + line=hits[0] + 1, + context=context_hash(lines, hits[0], radius), + count=len(hits), + problem=problem, + ) + + +def compare(recorded: str, found: str) -> str: + """What to say about two context hashes. + + An empty recorded hash is not a failure. Every citation written before this existed has one, + and turning those into errors on the day the checker landed would have meant either a wall of + red or a rule nobody could turn on. + """ + if not recorded: + return "no context recorded yet" + if not found: + return "nothing to compare against, the anchor did not resolve" + if recorded == found: + return "unchanged" + return f"the lines around it have changed, recorded {recorded} and found {found}" + + +def changed(recorded: str, found: str) -> bool: + """True only when both hashes exist and they differ.""" + return bool(recorded) and bool(found) and recorded != found diff --git a/kxray/source/kconfig.py b/kxray/source/kconfig.py new file mode 100644 index 0000000..5dc543e --- /dev/null +++ b/kxray/source/kconfig.py @@ -0,0 +1,362 @@ +"""Kconfig files, which are where a config symbol says what it is and what it drags in with it. + + from kxray.source import kconfig, tree + + file = kconfig.load(tree.find(), "kernel/Kconfig.preempt") + print(file["PREEMPT"].selects) + print(kconfig.why(file, "PREEMPT_BUILD")) + +`tools/kconfig.py` checks that the config fragments in this repository still say what the lessons +need. This reads the kernel's own Kconfig files, which is the other half: not what we asked for but +what the kernel does about it. + +The pinned kernel is built with `CONFIG_PREEMPT=y`, and a reader who looks at the resulting +`.config` finds `CONFIG_PREEMPT_BUILD=y` and `CONFIG_PREEMPTION=y` in there as well without having +asked for either. `kernel/Kconfig.preempt` is the whole explanation and it is 194 lines long. +`PREEMPT` carries `select PREEMPT_BUILD if !PREEMPT_DYNAMIC`, and `PREEMPT_BUILD` carries +`select PREEMPTION`. One answer pulls in two more. + +The thing to notice in that file is `PREEMPT_BUILD` itself. It is declared `bool` with nothing +after it, and a type with no string after it means no prompt, and no prompt means the symbol never +appears in `menuconfig` and nobody can turn it on by hand. Its value is decided entirely by who +selects it. That is a common shape and it is invisible if you only ever read `.config`, because in +`.config` it looks exactly like something a person chose. + +What this does not do is evaluate anything. `depends on EXPERT && ARCH_SUPPORTS_RT && !COMPILE_TEST` +is kept as the text it is. Working out whether that is true needs every Kconfig file in the tree, +the architecture, and the rest of the config, which is what `scripts/kconfig` is for and is a +program rather than a parser. Reading the condition is enough to answer the question a lesson +actually asks, which is why a symbol is on. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from kxray.models import READ, SKIPPED, UNPARSED, Lines, grid +from kxray.source.tree import File, Tree + +# A type line, optionally followed by the prompt in quotes. No prompt means no menu entry. +TYPES = ("bool", "tristate", "int", "hex", "string", "def_bool", "def_tristate") + +CONFIG_RE = re.compile(r"^(config|menuconfig)\s+(\w+)\s*$") +TYPE_RE = re.compile( + r"^(" + "|".join(TYPES) + r')(?:\s+"(?P(?:[^"\\]|\\.)*)")?\s*(?P.*)$' +) +DEPENDS_RE = re.compile(r"^depends on\s+(.*)$") +SELECT_RE = re.compile(r"^(select|imply)\s+(\w+)\s*(?:if\s+(.*))?$") +DEFAULT_RE = re.compile(r"^default\s+(.*)$") +PROMPT_RE = re.compile(r'^prompt\s+"(?P(?:[^"\\]|\\.)*)"\s*(?P.*)$') +CHOICE_RE = re.compile(r"^(choice|endchoice)\b") +HELP_RE = re.compile(r"^(help|---help---)\s*$") + + +@dataclass(frozen=True) +class Select: + symbol: str + condition: str = "" + + def __str__(self) -> str: + return f"{self.symbol} if {self.condition}" if self.condition else self.symbol + + +@dataclass +class Symbol: + """One `config` or `menuconfig` entry, kept as written.""" + + name: str + kind: str = "config" + type: str = "" + prompt: str = "" + depends: tuple[str, ...] = () + selects: tuple[Select, ...] = () + implies: tuple[Select, ...] = () + defaults: tuple[str, ...] = () + help: str = "" + choice: str = "" + line: int = 0 + source: str = "" + + @property + def visible(self) -> bool: + """Whether a person can find this in `menuconfig` and set it. + + A symbol with a type and no prompt cannot be set directly at all. It is on because + something selected it, and the only way to find out what is to read the files. + """ + return bool(self.prompt) + + @property + def config(self) -> str: + return f"CONFIG_{self.name}" + + def __str__(self) -> str: + shown = self.prompt or "no prompt, so it cannot be set by hand" + return f"{self.config} ({self.type or 'no type'}): {shown}" + + +@dataclass +class Choice: + """A `choice` block: a prompt, and the symbols under it, of which exactly one gets set. + + The block carries lines of its own before any `config` in it. `kernel/Kconfig.preempt` opens + with a prompt and two `default` lines that belong to the choice rather than to any symbol, and + a parser that only knows about symbols drops them on the floor. + """ + + prompt: str = "" + defaults: tuple[str, ...] = () + depends: tuple[str, ...] = () + members: tuple[str, ...] = () + line: int = 0 + + +@dataclass +class KconfigFile: + """One Kconfig file, read. Not a tree of them, and not an evaluator.""" + + source: str = "" + symbols: tuple[Symbol, ...] = () + sourced: tuple[str, ...] = () + choices: tuple[Choice, ...] = () + lines: Lines = field(default_factory=Lines) + unparsed: tuple[tuple[int, str], ...] = () + + def get(self, name: str) -> Symbol | None: + wanted = name.removeprefix("CONFIG_") + return next((s for s in self.symbols if s.name == wanted), None) + + def __getitem__(self, name: str) -> Symbol: + found = self.get(name) + if found is None: + raise KeyError(f"{name} is not declared in {self.source}") + return found + + def __contains__(self, name: str) -> bool: + return self.get(name) is not None + + @property + def names(self) -> tuple[str, ...]: + return tuple(s.name for s in self.symbols) + + @property + def hidden(self) -> tuple[Symbol, ...]: + return tuple(s for s in self.symbols if s.type and not s.visible) + + def selected_by(self, name: str) -> tuple[Symbol, ...]: + """Every symbol in this file that would turn the named one on. + + This is the lookup that makes a `.config` readable. `PREEMPT_BUILD` is on because `PREEMPT` + selects it, and nothing in `.config` says so. + """ + wanted = name.removeprefix("CONFIG_") + return tuple(s for s in self.symbols if any(x.symbol == wanted for x in s.selects)) + + def table(self) -> str: + rows = [("symbol", "type", "prompt", "selects")] + for symbol in self.symbols: + rows.append( + ( + symbol.name, + symbol.type or "none", + symbol.prompt or "none, so it is not in the menu", + ", ".join(str(s) for s in symbol.selects) or "nothing", + ) + ) + return grid(rows) + + +def parse(text: str, source: str = "") -> KconfigFile: + """Read one Kconfig file into its symbols. + + Help text is taken by indentation, which is what the format actually uses. Everything from the + `help` line until a line at the same indentation as the entry belongs to the help. + """ + symbols: list[Symbol] = [] + sourced: list[str] = [] + unparsed: list[tuple[int, str]] = [] + counted = Lines() + current: Symbol | None = None + choices: list[Choice] = [] + choice: Choice | None = None + in_help = False + help_lines: list[str] = [] + + def close() -> None: + nonlocal current, in_help, help_lines + if current is not None: + current.help = "\n".join(help_lines).strip("\n") + symbols.append(current) + if choice is not None: + choice.members = choice.members + (current.name,) + current, in_help, help_lines = None, False, [] + + for number, raw in enumerate(text.splitlines(), start=1): + line = raw.rstrip() + body = line.strip() + + if in_help: + if not body: + help_lines.append("") + counted.count(SKIPPED) + continue + if raw.startswith((" ", "\t")): + help_lines.append(body) + counted.count(SKIPPED) + continue + in_help = False + + if not body or body.startswith("#"): + counted.count(SKIPPED) + continue + + found = CONFIG_RE.match(body) + if found is not None: + close() + current = Symbol( + name=found.group(2), + kind=found.group(1), + choice=choice.prompt if choice is not None else "", + line=number, + source=source, + ) + counted.count(READ) + continue + + chosen = CHOICE_RE.match(body) + if chosen is not None: + close() + if chosen.group(1) == "choice": + choice = Choice(line=number) + choices.append(choice) + else: + choice = None + counted.count(READ) + continue + + if body.startswith("source "): + sourced.append(body.removeprefix("source ").strip().strip('"')) + counted.count(READ) + continue + + if current is None: + # Inside a `choice` block, before any `config`, these lines belong to the choice + # itself. The prompt is the menu entry the four preemption models sit under, and the + # defaults say which one is picked when nobody picks. + if choice is not None: + prompt = PROMPT_RE.match(body) + if prompt is not None: + choice.prompt = prompt.group("prompt") + counted.count(READ) + continue + default = DEFAULT_RE.match(body) + if default is not None: + choice.defaults = choice.defaults + (default.group(1).strip(),) + counted.count(READ) + continue + depends = DEPENDS_RE.match(body) + if depends is not None: + choice.depends = choice.depends + (depends.group(1).strip(),) + counted.count(READ) + continue + # Outside any `config` and outside any `choice`. `menu`, `if`, `endif` and `comment` + # all land here. This parser answers questions about symbols and does not model menus, + # so those are counted as unread rather than quietly passed over. + unparsed.append((number, line)) + counted.count(UNPARSED) + continue + + typed = TYPE_RE.match(body) + if typed is not None: + current.type = typed.group(1) + current.prompt = typed.group("prompt") or "" + counted.count(READ) + continue + + depends = DEPENDS_RE.match(body) + if depends is not None: + current.depends = current.depends + (depends.group(1).strip(),) + counted.count(READ) + continue + + select = SELECT_RE.match(body) + if select is not None: + entry = Select(symbol=select.group(2), condition=(select.group(3) or "").strip()) + if select.group(1) == "select": + current.selects = current.selects + (entry,) + else: + current.implies = current.implies + (entry,) + counted.count(READ) + continue + + default = DEFAULT_RE.match(body) + if default is not None: + current.defaults = current.defaults + (default.group(1).strip(),) + counted.count(READ) + continue + + prompt = PROMPT_RE.match(body) + if prompt is not None: + current.prompt = prompt.group("prompt") + counted.count(READ) + continue + + if HELP_RE.match(body): + in_help = True + help_lines = [] + counted.count(READ) + continue + + unparsed.append((number, line)) + counted.count(UNPARSED) + + close() + return KconfigFile( + source=source, + symbols=tuple(symbols), + sourced=tuple(sourced), + choices=tuple(choices), + lines=counted, + unparsed=tuple(unparsed), + ) + + +def parse_file(file: File) -> KconfigFile: + return parse(file.text, source=f"{file.root}/{file.path}") + + +def load(found: Tree, path: str) -> KconfigFile: + return parse_file(found.read(path)) + + +def why(file: KconfigFile, name: str) -> str: + """One sentence saying how a symbol could be on, out of this file alone. + + Out of this file alone is the caveat that matters and it is in the answer, because a symbol + can be selected from anywhere in the tree and reading one file can only ever rule things in. + """ + symbol = file.get(name) + if symbol is None: + return f"{name} is not declared in {file.source}" + if symbol.visible: + chose = f"{symbol.config} has a prompt, so somebody could have chosen it" + else: + chose = f"{symbol.config} has no prompt, so nobody chose it by hand" + by = file.selected_by(name) + if not by: + return f"{chose}, and nothing in {file.source} selects it" + names = ", ".join(s.config for s in by) + return f"{chose}, and in {file.source} it is selected by {names}" + + +def report(file: KconfigFile) -> str: + lines = [ + file.source, + f"symbols: {len(file.symbols)}", + f"hidden: {len(file.hidden)} with a type and no prompt, so they can only be selected", + f"sourced: {len(file.sourced)} other files", + ] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/source/maintainers.py b/kxray/source/maintainers.py new file mode 100644 index 0000000..664b585 --- /dev/null +++ b/kxray/source/maintainers.py @@ -0,0 +1,393 @@ +"""MAINTAINERS, which answers who to send a patch to and is a glob engine in disguise. + + from kxray.source import maintainers, tree + + book = maintainers.load(tree.find()) + for hit in book.lookup("mm/memory.c"): + print(hit.section.name, hit.tag, hit.pattern) + +The file is a header block describing the format, then a few thousand sections. A section is a name +on its own line and then tagged lines under it: `M:` for a maintainer, `L:` for a list, `S:` for +status, `F:` for the files it covers, `X:` for files it does not, `N:` for a regex over paths and +`K:` for a regex over content. Blank line, next section. + +Four things in here are worth knowing before leaning on any answer. + +A path has more than one maintainer and that is the normal case, not an edge case. On 7.2.2 +`mm/memory.c` matches MEMORY MANAGEMENT, MEMORY MANAGEMENT - CORE and THE REST. `lookup` returns +all of them in file order and refuses to pick, because picking is what `get_maintainer.pl` does +with more information than there is here. + +THE REST is the last section in the file and it carries `F: *` and `F: */`, so it matches every +path in the tree. Any lookup that takes the first hit and stops gets Linus for every file in +Linux, which is a sentence that has been in a lot of first drafts of a lot of tools. + +The globs are not fnmatch. The header block says `F: drivers/net/*` covers the files in that +directory and not the ones below it, and `F: fs/**/*foo*.c` covers subdirectories. So a single star +stops at a slash and a double star does not, which is git's rule rather than Python's. That is also +why THE REST needs two lines: `*` alone matches `MAINTAINERS` and `Makefile` and nothing deeper. + +The difference is not academic. FILESYSTEMS (VFS and infrastructure) carries `F: fs/*`. +`fnmatch.fnmatch("fs/proc/base.c", "fs/*")` is True, and the answer the file gives is False, because +`fs/proc/` belongs to PROC FILESYSTEM and the VFS section said so by using one star instead of a +trailing slash. A tool built on `fnmatch` sends every patch under `fs/` to the VFS maintainers. + +Exclusions are tested first. The header says so in as many words, and ABI/API is the section that +shows it: it covers `include/linux/syscalls.h` and excludes all of `include/uapi/`. + +What this is not is `scripts/get_maintainer.pl`. That script reads git history, weighs how recently +somebody touched a file, and decides who actually gets the mail. This reads the file and reports +what is in it. When the two disagree the script is right, and the value of this one is that it runs +in a notebook with no git and no kernel tree beyond the corpus. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +from kxray.models import READ, SKIPPED, UNPARSED, Lines, grid +from kxray.source.tree import File, Tree + +MAINTAINERS = "MAINTAINERS" + +# Where the section list starts. Everything above this is the block that documents the format, and +# it has lines in it that look exactly like tagged lines because it is describing tagged lines. +HEADING = "Maintainers List" + +# One tagged line: a single capital, a colon, then the value. The separator in the file is a tab, +# and there are entries that use spaces, so this does not insist. +TAG_RE = re.compile(r"^([A-Z]):\s*(.*)$") + +# `FullName `, with the name sometimes quoted because it has a comma in it. +PERSON_RE = re.compile(r'^\s*"?(?P[^"<]*?)"?\s*<(?P
[^>]+)>\s*$') + +# The statuses the header block documents. THE REST does not use one of these, which is the point +# of keeping the list separate from the parser rather than validating against it. +STATUSES = ("Supported", "Maintained", "Odd Fixes", "Orphan", "Obsolete") + +# Statuses that mean somebody is looking at it. Everything else, including the joke on the last +# section of the file, means the answer to "who fixes this" is nobody in particular. +LOOKED_AFTER = ("Supported", "Maintained") + + +@dataclass(frozen=True) +class Person: + name: str + address: str + + def __str__(self) -> str: + return f"{self.name} <{self.address}>" if self.name else self.address + + +def person(text: str) -> Person: + """A name and address line, or the whole line as an address when it is not that shape.""" + found = PERSON_RE.match(text) + if found is None: + return Person(name="", address=text.strip()) + return Person(name=found["name"].strip(), address=found["address"].strip()) + + +@dataclass(frozen=True) +class Section: + """One entry in the file: a name and its tagged lines, kept in the order they were written.""" + + name: str + tags: tuple[tuple[str, str], ...] = () + line: int = 0 + + def values(self, tag: str) -> tuple[str, ...]: + return tuple(value for key, value in self.tags if key == tag) + + @property + def maintainers(self) -> tuple[Person, ...]: + return tuple(person(value) for value in self.values("M")) + + @property + def reviewers(self) -> tuple[Person, ...]: + return tuple(person(value) for value in self.values("R")) + + @property + def lists(self) -> tuple[str, ...]: + return self.values("L") + + @property + def status(self) -> str: + found = self.values("S") + return found[0] if found else "" + + @property + def looked_after(self) -> bool: + return self.status in LOOKED_AFTER + + @property + def files(self) -> tuple[str, ...]: + return self.values("F") + + @property + def excluded(self) -> tuple[str, ...]: + return self.values("X") + + @property + def path_patterns(self) -> tuple[str, ...]: + return self.values("N") + + @property + def content_patterns(self) -> tuple[str, ...]: + return self.values("K") + + @property + def contacts(self) -> tuple[str, ...]: + """Everywhere a patch could go, maintainers first and then the lists. + + A section with no `M:` line is not a bug in the file. PROC FILESYSTEM has none: it has a + status of Maintained and two mailing lists and nobody named, and the honest answer to who + to mail is those lists. + """ + return tuple(str(who) for who in self.maintainers) + self.lists + + +@dataclass(frozen=True) +class Match: + """One section matching one path, and which line in it did the matching.""" + + section: Section + tag: str + pattern: str + + def __str__(self) -> str: + return f"{self.section.name} ({self.tag}: {self.pattern})" + + +@dataclass +class Maintainers: + """The whole file, or as much of it as was in front of the parser.""" + + source: str = "" + partial: bool = False + sections: tuple[Section, ...] = () + unparsed: tuple[tuple[int, str], ...] = field(default_factory=tuple) + lines: Lines = field(default_factory=Lines) + + def get(self, name: str) -> Section | None: + return next((s for s in self.sections if s.name == name), None) + + @property + def names(self) -> tuple[str, ...]: + return tuple(s.name for s in self.sections) + + def lookup(self, path: str) -> tuple[Match, ...]: + """Every section that covers this path, in the order the file lists them. + + `F:` first and then `N:`, and both kinds come back tagged, because the header block says + `get_maintainer.pl` treats them differently and a caller that cares can see which it got. + """ + found: list[Match] = [] + for section in self.sections: + if any(covers(pattern, path) for pattern in section.excluded): + continue + hit = next((p for p in section.files if covers(p, path)), None) + if hit is not None: + found.append(Match(section=section, tag="F", pattern=hit)) + continue + regex = next((p for p in section.path_patterns if _search(p, path)), None) + if regex is not None: + found.append(Match(section=section, tag="N", pattern=regex)) + return tuple(found) + + def specific(self, path: str) -> tuple[Match, ...]: + """The same lookup with the catch-all taken out. + + THE REST matches everything, so it is in every answer and tells nobody anything. This is + the list to print when the question is which subsystem a file belongs to. + """ + return tuple(hit for hit in self.lookup(path) if hit.section.name != CATCH_ALL) + + def contacts(self, path: str) -> tuple[str, ...]: + """Everyone the sections covering this path name, deduplicated, in file order.""" + seen: list[str] = [] + for hit in self.lookup(path): + for who in hit.section.contacts: + if who not in seen: + seen.append(who) + return tuple(seen) + + def content(self, text: str) -> tuple[Match, ...]: + """Sections whose `K:` regex appears in a body of text. + + This is the tag that reads a patch rather than a path, so a patch touching a subsystem's + functions anywhere in the tree can reach that subsystem's list even though the file it + touched belongs to somebody else. + + The flag matters. `scripts/get_maintainer.pl` applies these with perl's `/x`, which ignores + whitespace in the pattern, and this does the same so the two agree. Worth knowing because + the patterns in the file are hand written and not all of them do what they look like: the + one on AUDIT SUBSYSTEM is `\\baudit_[a-z_0-9]\\+\\b`, where `\\+` means one or more in the + basic regular expressions that `grep` speaks and a literal plus sign in the perl that + actually runs it. So it does not match `audit_log_start`, here or in the real script. + """ + found = [] + for section in self.sections: + hit = next((p for p in section.content_patterns if _search(p, text, re.X)), None) + if hit is not None: + found.append(Match(section=section, tag="K", pattern=hit)) + return tuple(found) + + def table(self, path: str) -> str: + rows = [("section", "matched", "status", "contacts")] + for hit in self.lookup(path): + rows.append( + ( + hit.section.name, + f"{hit.tag}: {hit.pattern}", + hit.section.status or "none given", + ", ".join(hit.section.contacts) or "nobody named", + ) + ) + return grid(rows) + + +# The section that covers the whole tree, by name, so that `specific` can leave it out without +# hunting for a section whose patterns happen to match everything. +CATCH_ALL = "THE REST" + + +def _search(pattern: str, text: str, flags: int = 0) -> bool: + """A regex out of the file, applied without letting a bad one take the process down. + + These are perl regexes written by hand in a text file. A pattern Python cannot compile is + reported as not matching rather than raised, because a lookup for one path should not fail on a + pattern belonging to a subsystem nobody asked about. + """ + try: + return re.search(pattern, text, flags) is not None + except re.error: + return False + + +def _glob(pattern: str) -> re.Pattern[str]: + """One `F:` or `X:` pattern as a regex, with the star rule the header block describes. + + A single star stops at a slash and a double star crosses them. `fnmatch` does not make that + distinction and lets one star cross, which turns `drivers/net/*` into every file under + `drivers/net` and is the reason this function exists instead of a one line call. + """ + out = [] + index = 0 + while index < len(pattern): + char = pattern[index] + if char == "*": + if pattern.startswith("**", index): + out.append(".*") + index += 2 + continue + out.append("[^/]*") + elif char == "?": + out.append("[^/]") + else: + out.append(re.escape(char)) + index += 1 + return re.compile("^" + "".join(out) + "$") + + +def covers(pattern: str, path: str) -> bool: + """Whether one pattern covers one path. + + A trailing slash means the directory and everything under it, which the file uses far more + than it uses stars. + """ + if not pattern: + return False + if pattern.endswith("/"): + return _glob(pattern + "**").match(path) is not None + return _glob(pattern).match(path) is not None + + +def parse(text: str, source: str = "", partial: bool = False) -> Maintainers: + """Read the file from the section list down. + + Everything above the `Maintainers List` heading is the block that documents the format, and it + is skipped rather than parsed, because it contains lines like `F: drivers/net/` as examples and + a parser that read those would invent a section covering half the network drivers. + """ + body = text.splitlines() + start = next((n for n, line in enumerate(body) if line.startswith(HEADING)), 0) + + sections: list[Section] = [] + unparsed: list[tuple[int, str]] = [] + counted = Lines() + name = "" + tags: list[tuple[str, str]] = [] + at = 0 + + # Everything above the heading. Counted as skipped rather than ignored, so that the buckets in + # `corpora/BASELINE.toml` still add up to the length of the file. + for _ in range(start): + counted.count(SKIPPED) + + def close() -> None: + """End the block being read, and say what its name line turned out to be worth. + + The name line is counted here rather than where it was seen, because whether it became a + section is not known until the block ends. `Maintainers List` and the underline under it + are two bare lines that never get a tag, and counting those as read would mean the baseline + recorded two sections this parser does not return. + """ + nonlocal name, tags, at + if name and tags: + sections.append(Section(name=name, tags=tuple(tags), line=at)) + counted.count(READ) + elif name: + counted.count(SKIPPED) + name, tags, at = "", [], 0 + + for number, line in enumerate(body[start:], start=start + 1): + if not line.strip(): + counted.count(SKIPPED) + close() + continue + found = TAG_RE.match(line) + if found is not None and name: + tags.append((found.group(1), found.group(2).strip())) + counted.count(READ) + continue + if name and tags: + unparsed.append((number, line)) + counted.count(UNPARSED) + continue + if name: + # A second bare line before any tag. The heading and its underline are the only place + # this happens in the real file, and the second one wins. + counted.count(SKIPPED) + name, at = line.strip(), number + tags = [] + close() + + return Maintainers( + source=source, + partial=partial, + sections=tuple(sections), + unparsed=tuple(unparsed), + lines=counted, + ) + + +def parse_file(file: File) -> Maintainers: + return parse(file.text, source=f"{file.root}/{file.path}", partial=file.partial) + + +def load(found: Tree) -> Maintainers: + """Read MAINTAINERS out of a tree, carrying the tree's partiality into the result.""" + return parse_file(found.read(MAINTAINERS)) + + +def report(book: Maintainers, path: str) -> str: + lines = [ + f"{path} in {book.source}", + f"sections: {len(book.sections)}{' (an excerpt)' if book.partial else ''}", + "", + book.table(path), + ] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/source/symbols.py b/kxray/source/symbols.py new file mode 100644 index 0000000..5ce9e63 --- /dev/null +++ b/kxray/source/symbols.py @@ -0,0 +1,195 @@ +"""A symbol to the file and line that defines it, and the gap between the two names. + + from kxray.source import symbols, tree + + found = tree.find() + for hit in symbols.find(found, "vfs_write", ("fs/read_write.c",)): + print(hit) + + print(symbols.unwrap("__ia32_sys_write")) + +Two halves. The first is text search with C shaped patterns, which is what `grep` in a kernel tree +is and what `cscope` does properly. The second is the part that is actually worth writing down. + +A name off a running kernel and a name in the source are not the same name. There are three names +for the write system call in the pinned build and no two of them are equal. + +`arch/x86/entry/syscalls/syscall_32.tbl` calls the entry point `sys_write`. `fs/read_write.c` has +no line defining anything by that name. What it has, on line 747, is: + + SYSCALL_DEFINE3(write, unsigned int, fd, const char __user *, buf, + +And `/proc/kallsyms` off the running box, in `corpora/proc/tier0/kallsyms-write.txt`, calls it +`__ia32_sys_write`. Macros stand between each pair. + +Grepping that file for `sys_write` is worse than finding nothing. It finds two lines, 728 and 750, +and both of them are `ksys_write`, which is a different function with a different signature that +the real entry point calls on line 750. So the reader lands three lines away from what they wanted, +on something that looks close enough to be believed. + +`unwrap` strips the wrappers a syscall picks up on the way to being a symbol, and `find` knows that +a syscall called `write` is defined by a `SYSCALL_DEFINE` line whose first argument is `write`. + +The wrappers depend on the architecture, which is why they are a list rather than a rule. +`__ia32_` is the 32 bit entry stub, `__x64_` the 64 bit one, `__se_` the sign extending wrapper and +`__do_` the body the other two call. So the same call is `__ia32_sys_write` on the box this project +pins and `__x64_sys_write` on the laptop the reader is sitting at, and a lesson that hardcoded +either one would be wrong for half its readers. + +What this is not is an index. It reads the files it is given and no others, and `find` takes the +paths from the caller for that reason: searching a full kernel tree for a common name means reading +seventy thousand files, and the answer to "where is `write` defined" is not improved by also +finding the eleven hundred other places that word appears. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass + +from kxray.models import grid +from kxray.source.tree import Tree + +# The wrappers a syscall name picks up between the source and the symbol table, longest first so +# that stripping is not order dependent. +WRAPPERS = ( + "__ia32_compat_sys_", + "__x64_compat_sys_", + "__ia32_sys_", + "__x64_sys_", + "__se_sys_", + "__do_sys_", + "compat_sys_", + "sys_", +) + +# What a definition looks like. Each entry is a kind and a template with `{name}` in it, tried in +# order, and the first kind that matches a line names the line. +PATTERNS = ( + ("syscall", r"^(COMPAT_)?SYSCALL_DEFINE\d\(\s*{name}\s*[,)]"), + ("macro", r"^#\s*define\s+{name}\b"), + ("struct", r"^(struct|union|enum)\s+{name}\s*\{{"), + ("typedef", r"^typedef\b.*\b{name}\s*[;(]"), + ("function", r"^[A-Za-z_].*\b{name}\s*\("), + ("variable", r"^[A-Za-z_][\w \t*]*\b{name}\s*(\[[^\]]*\])?\s*="), +) + +# The export macros, which say a symbol is available to modules. Not a definition, and worth +# reporting beside one, because a symbol that is defined and not exported is a symbol a module +# cannot call however visible it looks in the source. +EXPORT_RE = r"^EXPORT_SYMBOL[A-Z_]*\(\s*{name}\s*[,)]" + + +@dataclass(frozen=True) +class Definition: + """Where a symbol is written down, and what kind of thing was written.""" + + name: str + kind: str + path: str + line: int + text: str + + def __str__(self) -> str: + return f"{self.path}:{self.line} {self.kind} {self.text.strip()[:60]}" + + +def unwrap(symbol: str) -> str: + """A symbol table name reduced to the name the source would use. + + `__ia32_sys_write` comes back as `write`, and so does `sys_write`. A name with no wrapper on it + comes back unchanged, because most symbols are not syscalls and guessing at them would be worse + than doing nothing. + """ + for prefix in WRAPPERS: + if symbol.startswith(prefix): + return symbol[len(prefix) :] + return symbol + + +def wrapped(symbol: str) -> bool: + """Whether this name is a syscall entry point rather than a function somebody wrote.""" + return symbol != unwrap(symbol) + + +def _compiled(name: str) -> tuple[tuple[str, re.Pattern[str]], ...]: + quoted = re.escape(name) + return tuple((kind, re.compile(template.format(name=quoted))) for kind, template in PATTERNS) + + +def search(text: str, name: str, path: str = "") -> tuple[Definition, ...]: + """Every line in one file that looks like a definition of this name.""" + patterns = _compiled(name) + found = [] + for number, line in enumerate(text.splitlines(), start=1): + if name not in line: + continue + if line.rstrip().endswith(";"): + # A declaration in a header, or a call. Neither is where the thing lives, and headers + # are full of both. + continue + for kind, pattern in patterns: + if pattern.search(line): + found.append(Definition(name=name, kind=kind, path=path, line=number, text=line)) + break + return tuple(found) + + +def find(tree: Tree, name: str, paths: tuple[str, ...]) -> tuple[Definition, ...]: + """Look for a symbol in a named set of files. + + A syscall is looked for twice, once under the name as given and once with the wrappers off, + because `sys_write` and `write` are the same call and only one of them is in the source. + """ + wanted = [name] + bare = unwrap(name) + if bare != name: + wanted.append(bare) + + found: list[Definition] = [] + for path in paths: + file = tree.open(path) + if file is None: + continue + for candidate in wanted: + found.extend(search(file.text, candidate, path)) + return tuple(found) + + +def exported(tree: Tree, name: str, paths: tuple[str, ...]) -> tuple[Definition, ...]: + """Where a symbol is handed to modules, if it is.""" + pattern = re.compile(EXPORT_RE.format(name=re.escape(name))) + found = [] + for path in paths: + file = tree.open(path) + if file is None: + continue + for number, line in enumerate(file.lines, start=1): + if pattern.search(line): + found.append( + Definition(name=name, kind="export", path=path, line=number, text=line) + ) + return tuple(found) + + +def table(definitions: tuple[Definition, ...]) -> str: + rows = [("symbol", "kind", "where", "line")] + for found in definitions: + rows.append((found.name, found.kind, found.path, str(found.line))) + return grid(rows) + + +def report(tree: Tree, name: str, paths: tuple[str, ...]) -> str: + definitions = find(tree, name, paths) + lines = [f"{name} in {tree.describe()}"] + if wrapped(name): + lines.append(f"name: a syscall wrapper, the source calls it {unwrap(name)}") + if not definitions: + looked = ", ".join(paths) or "nothing" + lines.append(f"found: nothing, having looked in {looked}") + else: + lines.append("") + lines.append(table(definitions)) + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/source/syscalls.py b/kxray/source/syscalls.py new file mode 100644 index 0000000..63f41a5 --- /dev/null +++ b/kxray/source/syscalls.py @@ -0,0 +1,239 @@ +"""The syscall tables, which are where a number stops being a number and becomes a name. + + from kxray.source import syscalls, tree + + table = syscalls.load(tree.find(), "i386") + print(table.by_number(4).name, table.by_number(4).entry) + +`arch/x86/entry/syscalls/syscall_32.tbl` and `syscall_64.tbl` are tab separated text with a header +comment that gives the format in one line: + + [ [noreturn]] + +The single fact worth putting at the top of anything that reads these files is that write is 4 on +i386 and 1 on x86-64, and read is 3 and 0. A syscall number with no architecture attached is not +an identifier, it is a number somebody wrote down while looking at one machine. This project pins +a 32 bit kernel, so every number a lesson prints comes out of the i386 table, and a reader on their +own x86-64 laptop who compares against the numbers they know will find they do not line up. Loading +both tables and printing them side by side is cheaper than explaining that. + +Three shapes in these files catch a reader who assumes four fields per line. + +Nineteen rows in the 32 bit table have a name and no entry point at all: `break`, `stty`, `gtty`, +`ftime`, `prof` and the rest of the calls that were removed decades ago. The number stays reserved +forever because somebody's binary from 1994 might still make the call and has to get ENOSYS rather +than somebody else's syscall. + +Two rows carry a literal `-` where the compat entry point goes, followed by `noreturn`. That dash +is not an entry point named minus, it is a placeholder holding the column open so the word after it +lands in the right place. + +The 64 bit table has three abis in it, `common`, `64` and `x32`, and the same name appears under +more than one of them with different numbers. `rt_sigaction` is one of them. So a lookup by name in +that file has to say which abi it means, and `by_name` returns a tuple rather than pretending +there is one answer. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from kxray.models import READ, SKIPPED, UNPARSED, Lines, grid +from kxray.source.tree import File, Tree + +TABLES = { + "i386": "arch/x86/entry/syscalls/syscall_32.tbl", + "x86_64": "arch/x86/entry/syscalls/syscall_64.tbl", +} + +# The placeholder in the compat column. See the module docstring. +NONE = "-" + +NORETURN = "noreturn" + + +@dataclass(frozen=True) +class Syscall: + """One row: what the number means and what the kernel runs when it arrives.""" + + number: int + abi: str + name: str + entry: str = "" + compat: str = "" + noreturn: bool = False + line: int = 0 + + @property + def implemented(self) -> bool: + """Whether anything runs for this number. + + A row with no entry point is a number the kernel keeps reserved and answers with ENOSYS. + Removing the row instead would let a later call take the number, and then a very old binary + gets a wrong answer rather than an error. + """ + return bool(self.entry) + + def __str__(self) -> str: + where = self.entry or "nothing, the number is reserved" + return f"{self.abi} {self.number} {self.name} -> {where}" + + +@dataclass +class Table: + """One `.tbl` file, read.""" + + source: str = "" + arch: str = "" + calls: tuple[Syscall, ...] = () + unparsed: tuple[tuple[int, str], ...] = () + lines: Lines = field(default_factory=Lines) + + @property + def abis(self) -> tuple[str, ...]: + seen: list[str] = [] + for call in self.calls: + if call.abi not in seen: + seen.append(call.abi) + return tuple(seen) + + def by_number(self, number: int, abi: str = "") -> Syscall | None: + return next( + (c for c in self.calls if c.number == number and (not abi or c.abi == abi)), + None, + ) + + def by_name(self, name: str) -> tuple[Syscall, ...]: + """Every row with this name, because in the 64 bit table there can be more than one.""" + return tuple(c for c in self.calls if c.name == name) + + def number_of(self, name: str, abi: str = "") -> int | None: + """The number for a name, or None when the name is absent or the answer is ambiguous. + + None on ambiguity rather than the first hit. A name that exists under `64` and under `x32` + has two numbers, and returning one of them silently is how a tool ends up making the wrong + call on the machine where it matters. + """ + found = [c for c in self.by_name(name) if not abi or c.abi == abi] + return found[0].number if len(found) == 1 else None + + def entry_of(self, name: str, abi: str = "") -> str: + found = [c for c in self.by_name(name) if not abi or c.abi == abi] + return found[0].entry if len(found) == 1 else "" + + @property + def reserved(self) -> tuple[Syscall, ...]: + return tuple(c for c in self.calls if not c.implemented) + + def holes(self) -> tuple[int, ...]: + """Numbers inside the range that no row claims. + + Different from a reserved row. A hole is a number nobody has written down at all, and the + 64 bit table has a hundred and twenty seven of them because the x32 calls start at 512 and + everything between the end of the common numbers and there is empty. + """ + numbers = {c.number for c in self.calls} + if not numbers: + return () + return tuple(n for n in range(min(numbers), max(numbers)) if n not in numbers) + + def table(self, names: tuple[str, ...]) -> str: + rows = [("name", "number", "abi", "entry point")] + for name in names: + found = self.by_name(name) + if not found: + rows.append((name, "not here", "", "")) + continue + for call in found: + rows.append( + (call.name, str(call.number), call.abi, call.entry or "reserved, no entry") + ) + return grid(rows) + + +def parse(text: str, arch: str = "", source: str = "") -> Table: + """Read a `.tbl` file. Comments and blank lines are skipped, everything else has to fit.""" + calls: list[Syscall] = [] + unparsed: list[tuple[int, str]] = [] + counted = Lines() + + for number, line in enumerate(text.splitlines(), start=1): + body = line.strip() + if not body or body.startswith("#"): + counted.count(SKIPPED) + continue + fields = body.split() + if len(fields) < 3 or not fields[0].isdigit(): + unparsed.append((number, line)) + counted.count(UNPARSED) + continue + counted.count(READ) + entry = fields[3] if len(fields) > 3 else "" + compat = fields[4] if len(fields) > 4 else "" + calls.append( + Syscall( + number=int(fields[0]), + abi=fields[1], + name=fields[2], + entry="" if entry == NONE else entry, + compat="" if compat == NONE else compat, + noreturn=NORETURN in fields[3:], + line=number, + ) + ) + + return Table( + source=source, + arch=arch, + calls=tuple(calls), + unparsed=tuple(unparsed), + lines=counted, + ) + + +def parse_file(file: File, arch: str = "") -> Table: + return parse(file.text, arch=arch, source=f"{file.root}/{file.path}") + + +def load(found: Tree, arch: str = "i386") -> Table: + if arch not in TABLES: + known = ", ".join(sorted(TABLES)) + raise LookupError(f"no table here for {arch}, this reads {known}") + return parse_file(found.read(TABLES[arch]), arch=arch) + + +def compare(left: Table, right: Table, names: tuple[str, ...]) -> str: + """The same calls in two tables, side by side. + + This is the whole argument for keeping both files in the corpus. Nobody argues with + `write 4 1` once they have seen it. + """ + rows = [("name", left.arch or "left", right.arch or "right", "same number")] + for name in names: + here = _one(left, name) + there = _one(right, name) + rows.append((name, here, there, "yes" if here == there and here.isdigit() else "no")) + return grid(rows) + + +def _one(table: Table, name: str) -> str: + """A name's number as a word, saying which of the three ways it can fail to be one number.""" + found = table.by_name(name) + if not found: + return "not in this table" + if len(found) > 1: + return ", ".join(f"{c.number} under {c.abi}" for c in found) + return str(found[0].number) + + +def report(table: Table) -> str: + lines = [ + f"{table.source}", + f"arch: {table.arch or 'not said'}", + f"calls: {len(table.calls)} rows, abis {', '.join(table.abis)}", + f"reserved: {len(table.reserved)} numbers with no entry point", + f"holes: {len(table.holes())} numbers nothing claims", + ] + text = "\n".join(lines) + print(text) + return text diff --git a/kxray/source/tree.py b/kxray/source/tree.py new file mode 100644 index 0000000..5ca670b --- /dev/null +++ b/kxray/source/tree.py @@ -0,0 +1,172 @@ +"""A handle on a kernel tree, including the honest absence of one. + + from kxray.source import tree + + found = tree.find() + print(found.describe()) + print(found.read("MAINTAINERS").partial) + +Everything else in `kxray.source` reads files out of a kernel tree, so every one of them needs an +answer to the same question first: is there a tree here, and how much of one. There are three +answers on a machine somebody might be sitting at, and the difference between them changes what a +lookup is worth. + +The full tree is what `./kxbox/kernel/tree.sh` unpacks, about 1.6 GB of pristine kernel checked +against the sha256 in `pin.toml`. Anything asked of that gets a real answer. + +The partial tree is `corpora/source/pinned/`, which is five files committed to this repository so +that the parsers have something to run against in CI and in a notebook that has downloaded nothing. +It is a kernel tree in shape and almost none of one in content, and a lookup for anything not in it +has to come back as "not here" rather than as "not in the kernel". Those are different sentences +and only one of them is true. + +No tree at all is the state of a fresh checkout before anybody runs the script. That is normal, and +the thing to do about it is say so with the command that fixes it, rather than raise something the +reader has to go and decode. + +One file in the partial tree is an excerpt, `MAINTAINERS.excerpt`, because the real one is 916 KB +and 29847 lines. The `.excerpt` suffix is the whole mechanism: `read("MAINTAINERS")` finds it, +returns it, and sets `partial` on what it hands back, so the fact that this is a slice travels with +the content instead of living in a comment somebody has to remember. +""" + +from __future__ import annotations + +import tomllib +from dataclasses import dataclass +from pathlib import Path + +# Where a full tree lands, relative to the repository root, and what the committed partial one is +# called. Both are looked for in that order, because a reader who has unpacked the real thing wants +# answers from the real thing. +UNPACKED = Path("kxbox/kernel/build/tree") +PINNED = Path("corpora/source/pinned") +PIN = Path("kxbox/kernel/pin.toml") + +# A stand-in for a file too large to commit whole. See the module docstring. +EXCERPT = ".excerpt" + +UNPACK = "./kxbox/kernel/tree.sh" + + +@dataclass(frozen=True) +class File: + """One file out of a tree, knowing where it came from and whether it is all of itself.""" + + path: str + text: str + root: str = "" + partial: bool = False + + @property + def lines(self) -> list[str]: + return self.text.splitlines() + + def banner(self) -> str: + kind = "an excerpt" if self.partial else "whole" + return f"{self.path} ({kind}, from {self.root or 'nowhere'})" + + +@dataclass(frozen=True) +class Tree: + """A directory that holds kernel source, or claims to. + + `complete` is the one field worth reading before anything else. False means this is the + committed corpus, so a missing file means nobody committed it, and a lookup that comes back + empty says nothing at all about the kernel. + """ + + root: Path + version: str = "" + complete: bool = False + + @property + def present(self) -> bool: + return self.root.is_dir() + + def _candidates(self, path: str) -> tuple[Path, Path]: + target = self.root / path + return target, target.with_name(target.name + EXCERPT) + + def has(self, path: str) -> bool: + whole, excerpt = self._candidates(path) + return whole.is_file() or excerpt.is_file() + + def open(self, path: str) -> File | None: + """The file, or None when this tree does not have it. + + None rather than an exception, because "not in this tree" is the ordinary state of most + paths in the partial tree and callers here want to say so in a sentence. + """ + whole, excerpt = self._candidates(path) + for candidate, partial in ((whole, False), (excerpt, True)): + if candidate.is_file(): + return File( + path=path, + text=candidate.read_text(encoding="utf-8", errors="replace"), + root=self.root.as_posix(), + partial=partial, + ) + return None + + def read(self, path: str) -> File: + """The file, or a FileNotFoundError that says what to do about it.""" + found = self.open(path) + if found is not None: + return found + if not self.present: + raise FileNotFoundError(f"no kernel tree at {self.root}, run {UNPACK} to unpack one") + if not self.complete: + raise FileNotFoundError( + f"{path} is not in the committed corpus at {self.root}, " + f"run {UNPACK} for a full tree" + ) + raise FileNotFoundError(f"{path} is not in {self.root}") + + def missing(self, paths: tuple[str, ...]) -> tuple[str, ...]: + return tuple(path for path in paths if not self.has(path)) + + def files(self) -> tuple[str, ...]: + """Every path this tree holds, with the excerpt suffix taken back off. + + Usable on the corpus and a bad idea on a full tree, which has ninety thousand files in it. + """ + found = [] + for path in sorted(self.root.rglob("*")): + if not path.is_file(): + continue + name = path.relative_to(self.root).as_posix() + found.append(name[: -len(EXCERPT)] if name.endswith(EXCERPT) else name) + return tuple(found) + + def describe(self) -> str: + if not self.present: + return f"no kernel tree at {self.root}, run {UNPACK}" + kind = "full" if self.complete else "the committed corpus, a handful of files" + return f"linux {self.version or 'unknown'} at {self.root} ({kind})" + + +def pinned_version(root: Path | str = ".") -> str: + """The version out of `pin.toml`, or an empty string when there is no pin file to read.""" + path = Path(root) / PIN + if not path.is_file(): + return "" + document = tomllib.loads(path.read_text(encoding="utf-8")) + kernel = document.get("kernel") + return str(kernel.get("version", "")) if isinstance(kernel, dict) else "" + + +def find(root: Path | str = ".") -> Tree: + """The best tree this checkout has, full one first. + + The full tree is preferred without asking, because a reader who has spent the 1.6 GB wants + their answers from it, and a partial answer that quietly wins over a complete one is the kind + of thing nobody notices until a lookup disagrees with the source in front of them. + """ + base = Path(root) + version = pinned_version(base) + if version: + unpacked = base / UNPACKED / f"linux-{version}" + if unpacked.is_dir(): + return Tree(root=unpacked, version=version, complete=True) + return Tree(root=base / PINNED, version=version, complete=False) diff --git a/lessons/C09/refs.toml b/lessons/C09/refs.toml index 3c877c4..a01776e 100644 --- a/lessons/C09/refs.toml +++ b/lessons/C09/refs.toml @@ -18,6 +18,7 @@ anchor = "check_noncircular(struct held_lock *src, struct held_lock *target," kernel = "7.2.2" confirmed = true line = 2149 +context = "0a3e6e13796a" note = "The search itself. Before a new edge is added the checker walks the graph looking for a path back, and a path back is a cycle." [[references]] @@ -27,6 +28,7 @@ anchor = "The basic object the validator operates upon" kernel = "7.2.2" confirmed = true line = 11 +context = "ab9ac16b28dc" note = "The design document, and the place that says the checker's unit is the class rather than the instance. This is the paragraph to read before arguing with a report." [[references]] @@ -36,4 +38,5 @@ anchor = "the existing dependency chain (in reverse order) is" kernel = "7.2.2" confirmed = true line = 1973 +context = "0614bdf2cff7" note = "Where the report is printed, which is also where the order of the numbered entries is decided. The printing is what makes #0 the lock being acquired." diff --git a/lessons/S05/refs.toml b/lessons/S05/refs.toml index 0007ecf..fbba4b1 100644 --- a/lessons/S05/refs.toml +++ b/lessons/S05/refs.toml @@ -18,6 +18,7 @@ anchor = "else if (file->f_op->write_iter)" kernel = "7.2.2" confirmed = true line = 686 +context = "3e1a311a08b2" note = "The dispatch itself, inside vfs_write. vfs_write does not know how to write to anything, it reads a pointer out of the file and calls whatever is there. The anchor carries the else and the branch, because the bare expression appears nine times in this file." [[references]] @@ -27,6 +28,7 @@ anchor = "fops_get(inode->i_fop)" kernel = "7.2.2" confirmed = true line = 918 +context = "1b6b3d3cc6fb" note = "Where the pointer comes from. Opening a file copies the ops pointer out of the inode into the file, which is the moment the implementation gets chosen." [[references]] @@ -36,4 +38,5 @@ anchor = "if (file->f_op->write)" kernel = "7.2.2" confirmed = true line = 684 +context = "17ee1d46e88e" note = "The check before the call. vfs_write tries the write slot, then the write_iter slot, and returns EINVAL when both are empty, which is what makes a missing operation an error rather than a jump through a null pointer. This used to cite the file_operations section of Documentation/filesystems/vfs.rst. That section lists the members and describes each one, but it never says which of them may be left out, so it did not hold up the claim that rests on it. Confirming the citation is what made that obvious." diff --git a/lessons/Z02/refs.toml b/lessons/Z02/refs.toml index 9955c5d..0af5166 100644 --- a/lessons/Z02/refs.toml +++ b/lessons/Z02/refs.toml @@ -18,6 +18,7 @@ anchor = "print_graph_entry_leaf(struct trace_iterator *iter," kernel = "7.2.2" confirmed = true line = 947 +context = "727212f21687" note = "The function that prints a call and its return on one line when the call had no children. This is where the single line a leaf gets comes from." [[references]] @@ -27,4 +28,5 @@ anchor = "means that the function exceeded 100 usecs" kernel = "7.2.2" confirmed = true line = 2746 +context = "d6baa892ba5c" note = "The table of slowness markers. The kernel prints the marker and not the threshold, so this table is the only place the thresholds are written down." diff --git a/tests/test_refcheck.py b/tests/test_refcheck.py index 5d66af3..6b2ae8a 100644 --- a/tests/test_refcheck.py +++ b/tests/test_refcheck.py @@ -101,6 +101,19 @@ def test_a_confirmed_citation_still_carries_the_line_it_landed_on(): assert reference.line > 0, f"{reference.identifier} says confirmed with no line" +def test_every_confirmed_citation_carries_a_context_hash(): + """So that a citation whose anchor holds still while the code under it moves gets reported. + + All of them have one now, which is the only reason this can be an assertion rather than a + count. A new citation gets its hash from the same `--confirm` run that gets it its line. + """ + _, references = refcheck.check(ROOT) + for reference in references: + assert refcheck.CONTEXT.fullmatch(reference.context), ( + f"{reference.identifier} has no context hash, run refcheck --tree --confirm" + ) + + # -- paths into this repository --------------------------------------------------------------- @@ -345,21 +358,23 @@ def one_reference(anchor: str = "print_graph_entry_leaf") -> refcheck.Reference: ) -def test_an_anchor_is_found_and_gives_back_the_line(tmp_path): +def test_an_anchor_is_found_and_gives_back_the_line_and_a_hash(tmp_path): where = tree(tmp_path, "one\ntwo\nstatic void print_graph_entry_leaf(void)\n") - assert refcheck.resolve(one_reference(), where) == (3, "") + line, context, problem = refcheck.resolve(one_reference(), where) + assert (line, problem) == (3, "") + assert refcheck.CONTEXT.fullmatch(context) def test_an_anchor_that_moved_is_found_at_its_new_line(tmp_path): """The reason the anchor is text. Ten lines added above it changes nothing.""" where = tree(tmp_path, "\n" * 10 + "static void print_graph_entry_leaf(void)\n") - line, problem = refcheck.resolve(one_reference(), where) + line, _, problem = refcheck.resolve(one_reference(), where) assert (line, problem) == (11, "") def test_an_anchor_that_is_gone_says_so_with_the_file_and_the_text(tmp_path): where = tree(tmp_path, "nothing like it here\n") - line, problem = refcheck.resolve(one_reference(), where) + line, _, problem = refcheck.resolve(one_reference(), where) assert line is None assert "print_graph_entry_leaf" in problem assert "trace_functions_graph.c" in problem @@ -367,15 +382,56 @@ def test_an_anchor_that_is_gone_says_so_with_the_file_and_the_text(tmp_path): def test_an_anchor_that_appears_twice_says_to_pick_a_longer_one(tmp_path): where = tree(tmp_path, "print_graph_entry_leaf\nprint_graph_entry_leaf\n") - line, problem = refcheck.resolve(one_reference(), where) + line, _, problem = refcheck.resolve(one_reference(), where) assert line == 1 assert "appears 2 times" in problem +def test_an_anchor_that_stayed_while_its_surroundings_changed_is_a_finding(tmp_path): + """The failure the anchor cannot catch on its own. + + A signature is the most stable line in a function and the body under it is the part people + edit, so a citation supporting a sentence about behaviour goes stale without moving at all. + The wording is part of the test, because "still there and changed" needs a different answer + from a reader than "gone". + """ + where = tree(tmp_path, "print_graph_entry_leaf\ndoes one thing\n") + _, recorded, _ = refcheck.resolve(one_reference(), where) + + source = where / "kernel" / "trace" / "trace_functions_graph.c" + source.write_text("print_graph_entry_leaf\ndoes something else now\n") + reference = refcheck.Reference( + identifier="Z02-R1", + path="kernel/trace/trace_functions_graph.c", + anchor="print_graph_entry_leaf", + kernel="7.2.2", + context=recorded, + ) + line, found, problem = refcheck.resolve(reference, where) + assert line == 1 + assert found != recorded + assert "is still there and" in problem + assert "so go and read it" in problem + + +def test_a_citation_with_no_recorded_hash_yet_is_not_a_failure(tmp_path): + """Counted rather than failed, or the rule could never have been switched on.""" + where = tree(tmp_path, "print_graph_entry_leaf\n") + _, context, problem = refcheck.resolve(one_reference(), where) + assert problem == "" + assert context + + +def test_a_context_hash_that_is_not_a_hash_is_caught(tmp_path): + refs = REFS.replace("line = 0", 'line = 0\ncontext = "not a hash"') + root = repo(tmp_path, lesson={"meta.toml": META, "refs.toml": refs}) + assert "a context hash is 12 hex digits" in check(root) + + def test_a_file_that_is_not_in_the_tree_says_which_tree(tmp_path): where = tree(tmp_path, "anything\n") missing = refcheck.Reference("Z02-R1", "mm/memory.c", "handle_mm_fault", "7.2.2") - line, problem = refcheck.resolve(missing, where) + line, _, problem = refcheck.resolve(missing, where) assert line is None assert "mm/memory.c is not in" in problem @@ -393,6 +449,38 @@ def test_confirming_writes_the_line_and_the_flag_back_and_keeps_the_comments(tmp assert "# a comment" in written +def test_confirming_writes_a_context_hash_under_the_line(tmp_path): + """An entry written before hashes existed picks one up the first time anybody confirms it. + + Which is why this goes in on its own rather than being a field somebody has to add to seventy + three entries by hand before the check does anything. + """ + root = repo(tmp_path, lesson={"meta.toml": META, "refs.toml": REFS}) + where = tree(tmp_path, "\n" * 6 + "print_graph_entry_leaf\n") + refcheck.confirm(root, where, write=True) + + path = root / "lessons" / "Z02" / "refs.toml" + rows = path.read_text().splitlines() + assert rows[rows.index("line = 7") + 1].startswith("context = ") + + references, _ = refcheck.read_references(path) + assert refcheck.CONTEXT.fullmatch(references[0].context) + + +def test_confirming_twice_gives_the_same_file(tmp_path): + """The rewrite drops any hash it finds and reissues it, so running it again is a no-op.""" + root = repo(tmp_path, lesson={"meta.toml": META, "refs.toml": "# kept\n" + REFS}) + where = tree(tmp_path, "\n" * 6 + "print_graph_entry_leaf\n") + path = root / "lessons" / "Z02" / "refs.toml" + + refcheck.confirm(root, where, write=True) + once = path.read_text() + refcheck.confirm(root, where, write=True) + assert path.read_text() == once + assert once.count("context = ") == 1 + assert "# kept" in once + + def test_resolving_without_confirm_changes_nothing(tmp_path): root = repo(tmp_path, lesson={"meta.toml": META, "refs.toml": REFS}) where = tree(tmp_path, "print_graph_entry_leaf\n") diff --git a/tests/test_source.py b/tests/test_source.py new file mode 100644 index 0000000..698bac5 --- /dev/null +++ b/tests/test_source.py @@ -0,0 +1,493 @@ +"""Tests for reading the kernel's own source files. + +Most of these run against `corpora/source/pinned`, which is five real files out of the pinned +tarball rather than anything written for a test. That is deliberate. A parser for a hand written +text format is only worth what it does on the real thing, and a fixture invented alongside the +parser agrees with the parser by construction. + +The three worth reading are `test_one_star_stops_at_a_slash_where_fnmatch_crosses_it`, +`test_write_is_four_here_and_one_there` and `test_a_signature_that_stays_while_its_body_changes`. +Each is a case where the obvious implementation is wrong in a way nobody notices. +""" + +import re +from pathlib import Path + +import pytest + +from kxray.source import citations, kconfig, maintainers, symbols, syscalls, tree + +ROOT = Path(__file__).resolve().parents[1] +PINNED = ROOT / "corpora" / "source" / "pinned" + +READ_WRITE = "fs/read_write.c" + + +@pytest.fixture +def corpus(): + """The committed corpus, named directly rather than through `find`. + + `find` prefers a full unpacked tree, so a test that went through it would read 1.6 GB of Linux + on a machine that has run `tree.sh` and five files on one that has not, and would be testing a + different thing in each place. + """ + return tree.Tree(root=PINNED, version="7.2.2", complete=False) + + +# -- the tree handle ----------------------------------------------------------------------------- + + +def test_the_corpus_says_it_is_not_a_whole_kernel(corpus): + assert corpus.present + assert corpus.complete is False + assert "handful of files" in corpus.describe() + + +def test_a_file_that_is_here_comes_back_whole(corpus): + file = corpus.read(READ_WRITE) + assert file.partial is False + assert "vfs_write" in file.text + assert len(file.lines) > 1000 + + +def test_the_excerpt_suffix_carries_the_partiality(corpus): + """Asking for MAINTAINERS finds MAINTAINERS.excerpt and says so on the way back. + + The point of the suffix is that nothing downstream has to remember. `partial` travels on the + File, and `parse_file` copies it onto the Maintainers, so a caller three layers away still + knows it is looking at a slice. + """ + file = corpus.read("MAINTAINERS") + assert file.partial is True + assert "MAINTAINERS" in corpus.files() + assert "MAINTAINERS.excerpt" not in corpus.files() + + +def test_a_missing_file_is_not_in_the_corpus_rather_than_not_in_the_kernel(corpus): + """The wording is the test. `mm/memory.c` is very much in Linux and is not committed here.""" + assert corpus.open("mm/memory.c") is None + with pytest.raises(FileNotFoundError) as raised: + corpus.read("mm/memory.c") + message = str(raised.value) + assert "not in the committed corpus" in message + assert tree.UNPACK in message + + +def test_no_tree_at_all_says_how_to_get_one(tmp_path): + empty = tree.Tree(root=tmp_path / "nothing") + assert empty.present is False + with pytest.raises(FileNotFoundError) as raised: + empty.read("MAINTAINERS") + assert tree.UNPACK in str(raised.value) + + +def test_find_prefers_a_full_tree_and_falls_back_to_the_corpus(tmp_path): + (tmp_path / "kxbox" / "kernel").mkdir(parents=True) + (tmp_path / "kxbox" / "kernel" / "pin.toml").write_text('[kernel]\nversion = "7.2.2"\n') + (tmp_path / "corpora" / "source" / "pinned").mkdir(parents=True) + + fallen_back = tree.find(tmp_path) + assert fallen_back.complete is False + assert fallen_back.version == "7.2.2" + + (tmp_path / tree.UNPACKED / "linux-7.2.2").mkdir(parents=True) + assert tree.find(tmp_path).complete is True + + +# -- MAINTAINERS --------------------------------------------------------------------------------- + + +@pytest.fixture +def owners(corpus): + return maintainers.load(corpus) + + +def test_the_excerpt_parses_with_nothing_left_over(owners): + assert owners.partial is True + assert owners.unparsed == () + assert len(owners.sections) == 13 + + +def test_one_star_stops_at_a_slash_where_fnmatch_crosses_it(owners): + """The trap the whole module exists for. + + FILESYSTEMS (VFS and infrastructure) carries `F: fs/*`, and a tool built on `fnmatch` reads + that as everything under `fs/` and sends every proc patch to the VFS maintainers. The kernel's + own header block says one star stops at a slash, so `fs/proc/base.c` is not theirs. + """ + import fnmatch + + assert fnmatch.fnmatch("fs/proc/base.c", "fs/*") is True + assert maintainers.covers("fs/*", "fs/proc/base.c") is False + assert maintainers.covers("fs/*", "fs/read_write.c") is True + assert maintainers.covers("fs/**", "fs/proc/base.c") is True + + names = [hit.section.name for hit in owners.specific("fs/proc/base.c")] + assert "PROC FILESYSTEM" in names + assert not any(name.startswith("FILESYSTEMS") for name in names) + + +def test_a_trailing_slash_means_the_whole_subtree(owners): + assert maintainers.covers("fs/proc/", "fs/proc/base.c") is True + assert maintainers.covers("fs/proc/", "fs/proc/vmcore/main.c") is True + assert maintainers.covers("fs/proc/", "fs/read_write.c") is False + + +def test_a_path_has_more_than_one_owner_and_the_last_one_owns_everything(owners): + """Any lookup that takes the first hit and stops is wrong, and quietly.""" + names = [hit.section.name for hit in owners.lookup("mm/memory.c")] + assert "MEMORY MANAGEMENT" in names + assert maintainers.CATCH_ALL in names + assert names[-1] == maintainers.CATCH_ALL + assert maintainers.CATCH_ALL not in [hit.section.name for hit in owners.specific("mm/memory.c")] + + +def test_the_catch_all_really_does_catch_everything(owners): + rest = owners.get(maintainers.CATCH_ALL) + assert rest is not None + for path in ("mm/memory.c", "fs/proc/base.c", "arch/x86/entry/entry_32.S", "README"): + assert any(maintainers.covers(pattern, path) for pattern in rest.files), path + + +def test_a_section_can_name_nobody_and_still_be_maintained(owners): + """PROC FILESYSTEM has a status, two lists and no `M:` line at all. + + So `contacts` has to fall through to the lists rather than come back empty, because coming back + empty reads as nobody looks after this and that is not what the file says. + """ + proc = owners.get("PROC FILESYSTEM") + assert proc is not None + assert proc.maintainers == () + assert proc.status == "Maintained" + assert proc.looked_after is True + assert len(proc.lists) == 2 + assert proc.contacts == proc.lists + + +def test_an_exclusion_beats_a_match(owners): + """`X:` is tested first, so a section can claim a directory and hand back part of it. + + ABI/API is why the tag is in the corpus at all. Its own two `X:` lines happen not to overlap + its two `F:` lines, so the ordering itself is tested on a section written here, where the + overlap is the whole point and is three lines long. + """ + abi = owners.get("ABI/API") + assert abi is not None + assert abi.excluded == ("arch/*/include/uapi/", "include/uapi/") + + written = maintainers.parse( + "Maintainers List\n" + "----------------\n" + "\n" + "SOMETHING\n" + "M:\tA Person \n" + "S:\tMaintained\n" + "F:\tdrivers/thing/\n" + "X:\tdrivers/thing/legacy/\n" + ) + assert [hit.section.name for hit in written.lookup("drivers/thing/main.c")] == ["SOMETHING"] + assert written.lookup("drivers/thing/legacy/old.c") == () + + +def test_the_n_tag_is_a_regex_and_not_a_glob(owners): + """`N: tegra` matches anywhere in the path, which is what the header block's example says.""" + tegra = [hit for hit in owners.lookup("drivers/soc/tegra/pmc.c") if hit.tag == "N"] + assert [hit.section.name for hit in tegra] == ["TEGRA ARCHITECTURE SUPPORT"] + assert not any( + hit.section.name == "TEGRA ARCHITECTURE SUPPORT" + for hit in owners.lookup("drivers/bus/arm-integrator-lm.c") + ) + + +def test_the_audit_content_pattern_does_not_match_what_it_looks_like(owners): + """`\\baudit_[a-z_0-9]\\+\\b` has a literal plus in it, and the real script agrees. + + `scripts/get_maintainer.pl` applies `K:` patterns with perl's `/x`, where `\\+` is an escaped + plus sign rather than the one-or-more of the basic regular expressions `grep` speaks. So a patch + that touches `audit_log_start` does not reach the audit list through this tag. That is a fact + about the file rather than a bug here, and it is a test so that nobody helpfully fixes it. + """ + audit = owners.get("AUDIT SUBSYSTEM") + assert audit is not None + assert audit.content_patterns + assert owners.content("call to audit_log_start(context);") == () + # What it does match: one character out of the class, then a real plus sign, then a word. + assert any(hit.section.name == "AUDIT SUBSYSTEM" for hit in owners.content("audit_l+x")) + + +def test_a_pattern_python_cannot_compile_does_not_take_the_lookup_down(owners): + assert maintainers._search("(unclosed", "anything") is False + + +# -- the syscall tables -------------------------------------------------------------------------- + + +@pytest.fixture +def tables(corpus): + return syscalls.load(corpus, "i386"), syscalls.load(corpus, "x86_64") + + +def test_both_tables_parse_with_nothing_left_over(tables): + for table in tables: + assert table.unparsed == () + assert table.calls + + +def test_write_is_four_here_and_one_there(tables): + """A syscall number with no architecture attached is not an identifier. + + The pinned box is 32 bit, so every number a lesson prints comes out of the first table, and a + reader on their own machine comparing against the numbers they know finds different ones. + """ + i386, x86_64 = tables + assert i386.number_of("write") == 4 + assert x86_64.number_of("write") == 1 + assert i386.number_of("read") == 3 + assert x86_64.number_of("read") == 0 + assert i386.entry_of("write") == "sys_write" + + +def test_a_name_under_two_abis_has_no_single_number(tables): + """None rather than the first hit, because the first hit is a wrong answer with no warning.""" + _, x86_64 = tables + assert len(x86_64.by_name("rt_sigaction")) == 2 + assert x86_64.number_of("rt_sigaction") is None + assert x86_64.number_of("rt_sigaction", abi="64") == 13 + assert "13 under 64" in syscalls._one(x86_64, "rt_sigaction") + assert syscalls._one(x86_64, "break") == "not in this table" + + +def test_a_reserved_number_has_a_name_and_nothing_to_run(tables): + """Those numbers stay claimed forever so an old binary gets ENOSYS rather than a surprise.""" + i386, _ = tables + dead = i386.by_name("break") + assert len(dead) == 1 + assert dead[0].number == 17 + assert dead[0].implemented is False + assert len(i386.reserved) == 19 + + +def test_the_dash_in_the_compat_column_is_not_an_entry_point(tables): + """It holds the column open so that `noreturn` lands in the right place.""" + parsed = syscalls.parse("1\tcommon\texit\tsys_exit\t-\tnoreturn\n") + only = parsed.calls[0] + assert only.compat == "" + assert only.noreturn is True + assert only.implemented is True + + +def test_the_two_tables_have_different_numbers_of_abis(tables): + i386, x86_64 = tables + assert i386.abis == ("i386",) + assert set(x86_64.abis) == {"common", "64", "x32"} + + +def test_asking_for_an_architecture_nobody_committed_says_which_ones_there_are(corpus): + with pytest.raises(LookupError) as raised: + syscalls.load(corpus, "arm64") + assert "i386" in str(raised.value) + + +# -- Kconfig ------------------------------------------------------------------------------------- + + +@pytest.fixture +def preempt(corpus): + return kconfig.load(corpus, "kernel/Kconfig.preempt") + + +def test_the_preempt_file_parses_with_nothing_left_over(preempt): + assert preempt.unparsed == () + assert len(preempt.symbols) == 15 + + +def test_one_answer_pulls_in_two_more(preempt): + """This is the whole explanation for a `.config` with symbols nobody chose in it.""" + assert "PREEMPT_BUILD" in [s.symbol for s in preempt["PREEMPT"].selects] + assert "PREEMPTION" in [s.symbol for s in preempt["PREEMPT_BUILD"].selects] + assert "CONFIG_" not in preempt["PREEMPT"].name + assert preempt["PREEMPT"].config == "CONFIG_PREEMPT" + assert preempt.get("CONFIG_PREEMPT") is preempt.get("PREEMPT") + + +def test_a_symbol_with_no_prompt_cannot_be_chosen_by_hand(preempt): + """In a `.config` it looks exactly like something a person picked, and it never was.""" + build = preempt["PREEMPT_BUILD"] + assert build.type == "bool" + assert build.prompt == "" + assert build.visible is False + assert build in preempt.hidden + assert len(preempt.hidden) == 6 + + by = [s.name for s in preempt.selected_by("PREEMPT_BUILD")] + assert "PREEMPT" in by + assert "no prompt, so nobody chose it by hand" in kconfig.why(preempt, "PREEMPT_BUILD") + + +def test_the_choice_block_keeps_the_lines_that_belong_to_it(preempt): + """The `default` lines above the first `config` are the choice's, not any symbol's. + + They were the last two unparsed lines in this file. The parser grew a Choice rather than the + number being written down as acceptable. + """ + assert len(preempt.choices) == 1 + block = preempt.choices[0] + assert "Preemption Model" in block.prompt + assert block.defaults + assert "PREEMPT_NONE" in block.members + assert preempt["PREEMPT_NONE"].choice == block.prompt + + +def test_a_condition_is_kept_as_written_rather_than_evaluated(preempt): + """Evaluating one needs every Kconfig in the tree, and reading it answers the question asked.""" + selects = {s.symbol: s.condition for s in preempt["PREEMPT"].selects} + assert selects["PREEMPT_BUILD"] == "!PREEMPT_DYNAMIC" + assert str(kconfig.Select("A", "B")) == "A if B" + assert str(kconfig.Select("A")) == "A" + + +def test_asking_about_a_symbol_that_is_somewhere_else_says_so(preempt): + assert "not declared" in kconfig.why(preempt, "MMU") + with pytest.raises(KeyError): + preempt["MMU"] + + +# -- symbols ------------------------------------------------------------------------------------- + + +def test_three_names_for_one_call_and_no_two_of_them_equal(corpus): + """The table says `sys_write`, the source says `write`, the running kernel says + `__ia32_sys_write`. + + And grepping the source for `sys_write` is worse than finding nothing, because it finds + `ksys_write` twice, which is a different function three lines from the one that was wanted. + """ + text = corpus.read(READ_WRITE).text + assert re.findall(r"\bsys_write\b", text) == [] + assert sorted({m for m in re.findall(r"\w*sys_write\w*", text)}) == ["ksys_write"] + assert "SYSCALL_DEFINE3(write" in text + + assert symbols.unwrap("__ia32_sys_write") == "write" + assert symbols.unwrap("sys_write") == "write" + assert symbols.unwrap("vfs_write") == "vfs_write" + assert symbols.wrapped("__x64_sys_write") is True + assert symbols.wrapped("vfs_write") is False + + +def test_a_wrapped_name_is_looked_for_both_ways(corpus): + found = symbols.find(corpus, "__ia32_sys_write", (READ_WRITE,)) + assert [(f.kind, f.line) for f in found] == [("syscall", 747)] + + +def test_a_plain_function_is_found_where_it_is_defined(corpus): + found = symbols.find(corpus, "vfs_write", (READ_WRITE,)) + assert [(f.kind, f.line) for f in found] == [("function", 667)] + + +def test_a_declaration_is_not_a_definition(): + """A header is full of lines that name a function and are not where it lives.""" + assert ( + symbols.search("ssize_t vfs_write(struct file *, const char __user *);", "vfs_write") == () + ) + + +def test_being_in_kallsyms_and_being_callable_from_a_module_are_different_questions(corpus): + """`vfs_write` is in the symbol table of the running box and a module still cannot call it.""" + assert symbols.exported(corpus, "vfs_write", (READ_WRITE,)) == () + assert symbols.exported(corpus, "rw_verify_area", (READ_WRITE,)) + + +def test_looking_in_a_file_the_corpus_does_not_have_is_quiet(corpus): + assert symbols.find(corpus, "handle_mm_fault", ("mm/memory.c",)) == () + + +# -- citations ----------------------------------------------------------------------------------- + + +ANCHOR = "ssize_t vfs_write(struct file *file" + + +def test_an_anchor_resolves_to_a_line_and_a_hash(corpus): + hit = citations.resolve(corpus.read(READ_WRITE).text, ANCHOR) + assert hit.found is True + assert hit.line == 667 + assert hit.unique is True + assert hit.problem == "" + assert re.fullmatch(r"[0-9a-f]{12}", hit.context) + + +def test_a_signature_that_stays_while_its_body_changes(corpus): + """The failure the anchor alone cannot catch, and the reason the hash exists. + + An anchor is usually a signature, a signature is the most stable line in a function, and the + body underneath it is the part people edit. So a citation supporting a sentence about what the + function does goes stale with the anchor sitting exactly where it always was. + """ + lines = corpus.read(READ_WRITE).lines + before = citations.resolve("\n".join(lines), ANCHOR) + + edited = list(lines) + edited[before.line] = edited[before.line] + " /* a change two lines down */" + after = citations.resolve("\n".join(edited), ANCHOR) + + assert after.line == before.line + assert after.context != before.context + assert citations.changed(before.context, after.context) is True + assert "the lines around it have changed" in citations.compare(before.context, after.context) + + +def test_reindenting_does_not_fire_and_renaming_does(): + """A checker that cries at every reformat gets switched off, which is worse than not having + one.""" + original = "one\n two\nthree anchor here\n\tfour\nfive" + respaced = "one\n\ttwo\nthree anchor here\n four\n five" + renamed = "one\n two\nthree anchor here\n\tfour_renamed\nfive" + + assert citations.hash_text(original, "anchor here") == citations.hash_text( + respaced, "anchor here" + ) + assert citations.hash_text(original, "anchor here") != citations.hash_text( + renamed, "anchor here" + ) + + +def test_an_edit_outside_the_window_is_not_this_citation_going_stale(): + body = [f"line {n}" for n in range(40)] + body[20] = "the anchor is here" + first = citations.resolve("\n".join(body), "the anchor is here") + + body[35] = "something else entirely" + second = citations.resolve("\n".join(body), "the anchor is here") + assert second.context == first.context + + +def test_the_window_is_clipped_rather_than_padded(): + """Padding would make two different short windows at two ends of a file hash the same.""" + lines = ["a", "b", "c", "d", "e", "f", "g"] + assert citations.window(lines, 0) == ["a", "b", "c", "d"] + assert citations.window(lines, 6) == ["d", "e", "f", "g"] + assert citations.window(lines, 3) == lines + + +def test_an_anchor_in_two_places_still_answers_and_says_to_pick_a_longer_one(): + hit = citations.resolve("x\nsame line\ny\nsame line\n", "same line") + assert hit.line == 2 + assert hit.count == 2 + assert hit.unique is False + assert "pick a longer anchor" in hit.problem + + +def test_a_missing_anchor_is_not_found_and_has_no_hash(): + hit = citations.resolve("nothing to see", "an anchor") + assert hit.found is False + assert hit.context == "" + assert citations.hash_text("nothing to see", "an anchor") == "" + + +def test_no_recorded_hash_is_not_a_failure(): + """Every citation written before this existed has none, and turning those red on the day the + checker landed would have meant a rule nobody could switch on.""" + assert citations.changed("", "abc123abc123") is False + assert citations.compare("", "abc123abc123") == "no context recorded yet" + assert citations.compare("abc123abc123", "").startswith("nothing to compare") + assert citations.compare("abc123abc123", "abc123abc123") == "unchanged" diff --git a/tools/baseline.py b/tools/baseline.py index bc39727..fd102f5 100644 --- a/tools/baseline.py +++ b/tools/baseline.py @@ -50,6 +50,9 @@ from kxray.proc import percpu as proc_percpu from kxray.proc import pidstat as proc_pidstat from kxray.proc import version as proc_version +from kxray.source import kconfig as source_kconfig +from kxray.source import maintainers as source_maintainers +from kxray.source import syscalls as source_syscalls from kxray.trace import events, formats, parse_file from kxray.trace import function as trace_function @@ -69,6 +72,7 @@ ("corpora/traces/*/*.txt", "function_graph"), ("corpora/events/*/*.format", "event-format"), ("corpora/proc/*/kallsyms.txt", "kallsyms"), + ("corpora/proc/*/kallsyms-*.txt", "kallsyms"), ("corpora/proc/*/lockdep.txt", "lockdep-classes"), ("corpora/proc/*/lockdep_stats.txt", "lockdep-stats"), ("corpora/proc/*/lockdep-stats-*.txt", "lockdep-stats"), @@ -83,6 +87,13 @@ ("corpora/proc/*/softirqs.txt", "proc-percpu"), ("corpora/proc/*/self-maps.txt", "proc-maps"), ("corpora/proc/*/*-stat.txt", "proc-pidstat"), + # Files out of the pinned tarball rather than off a running kernel. `read_write.c` has no + # reader here on purpose: kxray.source.symbols opens it with a name to look for, so there is no + # whole file count to take, and saying so is better than inventing one. + ("corpora/source/*/MAINTAINERS*", "maintainers"), + ("corpora/source/*/*.tbl", "syscall-table"), + ("corpora/source/*/Kconfig*", "kconfig-source"), + ("corpora/source/*/*.c", "none"), ("corpora/oops/*/*.txt", "lockdep-splat"), ("corpora/btf/*/*.btf", "btf"), ("corpora/experiments/*/*.txt", "none"), @@ -179,6 +190,21 @@ def _proc_version(path: Path) -> tuple[int, Lines | None]: return len(found.parts), found.lines +def _maintainers(path: Path) -> tuple[int, Lines | None]: + found = source_maintainers.parse(path.read_text(encoding="utf-8"), source=path.as_posix()) + return len(found.sections), found.lines + + +def _syscall_table(path: Path) -> tuple[int, Lines | None]: + found = source_syscalls.parse(path.read_text(encoding="utf-8"), source=path.as_posix()) + return len(found.calls), found.lines + + +def _kconfig_source(path: Path) -> tuple[int, Lines | None]: + found = source_kconfig.parse(path.read_text(encoding="utf-8"), source=path.as_posix()) + return len(found.symbols), found.lines + + def _kallsyms(path: Path) -> tuple[int, Lines | None]: text = path.read_text(encoding="utf-8") return len(kallsyms.parse(text)), kallsyms.account(text) @@ -229,6 +255,9 @@ def _unread(path: Path) -> tuple[int, Lines | None]: "proc-maps": _proc_maps, "proc-pidstat": _proc_pidstat, "proc-version": _proc_version, + "maintainers": _maintainers, + "syscall-table": _syscall_table, + "kconfig-source": _kconfig_source, "kallsyms": _kallsyms, "lockdep-classes": _lockdep_classes, "lockdep-stats": _lockdep_stats, diff --git a/tools/refcheck.py b/tools/refcheck.py index 52ea2e6..aec1445 100644 --- a/tools/refcheck.py +++ b/tools/refcheck.py @@ -32,6 +32,14 @@ were files that had moved, and the rest were anchors that matched in more than one place, usually a function name that also appears at a call site or in a forward declaration. Both kinds are the reason this tool exists, and neither would have been caught by reading. + +A confirmed citation also carries a context hash, which covers the failure the anchor cannot. An +anchor is usually a function signature, the signature is the most stable line in a function, and +the body underneath it is the part people edit. So a citation supporting a sentence about what a +function does can go stale without the anchor moving at all. `--confirm` records a hash of the +lines around the anchor, and a later `--tree` run reports "still there and the lines around it have +changed", which is a different sentence from "gone" and needs a different answer from a reader. +`kxray.source.citations` does the hashing and says what it deliberately does not notice. """ from __future__ import annotations @@ -43,6 +51,8 @@ from dataclasses import dataclass from pathlib import Path +from kxray.source import citations + SCHEMA = 1 LESSONS = Path("lessons") @@ -100,6 +110,9 @@ # How a blueprint points at its evidence in the middle of a sentence: `[page-fault-R07]`. CITATION = re.compile(r"\[([a-z0-9][a-z0-9-]*-R\d+)\]") +# What a recorded context hash looks like. The width comes from `kxray.source.citations`. +CONTEXT = re.compile(r"[0-9a-f]{" + str(citations.WIDTH) + "}") + @dataclass(frozen=True) class Finding: @@ -120,6 +133,7 @@ class Reference: kernel: str confirmed: bool = False line: int = 0 + context: str = "" note: str = "" source: str = "" @@ -290,6 +304,7 @@ def read_references(path: Path) -> tuple[list[Reference], list[Finding]]: kernel=str(entry.get("kernel", "")), confirmed=bool(entry.get("confirmed", False)), line=int(entry.get("line", 0) or 0), + context=str(entry.get("context", "")), note=str(entry.get("note", "")), source=str(path), ) @@ -331,6 +346,11 @@ def check_reference(reference: Reference, lesson: str, versions: set[str]) -> li Finding(where, f"names kernel {reference.kernel!r}, which is not a pinned version") ) + if reference.context and not CONTEXT.fullmatch(reference.context): + findings.append( + Finding(where, f"context is {reference.context!r}, and a context hash is 12 hex digits") + ) + if reference.confirmed and reference.line <= 0: findings.append(Finding(where, "says confirmed with no line, so nobody has resolved it")) if reference.line > 0 and not reference.confirmed: @@ -403,27 +423,42 @@ def check_blueprint_citations(document: Path, references: list[Reference]) -> li return findings -def resolve(reference: Reference, tree: Path) -> tuple[int | None, str]: - """Find the anchor in a real kernel tree. Returns the line, and what went wrong if it did not. +def resolve(reference: Reference, tree: Path) -> tuple[int | None, str, str]: + """Find the anchor in a real kernel tree, and hash the lines around it. + + Returns the line, the context hash, and what went wrong if anything did. The first match wins, + and a second match is reported, because an anchor that appears twice is an anchor that will + silently point at the wrong one after the next refactor. - The first match wins, and a second match is reported, because an anchor that appears twice is - an anchor that will silently point at the wrong one after the next refactor. + The hash is the part that is new and it exists for a failure the anchor cannot catch. An anchor + is usually a function signature, the signature is the most stable line in a function, and the + body underneath it is the part people edit. So a citation supporting a sentence about what a + function does can go stale without the anchor moving at all, and until this was recorded + nothing noticed. `kxray.source.citations` does the hashing and says what it normalises away. """ path = tree / reference.path if not path.exists(): - return None, f"{reference.path} is not in {tree}" + return None, "", f"{reference.path} is not in {tree}" try: text = path.read_text(encoding="utf-8", errors="replace") except OSError as problem: - return None, f"cannot read {reference.path}: {problem}" - - lines = text.splitlines() - hits = [number for number, line in enumerate(lines, start=1) if reference.anchor in line] - if not hits: - return None, f"{reference.anchor!r} is not in {reference.path} any more" - if len(hits) > 1: - return hits[0], f"{reference.anchor!r} appears {len(hits)} times, so pick a longer anchor" - return hits[0], "" + return None, "", f"cannot read {reference.path}: {problem}" + + hit = citations.resolve(text, reference.anchor) + if not hit.found: + return None, "", f"{reference.anchor!r} is not in {reference.path} any more" + if hit.problem: + return hit.line, hit.context, f"{reference.anchor!r} {hit.problem}" + if citations.changed(reference.context, hit.context): + return ( + hit.line, + hit.context, + ( + f"{reference.anchor!r} is still there and " + f"{citations.compare(reference.context, hit.context)}, so go and read it" + ), + ) + return hit.line, hit.context, "" # -- putting it together ------------------------------------------------------------------------- @@ -520,37 +555,53 @@ def confirm(root: Path, tree: Path, *, write: bool) -> tuple[list[Finding], int] text = refs_file.read_text(encoding="utf-8") for reference in references: - line, problem = resolve(reference, tree) + line, context, problem = resolve(reference, tree) where = f"{refs_file}#{reference.identifier}" if problem: findings.append(Finding(where, problem)) if line is None: continue resolved += 1 - print(f"{reference.identifier} {reference.path}:{line}") + state = citations.compare(reference.context, context) + print(f"{reference.identifier} {reference.path}:{line} {context} {state}") if write and not problem: - text = _rewrite(text, reference.identifier, line) + text = _rewrite(text, reference.identifier, line, context) if write: refs_file.write_text(text, encoding="utf-8") return findings, resolved -def _rewrite(text: str, identifier: str, line: int) -> str: - """Put the resolved line and the confirmation into one entry, leaving the rest alone. +def _rewrite(text: str, identifier: str, line: int, context: str = "") -> str: + """Put the resolved line, the confirmation and the context hash into one entry. A rewrite rather than a dump of parsed TOML, because the comments in these files are the part a person wrote and a round trip through tomllib would throw all of them away. + + An entry with no `context =` line yet gets one written directly under its `line =`, so that a + citation written before context hashes existed picks one up the first time anybody confirms it + against a tree, without anybody having to go and edit seventy three entries by hand. Any + `context =` already in the entry is dropped and reissued in that same place, which keeps the + field where a reader expects it and makes running this twice give the same file. """ - out = [] + out: list[str] = [] inside = False for row in text.splitlines(): stripped = row.strip() if stripped.startswith("id ="): inside = f'"{identifier}"' in stripped - if inside and stripped.startswith("line ="): - row = f"line = {line}" - elif inside and stripped.startswith("confirmed ="): - row = "confirmed = true" + if not inside: + out.append(row) + continue + if stripped.startswith("context ="): + continue + if stripped.startswith("line ="): + out.append(f"line = {line}") + if context: + out.append(f'context = "{context}"') + continue + if stripped.startswith("confirmed ="): + out.append("confirmed = true") + continue out.append(row) return "\n".join(out) + "\n" @@ -586,7 +637,12 @@ def main(argv: list[str] | None = None) -> int: return 1 waiting = sum(1 for r in references if not r.confirmed) + # Counted rather than failed. Every citation written before context hashes existed has none, + # and a rule that turns those into errors on the day it lands gives a wall of red and gets + # switched off. The number goes down on its own, one `--confirm` run at a time. + bare = sum(1 for r in references if r.confirmed and not r.context) tail = f", {waiting} waiting on a kernel tree" if waiting else "" + tail += f", {bare} confirmed with no context hash yet" if bare else "" print(f"refcheck: paths clean, {len(references)} citation(s){tail}") return 0