From 66f2ef9659615bc8061cac9faa2cdce17ebc0ecf Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 2 Jun 2026 15:13:51 +0100 Subject: [PATCH 01/16] renamed to di.memstats --- di/memstats/init.q | 3 ++ di/memstats/memstats.md | 57 ++++++++++++++++++++++++++ di/memstats/memstats.q | 89 +++++++++++++++++++++++++++++++++++++++++ di/memstats/test.csv | 50 +++++++++++++++++++++++ 4 files changed, 199 insertions(+) create mode 100644 di/memstats/init.q create mode 100644 di/memstats/memstats.md create mode 100644 di/memstats/memstats.q create mode 100644 di/memstats/test.csv diff --git a/di/memstats/init.q b/di/memstats/init.q new file mode 100644 index 00000000..0c8d26cf --- /dev/null +++ b/di/memstats/init.q @@ -0,0 +1,3 @@ +\l ::memstats.q + +export:([objsize;memusageall;memusagevars]) diff --git a/di/memstats/memstats.md b/di/memstats/memstats.md new file mode 100644 index 00000000..751e97f9 --- /dev/null +++ b/di/memstats/memstats.md @@ -0,0 +1,57 @@ +# Memory Usage +This module can be used to calculate the approximate size of an object in memory, or for generating a table containing the approximate size of each object in memory. + +## Main funtions +The module contains two methods for calculating memory usage. + +The `memusage` functions generate a table containing the approximate memoryusage of each object in the kdb session in bytes / megabytes using -22!. This can be useful quick approximations. + +`memusagevars[]`:Generates a table of the approximate memory usage statistics of all variables in a kdb session. + +`memusageall[]`:Generates a table of the approximate memory usage statistics of all variables and views in a kdb session. + +---- + +The `objsize` function is more computationally expensive, it tries to calculate the actual memory size of an object by including nested types and attributes. + +`objsize[]`:Returns the approximate size of an individual kdb object including nested types and attributes. + +## memstats table schema +The memusage table is returned from either the `memusagevars` or `memusageall` functions. + +| Column | Type | Description | +|----------|-------------|---------------------------------------------| +| variable | `symbol` | Namespace and name of variable | +| size | `long` | The approximate size of the object in bytes | +| sizeMB | `int` | The approximatee size of the object in MB | + +## Example +Below is an example of loading the module into a session and viewing the size of different objects. + +```q +\\ Loading the module into a session +memstats: use `di.memstats + +\\ View dictionary of functions +memstats + +\\ Calculating the memory usage of an object + +a:1 / - an atom should return 16 + +b: ([]a:`a`b`c; b:1 2 3) + +memstats.objsize[a] + +memstats.objsize[b] + +// View a and b in the memstats table + +select from memstats.memusagevars[] where variable in `..a`..b + +variable size sizeMB +-------------------- +..b 69 0 +..a 17 0 + +``` \ No newline at end of file diff --git a/di/memstats/memstats.q b/di/memstats/memstats.q new file mode 100644 index 00000000..6bfa1678 --- /dev/null +++ b/di/memstats/memstats.q @@ -0,0 +1,89 @@ +/ library for viewing the approximate memory size of individual kdb objects +/ and viewing the approximate memory usage statistics of a kdb session + +/ functionality to return approximate memory size of kdb+ objects + +attrsize:{ + / `u#2 4 5 unique 32*u + $[`u=a:attr x;32*count distinct x; + / `p#2 2 1 parted (8*u;32*u;8*u+1) + `p=a;8+48*count distinct x; + 0] + }; + +/ (16 bytes + attribute overheads + raw size) to the nearest power of 2 +calcsize:{[c;s;a] `long$2 xexp ceiling 2 xlog 16+a+s*c}; + +vectorsize:{calcsize[count x;typesize x;attrsize x]}; + +/ raw size of atoms according to type, type 20h->76h have 4 bytes pointer size +typesize:{4^0N 1 16 0N 1 2 4 8 4 8 1 8 8 4 4 8 8 4 4 4 abs type x}; + +sampling:{[f;x] + / pick samples randomly accoding to threshold and apply function + threshold:100000; + $[thresholdt:type x;$[-2h=t;32;16]; + / list & enum list + t within 1 76h;vectorsize x; + / exit early for anything above 76h + 76h1000 has no attrbutes (i.e. table unlikely to have 1000 columns, list of strings unlikely to have attr for some objects only + (d[0] within 1 76h)&1=count d:distinct t;calcsize[count x;ptrsize;0]+"j"$scalesampling[{sum calcsize[count each x;typesize x 0;$[10000)&size>0),1,1,does large table show size and sizeMB + +run,0,0,q,tview::select from .test.smalltable,1,1,test memusage and views +true,0,0,q,1~count select from memstats.memusageall[] where variable in `..tview,1,1,is view shown in memusageall table +true,0,0,q,0~count select from memstats.memusagevars[] where variable in `..tview,1,1,no views shown in memusagevars + From ac34b031f58b8ababacc041d289b1d26b5ca32f1 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Wed, 3 Jun 2026 17:16:05 +0100 Subject: [PATCH 02/16] Fixed typos --- di/memstats/memstats.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/di/memstats/memstats.md b/di/memstats/memstats.md index 751e97f9..f19efd61 100644 --- a/di/memstats/memstats.md +++ b/di/memstats/memstats.md @@ -1,7 +1,7 @@ -# Memory Usage +# Memstats This module can be used to calculate the approximate size of an object in memory, or for generating a table containing the approximate size of each object in memory. -## Main funtions +## Main functions The module contains two methods for calculating memory usage. The `memusage` functions generate a table containing the approximate memoryusage of each object in the kdb session in bytes / megabytes using -22!. This can be useful quick approximations. @@ -16,26 +16,26 @@ The `objsize` function is more computationally expensive, it tries to calculate `objsize[]`:Returns the approximate size of an individual kdb object including nested types and attributes. -## memstats table schema +## Memstats table schema The memusage table is returned from either the `memusagevars` or `memusageall` functions. | Column | Type | Description | |----------|-------------|---------------------------------------------| | variable | `symbol` | Namespace and name of variable | | size | `long` | The approximate size of the object in bytes | -| sizeMB | `int` | The approximatee size of the object in MB | +| sizeMB | `int` | The approximate size of the object in MB | ## Example Below is an example of loading the module into a session and viewing the size of different objects. ```q -\\ Loading the module into a session +// Loading the module into a session memstats: use `di.memstats -\\ View dictionary of functions +// View dictionary of functions memstats -\\ Calculating the memory usage of an object +// Calculating the memory usage of an object a:1 / - an atom should return 16 From 886fa87829c3101533294e9490726b76708b97dc Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Fri, 5 Jun 2026 16:52:35 +0100 Subject: [PATCH 03/16] In progress rewrite of di.heartbeat --- di/heartbeat/deps.q | 4 + di/heartbeat/heartbeat.md | 111 ++++++++++++++++++++ di/heartbeat/heartbeat.q | 210 ++++++++++++++++++++++++++++++++++++++ di/heartbeat/init.q | 8 ++ di/heartbeat/test.csv | 56 ++++++++++ 5 files changed, 389 insertions(+) create mode 100644 di/heartbeat/deps.q create mode 100644 di/heartbeat/heartbeat.md create mode 100644 di/heartbeat/heartbeat.q create mode 100644 di/heartbeat/init.q create mode 100644 di/heartbeat/test.csv diff --git a/di/heartbeat/deps.q b/di/heartbeat/deps.q new file mode 100644 index 00000000..42185407 --- /dev/null +++ b/di/heartbeat/deps.q @@ -0,0 +1,4 @@ +/ hard module dependencies and their minimum versions, validated by di.depcheck +/ di.heartbeat has no hard dependencies - all runtime dependencies (log, timer, +/ handlers, pubsub, servers) are injected via init as dictionaries of functions +deps:(`$())!(); diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md new file mode 100644 index 00000000..251cd895 --- /dev/null +++ b/di/heartbeat/heartbeat.md @@ -0,0 +1,111 @@ +# Heartbeat + +This module lets every process publish a periodic heartbeat over pub/sub, and lets +monitoring processes detect when a process has stopped beating - i.e. it is stalled +or blocked - even when the underlying connection is still valid. + +It covers both sides: + +* **Publishing** - a process periodically publishes a heartbeat row over pub/sub. +* **Monitoring** - a process subscribes to other processes' heartbeats, stores the + latest beat per process, and raises a *warning* then an *error* when a process + stops heartbeating within the configured grace periods. + +## Dependencies + +All runtime dependencies are **injected** via `init` as dictionaries of functions, +so the module has no hard dependencies on any other module and runs standalone with +minimal built-in fallbacks (logging to stdout, no-op timer/handlers/pubsub). Inject +real implementations to get full functionality. + +| Dependency | Keys | Default fallback | Purpose | +|------------|------|------------------|---------| +| `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | writes to stdout | logging | +| `timer` | `addjob` `deletejobs` `enablejobs` `disablejobs` `getactivejobs` `cp` | no-op (`cp` returns `.z.p`) | scheduling publish/check/subscribe and the current-time source | +| `handlers` | `register` `remove` `list` | no-op | registering the connection-close (`.z.pc`) cleanup | +| `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | no-op | publishing heartbeats / subscribing to publishers | +| `servers` | `getservers` (`{[proctype]}` returning handles) | returns empty | discovering heartbeat publishers by process type | + +`log`, `timer` and `handlers` follow the standard kdb-x core dependency contracts; +`pubsub` and `servers` are heartbeat-specific. + +## Configuration + +`init[config;deps]` takes a configuration dictionary as its first argument. Any +recognised key may be supplied; unset keys keep their defaults. + +| Key | Default | Description | +|-----|---------|-------------| +| `enabled` | `1b` | publish and check heartbeats | +| `subenabled` | `0b` | act as a monitor: subscribe to other heartbeats and register the disconnect handler | +| `debug` | `1b` | log warning / error transitions | +| `publishinterval` | `0D00:00:30` | how often heartbeats are published | +| `checkinterval` | `0D00:00:10` | how often received heartbeats are checked | +| `warningtolerance` | `1.5` | warning after `warningtolerance*publishinterval` without a beat | +| `errortolerance` | `2f` | error after `errortolerance*publishinterval` without a beat | +| `proctype` | `` `unknown `` | this process's type (published as `sym`) | +| `procname` | `.z.h` | this process's name | +| `pid` `host` `port` | from `.z` | this process's identity | +| `connections` | `` `$() `` | process types this monitor subscribes to (used by `hbsubscriptions`) | +| `onwarning` | no-op | callback invoked with the rows entering warning state | +| `onerror` | no-op | callback invoked with the rows entering error state | + +## Public API + +| Function | Description | +|----------|-------------| +| `init[config;deps]` | wire dependencies and configuration, and schedule the timer jobs | +| `publishheartbeat[]` | publish a single heartbeat row and increment the counter | +| `checkheartbeat[]` | flag processes that have not heartbeated in time | +| `storeheartbeat[batch]` | store incoming heartbeat(s); call from `upd` on the monitor | +| `addprocs[proctypes;procnames]` | seed expected processes so a never-seen process is flagged | +| `subscribe[handles]` | subscribe to heartbeats on the given remote handle(s) | +| `hbsubscriptions[]` | subscribe to all configured publishers (by `connections` process type) | +| `gethb[]` | return the heartbeat store | + +## Heartbeat store schema + +`gethb[]` returns the store, keyed on `sym`,`procname`: + +| Column | Type | Description | +|--------|------|-------------| +| sym | `symbol` | process type | +| procname | `symbol` | process name | +| time | `timestamp` | time of the last received heartbeat | +| counter | `long` | counter from the last heartbeat | +| pid | `int` | publisher process id | +| host | `symbol` | publisher host | +| port | `int` | publisher port | +| warning | `boolean` | process is in warning state | +| error | `boolean` | process is in error state | + +## Example + +```q +// load the module +heartbeat: use `di.heartbeat + +// build dependency dictionaries (here using di.log and di.timer) +log: use `di.log +log.init[()!()] +logdep: `info`warn`error!(log.info;log.warn;log.error) + +timer: use `di.timer +timer.init[()!()] +timerdep: `addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!( + timer.addjob;timer.deletejobs;timer.enablejobs;timer.disablejobs;timer.getactivejobs;timer.cp) + +// initialise as a publishing RDB +heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer!(logdep;timerdep)] + +// publish a heartbeat immediately (normally the timer does this) +heartbeat.publishheartbeat[] +``` + +On the monitoring side, route received heartbeats into the store from `upd` and let +the scheduled `checkheartbeat` raise warnings / errors: + +```q +upd: {[t;x] if[t~`heartbeat; heartbeat.storeheartbeat[x]]; } +heartbeat.gethb[] +``` diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q new file mode 100644 index 00000000..6a09e051 --- /dev/null +++ b/di/heartbeat/heartbeat.q @@ -0,0 +1,210 @@ +/ heartbeat module for kdb-x +/ every process can publish a periodic heartbeat over pub/sub so that downstream +/ monitors can detect when a process has stopped beating - i.e. it is stalled or +/ blocked - even when the underlying connection is still valid +/ module-local state convention: read via .z.M, mutate via .z.m + +/ table used to publish heartbeats - sym holds the publishing process type +heartbeat:([] time:`timestamp$(); sym:`symbol$(); procname:`symbol$(); counter:`long$(); pid:`int$(); host:`symbol$(); port:`int$()); + +/ keyed store of the latest received heartbeat per process, with warning / error state +hb:update warning:0b,error:0b from `sym`procname xkey heartbeat; + +/ remote handles we have already subscribed to for heartbeats +subscribedhandles:`int$(); + +/ heartbeat counter +hbcounter:0; + +/ configuration defaults - overridden by the config dictionary passed to init +enabled:1b; / whether heartbeat publishing / checking is enabled +subenabled:0b; / whether this process monitors (subscribes to) other heartbeats +debug:1b; / whether to log warning / error transitions +publishinterval:0D00:00:30; / how often heartbeats are published +checkinterval:0D00:00:10; / how often received heartbeats are checked +warningtolerance:1.5; / warning after warningtolerance*publishinterval without a beat +errortolerance:2f; / error after errortolerance*publishinterval without a beat +proctype:`unknown; / this process's type (published as sym) +procname:.z.h; / this process's name +pid:.z.i; / this process's pid +host:.z.h; / this process's host +port:`int$system"p"; / this process's port +connections:`$(); / process types this monitor should subscribe to +onwarning:{[procs]}; / callback fired with the rows entering warning state +onerror:{[procs]}; / callback fired with the rows entering error state + +/ recognised configuration keys - anything else passed via config is ignored +configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolerance`errortolerance`proctype`procname`pid`host`port`connections`onwarning`onerror; + +/ minimal default dependencies so the module works standalone (with less functionality) +/ injected real implementations replace these in init, matched key-for-key +defaultlog:`info`warn`error!( + {[ctx;msg] -1 "INFO ",(string ctx),": ",msg;}; + {[ctx;msg] -1 "WARN ",(string ctx),": ",msg;}; + {[ctx;msg] -2 "ERROR ",(string ctx),": ",msg;}); + +defaulttimer:`addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!( + {[id;func;params;period;mode;opts]}; + {[ids]}; + {[ids]}; + {[ids]}; + {[] 0#`id`func`params`period!(`$();();();`int$())}; + {[] .z.p}); + +defaulthandlers:`register`remove`list!( + {[event;name;func]}; + {[event;name]}; + {[event] ()}); + +defaultpubsub:`publish`subscribe!( + {[tbl;data]}; + {[handle]}); + +defaultservers:enlist[`getservers]!enlist {[proctype] `int$()}; + +extractdep:{[deps;name] + / pull a named dependency dictionary out of deps, empty dict when absent or null + $[(name in key deps) and not (::)~deps name;deps name;()!()] + }; + +setdeps:{[deps] + / store injected dependencies, merging over the minimal defaults per key + if[not 99h=type deps;deps:()!()]; + .z.m.log:defaultlog,extractdep[deps;`log]; + .z.m.timer:defaulttimer,extractdep[deps;`timer]; + .z.m.handlers:defaulthandlers,extractdep[deps;`handlers]; + .z.m.pubsub:defaultpubsub,extractdep[deps;`pubsub]; + .z.m.servers:defaultservers,extractdep[deps;`servers]; + }; + +setconfig:{[config] + / apply recognised configuration overrides (a dictionary) on top of current values + if[not 99h=type config;:()]; + ks:configkeys inter key config; + {[k;v] (` sv `.z.m,k) set v}'[ks;config ks]; + }; + +tosecs:{[span] + / convert a timespan into whole seconds for the timer period + `int$span%0D00:00:01 + }; + +registertimers:{ + / schedule the periodic heartbeat jobs via the injected timer (mode 1 = fixed interval) + if[enabled; + .z.M.timer[`addjob][`hbpublish;publishheartbeat;();tosecs publishinterval;1;()!()]; + .z.M.timer[`addjob][`hbcheck;checkheartbeat;();tosecs checkinterval;1;()!()]]; + if[subenabled; + .z.M.timer[`addjob][`hbsubscribe;hbsubscriptions;();60;1;()!()]]; + }; + +registerhandlers:{ + / wire the connection-close cleanup through the injected handler manager + if[subenabled; + .z.M.handlers[`register][`.z.pc;`heartbeat;closeconnection]]; + }; + +init:{[config;deps] + / initialise the module with configuration and injected dependencies - see heartbeat.md + / config - dictionary of configuration overrides (see configkeys), or (::) for defaults + / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions + setdeps deps; + setconfig config; + registertimers[]; + registerhandlers[]; + .z.M.log[`info][`heartbeat;"di.heartbeat initialised"]; + }; + +publishheartbeat:{ + / publish a single heartbeat row over pub/sub and bump the counter + if[not enabled;:()]; + .z.M.pubsub[`publish][`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(.z.M.timer[`cp][];proctype;procname;.z.M.hbcounter;pid;host;port)]; + .z.m.hbcounter:.z.M.hbcounter+1; + }; + +storeheartbeat:{[batch] + / store one or more incoming heartbeats, keeping the latest per process and + / clearing warning / error state - call this from upd when a heartbeat arrives + .z.m.hb:.z.M.hb upsert update warning:0b,error:0b from select by sym,procname from batch; + }; + +addprocs:{[proctypes;procnames] + / seed the store with expected processes so a never-seen process is flagged + / real heartbeats arriving later override these seeded rows + seed:2!([]sym:proctypes,();procname:procnames,();time:.z.M.timer[`cp][];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); + .z.m.hb:seed,.z.M.hb; + }; + +logwarnproc:{[r] + / log a single process moving into warning state + .z.M.log[`warn][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + }; + +logerrproc:{[r] + / log a single process moving into error state + .z.M.log[`error][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + }; + +warn:{[procs] + / move processes into warning state, log and fire the warning callback + if[debug;logwarnproc each 0!procs]; + .z.m.hb:.z.M.hb upsert select sym,procname,warning:1b from procs; + .z.M.onwarning procs; + }; + +err:{[procs] + / move processes into error state, log and fire the error callback + if[debug;logerrproc each 0!procs]; + .z.m.hb:.z.M.hb upsert select sym,procname,error:1b from procs; + .z.M.onerror procs; + }; + +warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; +errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; + +checkheartbeat:{ + / flag processes that have not heartbeated within the warning / error grace periods + / status: 0 healthy, 1 warning, 2+ error + now:.z.M.timer[`cp][]; + stats:0!update + status:(`short$now>time+warningperiod each sym)+`short$2*now>time+errorperiod each sym + from .z.M.hb; + newwarn:select sym,procname,time from stats where status=1,not warning; + newerr:select sym,procname,time from stats where status>1,not error; + if[count newwarn;warn newwarn]; + if[count newerr;err newerr]; + }; + +subscribeone:{[h] + / subscribe to a single remote heartbeat publisher, logging and skipping on failure + ok:.[{.z.M.pubsub[`subscribe] x;1b};h;{[h;e] .z.M.log[`error][`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + if[ok;.z.m.subscribedhandles:distinct .z.M.subscribedhandles,h]; + }; + +subscribe:{[handles] + / subscribe to heartbeats on the given remote handle(s), tracking successful subscriptions + subscribeone each (),handles; + }; + +getheartbeats:{[proctype] + / subscribe to publishers of the given process type(s) that are not yet subscribed + handles:(.z.M.servers[`getservers] proctype) except .z.M.subscribedhandles; + if[count handles; + .z.M.log[`info][`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; + subscribe handles]; + }; + +hbsubscriptions:{ + / subscribe to all configured heartbeat publishers (by configured process type) + getheartbeats connections; + }; + +closeconnection:{[h] + / drop a closed handle from the tracked subscriptions - registered against .z.pc + .z.m.subscribedhandles:.z.M.subscribedhandles except h; + }; + +gethb:{ + / return the current heartbeat store for inspection + .z.M.hb + }; \ No newline at end of file diff --git a/di/heartbeat/init.q b/di/heartbeat/init.q new file mode 100644 index 00000000..77ff3d0e --- /dev/null +++ b/di/heartbeat/init.q @@ -0,0 +1,8 @@ +/ load core functionality into the module +\l ::heartbeat.q + +/ module version - compared against dependants' minimum requirements by di.depcheck +version:"0.1.0"; + +/ public api - only the functions intended to be called externally are exported +export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;hbsubscriptions;gethb;version]) diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv new file mode 100644 index 00000000..0478e8a9 --- /dev/null +++ b/di/heartbeat/test.csv @@ -0,0 +1,56 @@ +action,ms,bytes,lang,code,repeat,minver,comment +comment,,,,,,,Setup - load module and inject mock dependencies +before,0,0,q,heartbeat:use`di.heartbeat,1,1,load the heartbeat module +before,0,0,q,logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,silent log mock +before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the timer mock +before,0,0,q,timerdep:`addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!({[id;func;params;period;mode;opts]};{[ids]};{[ids]};{[ids]};{[]([]id:`$())};{[].test.now}),1,1,timer mock with controllable cp +before,0,0,q,psdep:`publish`subscribe!({[t;x] .test.pubt:t;.test.pubx:x;.test.pubcount:.test.pubcount+1};{[h] .test.subh:h}),1,1,pubsub mock capturing publish and subscribe +before,0,0,q,.test.pubcount:0,1,1,initialise publish counter +before,0,0,q,deps:`log`timer`pubsub!(logdep;timerdep;psdep),1,1,assemble dependency dictionary +before,0,0,q,heartbeat.init[`proctype`procname!(`rdb;`rdb1);deps],1,1,initialise with identity config and mocks + +comment,,,,,,,publishheartbeat publishes a row and bumps the counter +run,0,0,q,.test.pubcount:0,1,1,reset publish counter +run,0,0,q,heartbeat.publishheartbeat[],1,1,publish a heartbeat +true,0,0,q,1=.test.pubcount,1,1,publish dependency was called once +true,0,0,q,`heartbeat~.test.pubt,1,1,published to the heartbeat table +true,0,0,q,`rdb~(first .test.pubx)`sym,1,1,published row carries the process type +true,0,0,q,1=.m.di.0heartbeat.hbcounter,1,1,counter incremented + +comment,,,,,,,disabled publishing is a no-op +run,0,0,q,.m.di.0heartbeat.enabled:0b,1,1,disable heartbeating +run,0,0,q,.test.pubcount:0,1,1,reset publish counter +run,0,0,q,heartbeat.publishheartbeat[],1,1,attempt to publish while disabled +true,0,0,q,0=.test.pubcount,1,1,nothing published when disabled +run,0,0,q,.m.di.0heartbeat.enabled:1b,1,1,re-enable heartbeating + +comment,,,,,,,storeheartbeat records incoming beats with cleared state +run,0,0,q,.m.di.0heartbeat.hb:0#.m.di.0heartbeat.hb,1,1,clear the store +run,0,0,q,.test.batch:([]time:enlist .test.now;sym:enlist`rdb;procname:enlist`rdb1;counter:enlist 1;pid:enlist 1i;host:enlist`h;port:enlist 5000i),1,1,build an incoming heartbeat +run,0,0,q,heartbeat.storeheartbeat[.test.batch],1,1,store the heartbeat +true,0,0,q,1=count heartbeat.gethb[],1,1,one process recorded +true,0,0,q,not first exec warning from heartbeat.gethb[],1,1,stored beat has warning cleared + +comment,,,,,,,addprocs seeds an expected process +run,0,0,q,heartbeat.addprocs[`rdb;`rdb2],1,1,seed an expected process +true,0,0,q,`rdb2 in exec procname from heartbeat.gethb[],1,1,seeded process present + +comment,,,,,,,checkheartbeat flags warning then error as time passes +run,0,0,q,.m.di.0heartbeat.hb:0#.m.di.0heartbeat.hb,1,1,clear the store +run,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,reset time to t0 +run,0,0,q,heartbeat.addprocs[`rdb;`stale],1,1,seed a process that will go stale +run,0,0,q,.test.now:2025.01.01D00:00:50.000,1,1,advance 50s past the 45s warning period +run,0,0,q,heartbeat.checkheartbeat[],1,1,run the check +true,0,0,q,first exec warning from heartbeat.gethb[] where procname=`stale,1,1,process moved to warning +true,0,0,q,not first exec error from heartbeat.gethb[] where procname=`stale,1,1,not yet in error +run,0,0,q,.test.now:2025.01.01D00:02:00.000,1,1,advance past the 60s error period +run,0,0,q,heartbeat.checkheartbeat[],1,1,run the check again +true,0,0,q,first exec error from heartbeat.gethb[] where procname=`stale,1,1,process moved to error + +comment,,,,,,,subscribe tracks handles and closeconnection removes them +run,0,0,q,.m.di.0heartbeat.subscribedhandles:`int$(),1,1,clear tracked handles +run,0,0,q,heartbeat.subscribe[5i],1,1,subscribe to a remote handle +true,0,0,q,5i in .m.di.0heartbeat.subscribedhandles,1,1,handle tracked after subscribe +true,0,0,q,5i~.test.subh,1,1,subscribe dependency was invoked with the handle +run,0,0,q,.m.di.0heartbeat.closeconnection[5i],1,1,simulate connection close +true,0,0,q,not 5i in .m.di.0heartbeat.subscribedhandles,1,1,handle removed on close From 8e5d41d34b4f2ec0af1fa240711e6a3cc2de3dd7 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 9 Jun 2026 11:57:59 +0100 Subject: [PATCH 04/16] Changed structure to follow the dependency guide --- di/heartbeat/heartbeat.md | 50 ++++++++----- di/heartbeat/heartbeat.q | 144 +++++++++++++++++++------------------- di/heartbeat/test.csv | 6 ++ 3 files changed, 110 insertions(+), 90 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 251cd895..b4be2b5d 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -13,21 +13,28 @@ It covers both sides: ## Dependencies -All runtime dependencies are **injected** via `init` as dictionaries of functions, -so the module has no hard dependencies on any other module and runs standalone with -minimal built-in fallbacks (logging to stdout, no-op timer/handlers/pubsub). Inject -real implementations to get full functionality. - -| Dependency | Keys | Default fallback | Purpose | -|------------|------|------------------|---------| -| `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | writes to stdout | logging | -| `timer` | `addjob` `deletejobs` `enablejobs` `disablejobs` `getactivejobs` `cp` | no-op (`cp` returns `.z.p`) | scheduling publish/check/subscribe and the current-time source | -| `handlers` | `register` `remove` `list` | no-op | registering the connection-close (`.z.pc`) cleanup | -| `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | no-op | publishing heartbeats / subscribing to publishers | -| `servers` | `getservers` (`{[proctype]}` returning handles) | returns empty | discovering heartbeat publishers by process type | +All runtime dependencies are **injected** via `init` as dictionaries of functions +(`` `dependency!(dict of functions) ``). They are **required** - `init` errors +immediately with a clear message if a required dependency is missing. There is no +hard dependency on any other module: any module exporting the contracted function +signatures can be supplied. + +| Dependency | Keys | Required | Purpose | +|------------|------|----------|---------| +| `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | always | logging | +| `timer` | `addjob` `deletejobs` `enablejobs` `disablejobs` `getactivejobs` `cp` | always | scheduling publish/check/subscribe and the current-time source (`cp`) | +| `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | +| `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | +| `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | `log`, `timer` and `handlers` follow the standard kdb-x core dependency contracts; -`pubsub` and `servers` are heartbeat-specific. +`pubsub` and `servers` are heartbeat-specific. `servers` and `handlers` are only +required when `subenabled` is set (i.e. this process monitors other heartbeats); +a pure publisher needs only `log`, `timer` and `pubsub`. + +Only the functions the module actually calls are accessed (`timer`'s `addjob`/`cp`, +`handlers`' `register`), but supplying the full contracted dictionary keeps the +dependency interchangeable with the real `di.*` modules. ## Configuration @@ -85,18 +92,23 @@ recognised key may be supplied; unset keys keep their defaults. // load the module heartbeat: use `di.heartbeat -// build dependency dictionaries (here using di.log and di.timer) -log: use `di.log -log.init[()!()] -logdep: `info`warn`error!(log.info;log.warn;log.error) +// build dependency dictionaries (here from di.log and di.timer) +// note: bound as logmod, not log, since log is a reserved q word +logmod: use `di.log +logmod.init[()!()] +logdep: `info`warn`error!(logmod.info;logmod.warn;logmod.error) timer: use `di.timer timer.init[()!()] timerdep: `addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!( timer.addjob;timer.deletejobs;timer.enablejobs;timer.disablejobs;timer.getactivejobs;timer.cp) -// initialise as a publishing RDB -heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer!(logdep;timerdep)] +// a pubsub dependency must provide publish[table;data] and subscribe[handle] +pubsub: use `di.pubsub +psdep: `publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) + +// initialise as a publishing RDB (log, timer and pubsub are all required) +heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(logdep;timerdep;psdep)] // publish a heartbeat immediately (normally the timer does this) heartbeat.publishheartbeat[] diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 6a09e051..3b8c4709 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -2,7 +2,12 @@ / every process can publish a periodic heartbeat over pub/sub so that downstream / monitors can detect when a process has stopped beating - i.e. it is stalled or / blocked - even when the underlying connection is still valid -/ module-local state convention: read via .z.M, mutate via .z.m +/ the module handles both publishing heartbeats and, on the monitoring side, +/ storing received heartbeats and raising warnings / errors when they stop +/ runtime dependencies are injected via init and are required - the module errors +/ immediately if a required dependency is missing - see heartbeat.md +/ module-local state convention: read config bare, mutate via .z.m, and access +/ injected dependencies via .z.m at every call site / table used to publish heartbeats - sym holds the publishing process type heartbeat:([] time:`timestamp$(); sym:`symbol$(); procname:`symbol$(); counter:`long$(); pid:`int$(); host:`symbol$(); port:`int$()); @@ -36,52 +41,43 @@ onerror:{[procs]}; / callback fired with the rows entering error state / recognised configuration keys - anything else passed via config is ignored configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolerance`errortolerance`proctype`procname`pid`host`port`connections`onwarning`onerror; -/ minimal default dependencies so the module works standalone (with less functionality) -/ injected real implementations replace these in init, matched key-for-key -defaultlog:`info`warn`error!( - {[ctx;msg] -1 "INFO ",(string ctx),": ",msg;}; - {[ctx;msg] -1 "WARN ",(string ctx),": ",msg;}; - {[ctx;msg] -2 "ERROR ",(string ctx),": ",msg;}); - -defaulttimer:`addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!( - {[id;func;params;period;mode;opts]}; - {[ids]}; - {[ids]}; - {[ids]}; - {[] 0#`id`func`params`period!(`$();();();`int$())}; - {[] .z.p}); - -defaulthandlers:`register`remove`list!( - {[event;name;func]}; - {[event;name]}; - {[event] ()}); - -defaultpubsub:`publish`subscribe!( - {[tbl;data]}; - {[handle]}); - -defaultservers:enlist[`getservers]!enlist {[proctype] `int$()}; +/ warning / error grace periods - vary by process type if required +warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; +errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; -extractdep:{[deps;name] - / pull a named dependency dictionary out of deps, empty dict when absent or null - $[(name in key deps) and not (::)~deps name;deps name;()!()] +requiredep:{[deps;name] + / extract a required dependency dictionary, erroring immediately if absent or null + d:$[99h=type deps;$[(name in key deps) and not (::)~deps name;deps name;()!()];()!()]; + if[not count d; + '"di.heartbeat: ",(string name)," dependency is required; pass it via init deps - see di.",string name]; + d }; setdeps:{[deps] - / store injected dependencies, merging over the minimal defaults per key - if[not 99h=type deps;deps:()!()]; - .z.m.log:defaultlog,extractdep[deps;`log]; - .z.m.timer:defaulttimer,extractdep[deps;`timer]; - .z.m.handlers:defaulthandlers,extractdep[deps;`handlers]; - .z.m.pubsub:defaultpubsub,extractdep[deps;`pubsub]; - .z.m.servers:defaultservers,extractdep[deps;`servers]; + / extract and store the required dependencies, flattened for access via .z.m + / log, timer and pubsub are always required; servers and handlers only when monitoring + logdict:requiredep[deps;`log]; + .z.m.loginfo:logdict`info; + .z.m.logwarn:logdict`warn; + .z.m.logerr:logdict`error; + timerdict:requiredep[deps;`timer]; + .z.m.timeraddjob:timerdict`addjob; + .z.m.timercp:timerdict`cp; + pubsubdict:requiredep[deps;`pubsub]; + .z.m.pubsubpublish:pubsubdict`publish; + .z.m.pubsubsubscribe:pubsubdict`subscribe; + if[subenabled; + serversdict:requiredep[deps;`servers]; + .z.m.serversgetservers:serversdict`getservers; + handlersdict:requiredep[deps;`handlers]; + .z.m.handlersregister:handlersdict`register]; }; setconfig:{[config] / apply recognised configuration overrides (a dictionary) on top of current values - if[not 99h=type config;:()]; - ks:configkeys inter key config; - {[k;v] (` sv `.z.m,k) set v}'[ks;config ks]; + cfg:$[99h=type config;config;()!()]; + ks:configkeys inter key cfg; + (.Q.dd[.z.M] each ks) set' cfg ks; }; tosecs:{[span] @@ -92,83 +88,89 @@ tosecs:{[span] registertimers:{ / schedule the periodic heartbeat jobs via the injected timer (mode 1 = fixed interval) if[enabled; - .z.M.timer[`addjob][`hbpublish;publishheartbeat;();tosecs publishinterval;1;()!()]; - .z.M.timer[`addjob][`hbcheck;checkheartbeat;();tosecs checkinterval;1;()!()]]; + .z.m.timeraddjob[`hbpublish;publishheartbeat;();tosecs publishinterval;1;()!()]; + .z.m.timeraddjob[`hbcheck;checkheartbeat;();tosecs checkinterval;1;()!()]]; if[subenabled; - .z.M.timer[`addjob][`hbsubscribe;hbsubscriptions;();60;1;()!()]]; + .z.m.timeraddjob[`hbsubscribe;hbsubscriptions;();60;1;()!()]]; }; registerhandlers:{ / wire the connection-close cleanup through the injected handler manager if[subenabled; - .z.M.handlers[`register][`.z.pc;`heartbeat;closeconnection]]; + .z.m.handlersregister[`.z.pc;`heartbeat;closeconnection]]; }; init:{[config;deps] / initialise the module with configuration and injected dependencies - see heartbeat.md / config - dictionary of configuration overrides (see configkeys), or (::) for defaults - / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions - setdeps deps; + / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: + / `log - `info`warn`error - required + / `timer - `addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp - required + / `pubsub - `publish`subscribe - required + / `servers - `getservers - required when subenabled + / `handlers - `register`remove`list - required when subenabled + / example: + / heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(logdep;timerdep;psdep)] setconfig config; + setdeps deps; registertimers[]; registerhandlers[]; - .z.M.log[`info][`heartbeat;"di.heartbeat initialised"]; + .z.m.loginfo[`heartbeat;"di.heartbeat initialised"]; }; publishheartbeat:{ / publish a single heartbeat row over pub/sub and bump the counter if[not enabled;:()]; - .z.M.pubsub[`publish][`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(.z.M.timer[`cp][];proctype;procname;.z.M.hbcounter;pid;host;port)]; - .z.m.hbcounter:.z.M.hbcounter+1; + .z.m.pubsubpublish[`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(.z.m.timercp[];proctype;procname;hbcounter;pid;host;port)]; + .z.m.hbcounter:hbcounter+1; }; storeheartbeat:{[batch] / store one or more incoming heartbeats, keeping the latest per process and / clearing warning / error state - call this from upd when a heartbeat arrives - .z.m.hb:.z.M.hb upsert update warning:0b,error:0b from select by sym,procname from batch; + .z.m.hb:hb upsert update warning:0b,error:0b from select by sym,procname from batch; }; addprocs:{[proctypes;procnames] / seed the store with expected processes so a never-seen process is flagged / real heartbeats arriving later override these seeded rows - seed:2!([]sym:proctypes,();procname:procnames,();time:.z.M.timer[`cp][];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); - .z.m.hb:seed,.z.M.hb; + seed:2!([]sym:proctypes,();procname:procnames,();time:.z.m.timercp[];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); + .z.m.hb:seed,hb; }; logwarnproc:{[r] / log a single process moving into warning state - .z.M.log[`warn][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.logwarn[`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; logerrproc:{[r] / log a single process moving into error state - .z.M.log[`error][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.logerr[`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; warn:{[procs] / move processes into warning state, log and fire the warning callback if[debug;logwarnproc each 0!procs]; - .z.m.hb:.z.M.hb upsert select sym,procname,warning:1b from procs; - .z.M.onwarning procs; + .z.m.hb:hb upsert select sym,procname,warning:1b from procs; + onwarning procs; }; err:{[procs] / move processes into error state, log and fire the error callback if[debug;logerrproc each 0!procs]; - .z.m.hb:.z.M.hb upsert select sym,procname,error:1b from procs; - .z.M.onerror procs; + .z.m.hb:hb upsert select sym,procname,error:1b from procs; + onerror procs; }; -warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; -errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; - checkheartbeat:{ / flag processes that have not heartbeated within the warning / error grace periods / status: 0 healthy, 1 warning, 2+ error - now:.z.M.timer[`cp][]; - stats:0!update - status:(`short$now>time+warningperiod each sym)+`short$2*now>time+errorperiod each sym - from .z.M.hb; + / grace periods are computed as locals first - module functions do not resolve inside qsql + now:.z.m.timercp[]; + t:0!hb; + wp:warningperiod each t`sym; + ep:errorperiod each t`sym; + stats:update status:(`short$now>time+wp)+`short$2*now>time+ep from t; newwarn:select sym,procname,time from stats where status=1,not warning; newerr:select sym,procname,time from stats where status>1,not error; if[count newwarn;warn newwarn]; @@ -177,8 +179,8 @@ checkheartbeat:{ subscribeone:{[h] / subscribe to a single remote heartbeat publisher, logging and skipping on failure - ok:.[{.z.M.pubsub[`subscribe] x;1b};h;{[h;e] .z.M.log[`error][`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; - if[ok;.z.m.subscribedhandles:distinct .z.M.subscribedhandles,h]; + ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.logerr[`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; }; subscribe:{[handles] @@ -188,9 +190,9 @@ subscribe:{[handles] getheartbeats:{[proctype] / subscribe to publishers of the given process type(s) that are not yet subscribed - handles:(.z.M.servers[`getservers] proctype) except .z.M.subscribedhandles; + handles:(.z.m.serversgetservers proctype) except subscribedhandles; if[count handles; - .z.M.log[`info][`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; + .z.m.loginfo[`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; subscribe handles]; }; @@ -201,10 +203,10 @@ hbsubscriptions:{ closeconnection:{[h] / drop a closed handle from the tracked subscriptions - registered against .z.pc - .z.m.subscribedhandles:.z.M.subscribedhandles except h; + .z.m.subscribedhandles:subscribedhandles except h; }; gethb:{ / return the current heartbeat store for inspection - .z.M.hb - }; \ No newline at end of file + hb + }; diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 0478e8a9..38ea513a 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -54,3 +54,9 @@ true,0,0,q,5i in .m.di.0heartbeat.subscribedhandles,1,1,handle tracked after sub true,0,0,q,5i~.test.subh,1,1,subscribe dependency was invoked with the handle run,0,0,q,.m.di.0heartbeat.closeconnection[5i],1,1,simulate connection close true,0,0,q,not 5i in .m.di.0heartbeat.subscribedhandles,1,1,handle removed on close + +comment,,,,,,,init errors immediately when a required dependency is missing +fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when log dependency missing +fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when timer dependency missing +fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing +fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing From 4af597939c1940d769a304eca71ec155107a6f82 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Fri, 12 Jun 2026 14:07:29 +0100 Subject: [PATCH 05/16] Made changes inline with message about dependencies --- di/heartbeat/heartbeat.md | 14 ++++++++++---- di/heartbeat/heartbeat.q | 26 +++++++++++++++++--------- di/heartbeat/init.q | 2 +- di/heartbeat/test.csv | 3 ++- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index b4be2b5d..f623aeae 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -22,7 +22,7 @@ signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| | `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | always | logging | -| `timer` | `addjob` `deletejobs` `enablejobs` `disablejobs` `getactivejobs` `cp` | always | scheduling publish/check/subscribe and the current-time source (`cp`) | +| `timer` | `addjob` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | @@ -32,10 +32,15 @@ signatures can be supplied. required when `subenabled` is set (i.e. this process monitors other heartbeats); a pure publisher needs only `log`, `timer` and `pubsub`. -Only the functions the module actually calls are accessed (`timer`'s `addjob`/`cp`, +Only the functions the module actually calls are accessed (`timer`'s `addjob`, `handlers`' `register`), but supplying the full contracted dictionary keeps the dependency interchangeable with the real `di.*` modules. +The module keeps its **own** current-time function rather than taking it from the +timer dependency (so it doesn't rely on the timer exporting a clock getter). It +defaults to `.z.p`; override it with `setcp` for deterministic tests or simulation, +e.g. `heartbeat.setcp[{2025.01.01D00:00:00.000}]`. + ## Configuration `init[config;deps]` takes a configuration dictionary as its first argument. Any @@ -69,6 +74,7 @@ recognised key may be supplied; unset keys keep their defaults. | `subscribe[handles]` | subscribe to heartbeats on the given remote handle(s) | | `hbsubscriptions[]` | subscribe to all configured publishers (by `connections` process type) | | `gethb[]` | return the heartbeat store | +| `setcp[f]` | replace the current-time function (for tests / simulation) | ## Heartbeat store schema @@ -100,8 +106,8 @@ logdep: `info`warn`error!(logmod.info;logmod.warn;logmod.error) timer: use `di.timer timer.init[()!()] -timerdep: `addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!( - timer.addjob;timer.deletejobs;timer.enablejobs;timer.disablejobs;timer.getactivejobs;timer.cp) +// heartbeat only needs addjob from the timer - it keeps its own clock (see setcp) +timerdep: enlist[`addjob]!enlist timer.addjob // a pubsub dependency must provide publish[table;data] and subscribe[handle] pubsub: use `di.pubsub diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 3b8c4709..ea83bd97 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -21,6 +21,14 @@ subscribedhandles:`int$(); / heartbeat counter hbcounter:0; +/ current-time function - heartbeat owns its clock; override via setcp for testing / simulation +cp:{.z.p}; + +setcp:{[f] + / replace the current-time function (used by tests and simulation) + .z.m.cp:f; + }; + / configuration defaults - overridden by the config dictionary passed to init enabled:1b; / whether heartbeat publishing / checking is enabled subenabled:0b; / whether this process monitors (subscribes to) other heartbeats @@ -62,7 +70,6 @@ setdeps:{[deps] .z.m.logerr:logdict`error; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; - .z.m.timercp:timerdict`cp; pubsubdict:requiredep[deps;`pubsub]; .z.m.pubsubpublish:pubsubdict`publish; .z.m.pubsubsubscribe:pubsubdict`subscribe; @@ -104,11 +111,12 @@ init:{[config;deps] / initialise the module with configuration and injected dependencies - see heartbeat.md / config - dictionary of configuration overrides (see configkeys), or (::) for defaults / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: - / `log - `info`warn`error - required - / `timer - `addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp - required - / `pubsub - `publish`subscribe - required - / `servers - `getservers - required when subenabled - / `handlers - `register`remove`list - required when subenabled + / `log - `info`warn`error - required + / `timer - `addjob (full di.timer dict may be passed) - required + / `pubsub - `publish`subscribe - required + / `servers - `getservers - required when subenabled + / `handlers - `register`remove`list - required when subenabled + / note: the module keeps its own clock (cp, default .z.p) - override via setcp / example: / heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(logdep;timerdep;psdep)] setconfig config; @@ -121,7 +129,7 @@ init:{[config;deps] publishheartbeat:{ / publish a single heartbeat row over pub/sub and bump the counter if[not enabled;:()]; - .z.m.pubsubpublish[`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(.z.m.timercp[];proctype;procname;hbcounter;pid;host;port)]; + .z.m.pubsubpublish[`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(cp[];proctype;procname;hbcounter;pid;host;port)]; .z.m.hbcounter:hbcounter+1; }; @@ -134,7 +142,7 @@ storeheartbeat:{[batch] addprocs:{[proctypes;procnames] / seed the store with expected processes so a never-seen process is flagged / real heartbeats arriving later override these seeded rows - seed:2!([]sym:proctypes,();procname:procnames,();time:.z.m.timercp[];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); + seed:2!([]sym:proctypes,();procname:procnames,();time:cp[];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); .z.m.hb:seed,hb; }; @@ -166,7 +174,7 @@ checkheartbeat:{ / flag processes that have not heartbeated within the warning / error grace periods / status: 0 healthy, 1 warning, 2+ error / grace periods are computed as locals first - module functions do not resolve inside qsql - now:.z.m.timercp[]; + now:cp[]; t:0!hb; wp:warningperiod each t`sym; ep:errorperiod each t`sym; diff --git a/di/heartbeat/init.q b/di/heartbeat/init.q index 77ff3d0e..22294939 100644 --- a/di/heartbeat/init.q +++ b/di/heartbeat/init.q @@ -5,4 +5,4 @@ version:"0.1.0"; / public api - only the functions intended to be called externally are exported -export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;hbsubscriptions;gethb;version]) +export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;hbsubscriptions;gethb;setcp;version]) diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 38ea513a..351fb78e 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -3,11 +3,12 @@ comment,,,,,,,Setup - load module and inject mock dependencies before,0,0,q,heartbeat:use`di.heartbeat,1,1,load the heartbeat module before,0,0,q,logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,silent log mock before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the timer mock -before,0,0,q,timerdep:`addjob`deletejobs`enablejobs`disablejobs`getactivejobs`cp!({[id;func;params;period;mode;opts]};{[ids]};{[ids]};{[ids]};{[]([]id:`$())};{[].test.now}),1,1,timer mock with controllable cp +before,0,0,q,timerdep:enlist[`addjob]!enlist {[id;func;params;period;mode;opts]},1,1,timer mock - heartbeat only requires addjob before,0,0,q,psdep:`publish`subscribe!({[t;x] .test.pubt:t;.test.pubx:x;.test.pubcount:.test.pubcount+1};{[h] .test.subh:h}),1,1,pubsub mock capturing publish and subscribe before,0,0,q,.test.pubcount:0,1,1,initialise publish counter before,0,0,q,deps:`log`timer`pubsub!(logdep;timerdep;psdep),1,1,assemble dependency dictionary before,0,0,q,heartbeat.init[`proctype`procname!(`rdb;`rdb1);deps],1,1,initialise with identity config and mocks +before,0,0,q,heartbeat.setcp[{[].test.now}],1,1,point heartbeat clock at the controllable test time comment,,,,,,,publishheartbeat publishes a row and bumps the counter run,0,0,q,.test.pubcount:0,1,1,reset publish counter From 76099fa1afab1a83406ae70e3b9a478f8b68caec Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Fri, 12 Jun 2026 16:20:37 +0100 Subject: [PATCH 06/16] Made some changes.Use timer mode 2 to avoid catch-up storms, fix timer dep example found during manual test with a real di.timer --- di/heartbeat/heartbeat.md | 3 ++- di/heartbeat/heartbeat.q | 10 ++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index f623aeae..5708c6b6 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -107,7 +107,8 @@ logdep: `info`warn`error!(logmod.info;logmod.warn;logmod.error) timer: use `di.timer timer.init[()!()] // heartbeat only needs addjob from the timer - it keeps its own clock (see setcp) -timerdep: enlist[`addjob]!enlist timer.addjob +// note: di.timer's addjob is a namespace; addjob.custom has the [id;func;params;period;mode;opts] signature heartbeat calls +timerdep: enlist[`addjob]!enlist timer.addjob.custom // a pubsub dependency must provide publish[table;data] and subscribe[handle] pubsub: use `di.pubsub diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index ea83bd97..095eb470 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -93,12 +93,14 @@ tosecs:{[span] }; registertimers:{ - / schedule the periodic heartbeat jobs via the injected timer (mode 1 = fixed interval) + / schedule the periodic heartbeat jobs via the injected timer + / mode 2 = period after previous actual start - a heartbeat says "alive now", so + / missed beats must not be replayed as a catch-up storm (which mode 1 would do) if[enabled; - .z.m.timeraddjob[`hbpublish;publishheartbeat;();tosecs publishinterval;1;()!()]; - .z.m.timeraddjob[`hbcheck;checkheartbeat;();tosecs checkinterval;1;()!()]]; + .z.m.timeraddjob[`hbpublish;publishheartbeat;();tosecs publishinterval;2;()!()]; + .z.m.timeraddjob[`hbcheck;checkheartbeat;();tosecs checkinterval;2;()!()]]; if[subenabled; - .z.m.timeraddjob[`hbsubscribe;hbsubscriptions;();60;1;()!()]]; + .z.m.timeraddjob[`hbsubscribe;hbsubscriptions;();60;2;()!()]]; }; registerhandlers:{ From 249090020a9e292497a781f87a4fbe451666137a Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Mon, 22 Jun 2026 10:39:37 +0100 Subject: [PATCH 07/16] Added some integration tests using fake publisher. Fixed mode-2 as well --- di/heartbeat/heartbeat.q | 1 - di/heartbeat/test.csv | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 095eb470..30088320 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -28,7 +28,6 @@ setcp:{[f] / replace the current-time function (used by tests and simulation) .z.m.cp:f; }; - / configuration defaults - overridden by the config dictionary passed to init enabled:1b; / whether heartbeat publishing / checking is enabled subenabled:0b; / whether this process monitors (subscribes to) other heartbeats diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 351fb78e..4480f486 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -61,3 +61,32 @@ fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when log dependency missing fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when timer dependency missing fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing + +comment,,,,,,,real-process integration - a background publisher beats over real di.pubsub into this process +run,0,0,q,system"q -p 5050 -q &",1,,start a background heartbeat publisher +run,0,0,q,system"sleep 1",1,,wait for the publisher to come up +run,0,0,q,h:hopen`::5050,1,,connect to the publisher + +true,0,0,q,h>0,1,,connected to the publisher +run,0,0,q,h"hbmod:use`di.heartbeat",1,,load heartbeat on the publisher +run,0,0,q,h"pubsub:use`di.pubsub",1,,load pubsub on the publisher +run,0,0,q,h"heartbeat:.m.di.0heartbeat.heartbeat",1,,expose the heartbeat schema at root for pubsub +run,0,0,q,h"pubsub.init[]",1,,initialise the pubsub registry on the publisher +run,0,0,q,h"logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,silent log dep on the publisher +run,0,0,q,h"timerdep:enlist[`addjob]!enlist{[id;func;params;period;mode;opts]}",1,,no-op timer dep - publish is triggered manually for determinism +run,0,0,q,h"psdep:`publish`subscribe!(pubsub.publish;{[x]})",1,,inject the real pubsub publish into heartbeat +run,0,0,q,h"hbmod.init[`proctype`procname`port!(`rdb;`pub1;5050i);`log`timer`pubsub!(logdep;timerdep;psdep)]",1,,init the publisher as an rdb named pub1 +run,0,0,q,upd:{[t;x] if[`heartbeat~t; heartbeat.storeheartbeat x]},1,,route incoming beats into the local store +run,0,0,q,.m.di.0heartbeat.hb:0#.m.di.0heartbeat.hb,1,,clear the local store before subscribing +run,0,0,q,h(`pubsub.subscribe;`heartbeat;`),1,,subscribe this process to the publisher's heartbeat feed +run,0,0,q,h".m.di.0heartbeat.publishheartbeat[]",1,,publisher emits one heartbeat over real ipc +true,0,0,q,1=count heartbeat.gethb[],1,,beat received and stored via real pubsub +true,0,0,q,`pub1 in exec procname from heartbeat.gethb[],1,,stored beat carries the remote publisher identity +run,0,0,q,heartbeat.setcp[{.z.p}],1,,use real time - the just-received beat is fresh +run,0,0,q,heartbeat.checkheartbeat[],1,,check staleness while the beat is fresh +true,0,0,q,not first exec error from heartbeat.gethb[] where procname=`pub1,1,,fresh beat is not in error +run,0,0,q,heartbeat.setcp[{.z.p+0D01:00:00}],1,,skew the clock 1h ahead so the beat looks stale +run,0,0,q,heartbeat.checkheartbeat[],1,,re-check after the publisher has effectively gone silent +true,0,0,q,first exec error from heartbeat.gethb[] where procname=`pub1,1,,stale publisher escalated to error over real ipc +run,0,0,q,neg[h](exit;0);neg[h](::),1,,shut down the background publisher +run,0,0,q,hclose h,1,,close the handle From c95f20babd0aec4e4504de4736efc3219c12dcc3 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Mon, 22 Jun 2026 14:22:40 +0100 Subject: [PATCH 08/16] Init is now re-runnable --- di/heartbeat/heartbeat.md | 12 ++++++------ di/heartbeat/heartbeat.q | 5 ++++- di/heartbeat/test.csv | 11 +++++++++-- 3 files changed, 19 insertions(+), 9 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 5708c6b6..dd2a1ed6 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -22,7 +22,7 @@ signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| | `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | always | logging | -| `timer` | `addjob` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs | +| `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | @@ -32,9 +32,9 @@ signatures can be supplied. required when `subenabled` is set (i.e. this process monitors other heartbeats); a pure publisher needs only `log`, `timer` and `pubsub`. -Only the functions the module actually calls are accessed (`timer`'s `addjob`, -`handlers`' `register`), but supplying the full contracted dictionary keeps the -dependency interchangeable with the real `di.*` modules. +Only the functions the module actually calls are accessed (`timer`'s `addjob` and +`deletejobs`, `handlers`' `register`), but supplying the full contracted dictionary +keeps the dependency interchangeable with the real `di.*` modules. The module keeps its **own** current-time function rather than taking it from the timer dependency (so it doesn't rely on the timer exporting a clock getter). It @@ -106,9 +106,9 @@ logdep: `info`warn`error!(logmod.info;logmod.warn;logmod.error) timer: use `di.timer timer.init[()!()] -// heartbeat only needs addjob from the timer - it keeps its own clock (see setcp) +// heartbeat needs addjob and deletejobs from the timer - it keeps its own clock (see setcp) // note: di.timer's addjob is a namespace; addjob.custom has the [id;func;params;period;mode;opts] signature heartbeat calls -timerdep: enlist[`addjob]!enlist timer.addjob.custom +timerdep: `addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) // a pubsub dependency must provide publish[table;data] and subscribe[handle] pubsub: use `di.pubsub diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 30088320..76c30739 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -69,6 +69,7 @@ setdeps:{[deps] .z.m.logerr:logdict`error; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; + .z.m.timerdeletejobs:timerdict`deletejobs; pubsubdict:requiredep[deps;`pubsub]; .z.m.pubsubpublish:pubsubdict`publish; .z.m.pubsubsubscribe:pubsubdict`subscribe; @@ -95,6 +96,8 @@ registertimers:{ / schedule the periodic heartbeat jobs via the injected timer / mode 2 = period after previous actual start - a heartbeat says "alive now", so / missed beats must not be replayed as a catch-up storm (which mode 1 would do) + / clear any previously-registered jobs first so init is safe to call again + .z.m.timerdeletejobs[`hbpublish`hbcheck`hbsubscribe]; if[enabled; .z.m.timeraddjob[`hbpublish;publishheartbeat;();tosecs publishinterval;2;()!()]; .z.m.timeraddjob[`hbcheck;checkheartbeat;();tosecs checkinterval;2;()!()]]; @@ -113,7 +116,7 @@ init:{[config;deps] / config - dictionary of configuration overrides (see configkeys), or (::) for defaults / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: / `log - `info`warn`error - required - / `timer - `addjob (full di.timer dict may be passed) - required + / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required / `pubsub - `publish`subscribe - required / `servers - `getservers - required when subenabled / `handlers - `register`remove`list - required when subenabled diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 4480f486..4c972acb 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -3,7 +3,7 @@ comment,,,,,,,Setup - load module and inject mock dependencies before,0,0,q,heartbeat:use`di.heartbeat,1,1,load the heartbeat module before,0,0,q,logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,silent log mock before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the timer mock -before,0,0,q,timerdep:enlist[`addjob]!enlist {[id;func;params;period;mode;opts]},1,1,timer mock - heartbeat only requires addjob +before,0,0,q,timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]}),1,1,timer mock - heartbeat requires addjob and deletejobs before,0,0,q,psdep:`publish`subscribe!({[t;x] .test.pubt:t;.test.pubx:x;.test.pubcount:.test.pubcount+1};{[h] .test.subh:h}),1,1,pubsub mock capturing publish and subscribe before,0,0,q,.test.pubcount:0,1,1,initialise publish counter before,0,0,q,deps:`log`timer`pubsub!(logdep;timerdep;psdep),1,1,assemble dependency dictionary @@ -62,6 +62,13 @@ fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when time fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing +comment,,,,,,,init is idempotent - re-running clears jobs first instead of colliding on ids +run,0,0,q,.test.jobids:(`$())!`boolean$(),1,1,track registered job ids +run,0,0,q,timerdep2:`addjob`deletejobs!({[id;func;params;period;mode;opts] if[id in key .test.jobids;'"duplicate job id"]; .test.jobids[id]:1b};{[ids] .test.jobids:(key[.test.jobids] except ids)#.test.jobids}),1,1,stateful timer mock that rejects duplicate job ids like di.timer +run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(logdep;timerdep2;psdep)],1,1,first init registers the jobs +run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(logdep;timerdep2;psdep)],1,1,second init must not collide - clears then re-adds +true,0,0,q,`hbcheck`hbpublish~asc key .test.jobids,1,1,exactly the publish and check jobs are registered after re-init + comment,,,,,,,real-process integration - a background publisher beats over real di.pubsub into this process run,0,0,q,system"q -p 5050 -q &",1,,start a background heartbeat publisher run,0,0,q,system"sleep 1",1,,wait for the publisher to come up @@ -73,7 +80,7 @@ run,0,0,q,h"pubsub:use`di.pubsub",1,,load pubsub on the publisher run,0,0,q,h"heartbeat:.m.di.0heartbeat.heartbeat",1,,expose the heartbeat schema at root for pubsub run,0,0,q,h"pubsub.init[]",1,,initialise the pubsub registry on the publisher run,0,0,q,h"logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,silent log dep on the publisher -run,0,0,q,h"timerdep:enlist[`addjob]!enlist{[id;func;params;period;mode;opts]}",1,,no-op timer dep - publish is triggered manually for determinism +run,0,0,q,h"timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]})",1,,no-op timer dep - publish is triggered manually for determinism run,0,0,q,h"psdep:`publish`subscribe!(pubsub.publish;{[x]})",1,,inject the real pubsub publish into heartbeat run,0,0,q,h"hbmod.init[`proctype`procname`port!(`rdb;`pub1;5050i);`log`timer`pubsub!(logdep;timerdep;psdep)]",1,,init the publisher as an rdb named pub1 run,0,0,q,upd:{[t;x] if[`heartbeat~t; heartbeat.storeheartbeat x]},1,,route incoming beats into the local store From 42eba96a59a5aaf5a217e847c460661f5e3313a2 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 23 Jun 2026 09:50:36 +0100 Subject: [PATCH 09/16] Added kx.logging functionality --- di/heartbeat/heartbeat.md | 29 +++++++++++++++-------------- di/heartbeat/heartbeat.q | 28 +++++++++++++++++----------- di/heartbeat/test.csv | 20 +++++++++++++++----- 3 files changed, 47 insertions(+), 30 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index dd2a1ed6..20cdb848 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -14,23 +14,26 @@ It covers both sides: ## Dependencies All runtime dependencies are **injected** via `init` as dictionaries of functions -(`` `dependency!(dict of functions) ``). They are **required** - `init` errors -immediately with a clear message if a required dependency is missing. There is no -hard dependency on any other module: any module exporting the contracted function -signatures can be supplied. +(`` `dependency!(dict of functions) ``). The functional dependencies are **required** - +`init` errors immediately with a clear message if one is missing. `log` is **optional** +and falls back to a no-op logger. There is no hard dependency on any other module: +any module exporting the contracted function signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| -| `log` | `info` `warn` `error` (each `{[ctx;msg]}`) | always | logging | +| `log` | `info` `warn` `error` (each unary `{[msg]}` - a `kx.log` logger) | optional (no-op fallback) | logging | | `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | -`log`, `timer` and `handlers` follow the standard kdb-x core dependency contracts; +`log` is a `kx.log` logger (its `info`/`warn`/`error` are **unary** `{[msg]}` - the +context tag is folded into the message, e.g. `"heartbeat: ..."`). It is optional: if +absent or missing any of `info`/`warn`/`error`, the module logs to a silent no-op. +`timer` and `handlers` otherwise follow the standard kdb-x core dependency contracts; `pubsub` and `servers` are heartbeat-specific. `servers` and `handlers` are only required when `subenabled` is set (i.e. this process monitors other heartbeats); -a pure publisher needs only `log`, `timer` and `pubsub`. +a pure publisher needs only `timer` and `pubsub`. Only the functions the module actually calls are accessed (`timer`'s `addjob` and `deletejobs`, `handlers`' `register`), but supplying the full contracted dictionary @@ -98,11 +101,9 @@ recognised key may be supplied; unset keys keep their defaults. // load the module heartbeat: use `di.heartbeat -// build dependency dictionaries (here from di.log and di.timer) -// note: bound as logmod, not log, since log is a reserved q word -logmod: use `di.log -logmod.init[()!()] -logdep: `info`warn`error!(logmod.info;logmod.warn;logmod.error) +// log is an optional kx.log logger instance (its info/warn/error are unary) +logger: use `kx.log +log: logger.createLog[] timer: use `di.timer timer.init[()!()] @@ -114,8 +115,8 @@ timerdep: `addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) pubsub: use `di.pubsub psdep: `publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) -// initialise as a publishing RDB (log, timer and pubsub are all required) -heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(logdep;timerdep;psdep)] +// initialise as a publishing RDB - timer and pubsub required, log optional +heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(log;timerdep;psdep)] // publish a heartbeat immediately (normally the timer does this) heartbeat.publishheartbeat[] diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 76c30739..19db6d35 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -4,8 +4,8 @@ / blocked - even when the underlying connection is still valid / the module handles both publishing heartbeats and, on the monitoring side, / storing received heartbeats and raising warnings / errors when they stop -/ runtime dependencies are injected via init and are required - the module errors -/ immediately if a required dependency is missing - see heartbeat.md +/ runtime dependencies are injected via init - timer and pubsub are required (init errors +/ if missing); log is an optional kx.log-style logger with a no-op fallback - see heartbeat.md / module-local state convention: read config bare, mutate via .z.m, and access / injected dependencies via .z.m at every call site @@ -52,6 +52,10 @@ configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolera warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; +/ no-op fallback logger - used when no (or an incomplete) log dependency is injected +/ functions are unary {[msg]} to match the kx.log logger contract +defaultlog:`info`warn`error!({[m]};{[m]};{[m]}); + requiredep:{[deps;name] / extract a required dependency dictionary, erroring immediately if absent or null d:$[99h=type deps;$[(name in key deps) and not (::)~deps name;deps name;()!()];()!()]; @@ -61,9 +65,11 @@ requiredep:{[deps;name] }; setdeps:{[deps] - / extract and store the required dependencies, flattened for access via .z.m - / log, timer and pubsub are always required; servers and handlers only when monitoring - logdict:requiredep[deps;`log]; + / extract and store injected dependencies, flattened for access via .z.m + / log is optional (a kx.log logger; no-op fallback if absent or incomplete); + / timer and pubsub are always required; servers and handlers only when monitoring + lograw:$[99h=type deps;$[`log in key deps;deps`log;(::)];(::)]; + logdict:$[99h=type lograw;$[all `info`warn`error in key lograw;lograw;defaultlog];defaultlog]; .z.m.loginfo:logdict`info; .z.m.logwarn:logdict`warn; .z.m.logerr:logdict`error; @@ -115,7 +121,7 @@ init:{[config;deps] / initialise the module with configuration and injected dependencies - see heartbeat.md / config - dictionary of configuration overrides (see configkeys), or (::) for defaults / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: - / `log - `info`warn`error - required + / `log - a kx.log logger (unary info/warn/error) - optional, no-op fallback / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required / `pubsub - `publish`subscribe - required / `servers - `getservers - required when subenabled @@ -127,7 +133,7 @@ init:{[config;deps] setdeps deps; registertimers[]; registerhandlers[]; - .z.m.loginfo[`heartbeat;"di.heartbeat initialised"]; + .z.m.loginfo["heartbeat: di.heartbeat initialised"]; }; publishheartbeat:{ @@ -152,12 +158,12 @@ addprocs:{[proctypes;procnames] logwarnproc:{[r] / log a single process moving into warning state - .z.m.logwarn[`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.logwarn["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; logerrproc:{[r] / log a single process moving into error state - .z.m.logerr[`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.logerr["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; warn:{[procs] @@ -191,7 +197,7 @@ checkheartbeat:{ subscribeone:{[h] / subscribe to a single remote heartbeat publisher, logging and skipping on failure - ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.logerr[`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.logerr["heartbeat: failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; }; @@ -204,7 +210,7 @@ getheartbeats:{[proctype] / subscribe to publishers of the given process type(s) that are not yet subscribed handles:(.z.m.serversgetservers proctype) except subscribedhandles; if[count handles; - .z.m.loginfo[`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; + .z.m.loginfo["heartbeat: subscribing to new heartbeat handle(s) ",", " sv string handles]; subscribe handles]; }; diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 4c972acb..64796471 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -1,7 +1,7 @@ action,ms,bytes,lang,code,repeat,minver,comment comment,,,,,,,Setup - load module and inject mock dependencies before,0,0,q,heartbeat:use`di.heartbeat,1,1,load the heartbeat module -before,0,0,q,logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,silent log mock +before,0,0,q,logdep:`info`warn`error!({[m]};{[m]};{[m]}),1,1,silent log mock - unary {[m]} matches the kx.log logger contract before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the timer mock before,0,0,q,timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]}),1,1,timer mock - heartbeat requires addjob and deletejobs before,0,0,q,psdep:`publish`subscribe!({[t;x] .test.pubt:t;.test.pubx:x;.test.pubcount:.test.pubcount+1};{[h] .test.subh:h}),1,1,pubsub mock capturing publish and subscribe @@ -56,11 +56,21 @@ true,0,0,q,5i~.test.subh,1,1,subscribe dependency was invoked with the handle run,0,0,q,.m.di.0heartbeat.closeconnection[5i],1,1,simulate connection close true,0,0,q,not 5i in .m.di.0heartbeat.subscribedhandles,1,1,handle removed on close -comment,,,,,,,init errors immediately when a required dependency is missing -fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when log dependency missing -fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when timer dependency missing +comment,,,,,,,log is optional - omitting it falls back to a no-op logger (init still logs at the end) +run,0,0,q,heartbeat.init[`proctype`procname!(`rdb;`rdb1);`timer`pubsub!(timerdep;psdep)],1,1,init without a log dep falls back to no-op logging without error +run,0,0,q,heartbeat.checkheartbeat[],1,1,a logging code path runs safely under the no-op logger + +comment,,,,,,,timer and pubsub are required - init errors when they are missing +fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when timer dependency missing (no deps supplied) fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing +run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);deps],1,1,re-init with full mock deps and subenabled 0b to restore clean module state (prior fail test left subenabled 1b) + +comment,,,,,,,an injected logger is actually invoked (guards against silent no-op fallback) +run,0,0,q,.test.loginfo:"",1,1,reset the capture +run,0,0,q,caplog:`info`warn`error!({[m] .test.loginfo:m};{[m] .test.logwarn:m};{[m] .test.logerr:m}),1,1,capturing unary logger - records the last message at each level +run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(caplog;timerdep;psdep)],1,1,init with the capturing logger +true,0,0,q,.test.loginfo~"heartbeat: di.heartbeat initialised",1,1,init message routed through the injected logger not the no-op fallback comment,,,,,,,init is idempotent - re-running clears jobs first instead of colliding on ids run,0,0,q,.test.jobids:(`$())!`boolean$(),1,1,track registered job ids @@ -79,7 +89,7 @@ run,0,0,q,h"hbmod:use`di.heartbeat",1,,load heartbeat on the publisher run,0,0,q,h"pubsub:use`di.pubsub",1,,load pubsub on the publisher run,0,0,q,h"heartbeat:.m.di.0heartbeat.heartbeat",1,,expose the heartbeat schema at root for pubsub run,0,0,q,h"pubsub.init[]",1,,initialise the pubsub registry on the publisher -run,0,0,q,h"logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,silent log dep on the publisher +run,0,0,q,h"logdep:`info`warn`error!({[m]};{[m]};{[m]})",1,,silent unary log dep on the publisher (kx.log contract) run,0,0,q,h"timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]})",1,,no-op timer dep - publish is triggered manually for determinism run,0,0,q,h"psdep:`publish`subscribe!(pubsub.publish;{[x]})",1,,inject the real pubsub publish into heartbeat run,0,0,q,h"hbmod.init[`proctype`procname`port!(`rdb;`pub1;5050i);`log`timer`pubsub!(logdep;timerdep;psdep)]",1,,init the publisher as an rdb named pub1 From 8cc3a5e6fb155a193382ab8dc1b7a2cad5818b64 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 23 Jun 2026 10:32:52 +0100 Subject: [PATCH 10/16] Added mandatory /info/warn/error in logs but allowed user to add more of their own log messages --- di/heartbeat/heartbeat.md | 11 +++++++---- di/heartbeat/heartbeat.q | 21 ++++++++++----------- di/heartbeat/init.q | 2 +- di/heartbeat/test.csv | 6 ++++++ 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 20cdb848..4175f993 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -21,15 +21,18 @@ any module exporting the contracted function signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| -| `log` | `info` `warn` `error` (each unary `{[msg]}` - a `kx.log` logger) | optional (no-op fallback) | logging | +| `log` | a `kx.log` logger - **must** provide unary `info` `warn` `error` (`{[msg]}`); extra levels allowed | optional (no-op fallback) | logging | | `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | -`log` is a `kx.log` logger (its `info`/`warn`/`error` are **unary** `{[msg]}` - the -context tag is folded into the message, e.g. `"heartbeat: ..."`). It is optional: if -absent or missing any of `info`/`warn`/`error`, the module logs to a silent no-op. +`log` is a `kx.log` logger. Only `info`/`warn`/`error` are **mandated** (each unary +`{[msg]}` - the context tag is folded into the message, e.g. `"heartbeat: ..."`); the +**whole logger is retained**, so any extra levels or controls it provides (`debug`, +`fatal`, custom levels, `kx.log` format/level setters) remain available and are not +stripped. It is optional: if absent, or missing any of `info`/`warn`/`error`, the +module logs to a silent no-op. `timer` and `handlers` otherwise follow the standard kdb-x core dependency contracts; `pubsub` and `servers` are heartbeat-specific. `servers` and `handlers` are only required when `subenabled` is set (i.e. this process monitors other heartbeats); diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 19db6d35..f66a8a3e 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -66,13 +66,12 @@ requiredep:{[deps;name] setdeps:{[deps] / extract and store injected dependencies, flattened for access via .z.m - / log is optional (a kx.log logger; no-op fallback if absent or incomplete); + / log is optional (a kx.log logger; no-op fallback if absent or missing a mandatory level); + / only info/warn/error are mandated - the whole logger is kept, so any extra levels the + / user provides (debug/fatal/custom, kx.log format controls) remain available / timer and pubsub are always required; servers and handlers only when monitoring lograw:$[99h=type deps;$[`log in key deps;deps`log;(::)];(::)]; - logdict:$[99h=type lograw;$[all `info`warn`error in key lograw;lograw;defaultlog];defaultlog]; - .z.m.loginfo:logdict`info; - .z.m.logwarn:logdict`warn; - .z.m.logerr:logdict`error; + .z.m.log:$[99h=type lograw;$[all `info`warn`error in key lograw;lograw;defaultlog];defaultlog]; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; .z.m.timerdeletejobs:timerdict`deletejobs; @@ -121,7 +120,7 @@ init:{[config;deps] / initialise the module with configuration and injected dependencies - see heartbeat.md / config - dictionary of configuration overrides (see configkeys), or (::) for defaults / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: - / `log - a kx.log logger (unary info/warn/error) - optional, no-op fallback + / `log - a kx.log logger - optional; must provide unary info/warn/error, may provide more / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required / `pubsub - `publish`subscribe - required / `servers - `getservers - required when subenabled @@ -133,7 +132,7 @@ init:{[config;deps] setdeps deps; registertimers[]; registerhandlers[]; - .z.m.loginfo["heartbeat: di.heartbeat initialised"]; + .z.m.log[`info]["heartbeat: di.heartbeat initialised"]; }; publishheartbeat:{ @@ -158,12 +157,12 @@ addprocs:{[proctypes;procnames] logwarnproc:{[r] / log a single process moving into warning state - .z.m.logwarn["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.log[`warn]["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; logerrproc:{[r] / log a single process moving into error state - .z.m.logerr["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.log[`error]["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; warn:{[procs] @@ -197,7 +196,7 @@ checkheartbeat:{ subscribeone:{[h] / subscribe to a single remote heartbeat publisher, logging and skipping on failure - ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.logerr["heartbeat: failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.log[`error]["heartbeat: failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; }; @@ -210,7 +209,7 @@ getheartbeats:{[proctype] / subscribe to publishers of the given process type(s) that are not yet subscribed handles:(.z.m.serversgetservers proctype) except subscribedhandles; if[count handles; - .z.m.loginfo["heartbeat: subscribing to new heartbeat handle(s) ",", " sv string handles]; + .z.m.log[`info]["heartbeat: subscribing to new heartbeat handle(s) ",", " sv string handles]; subscribe handles]; }; diff --git a/di/heartbeat/init.q b/di/heartbeat/init.q index 22294939..e4d90ee8 100644 --- a/di/heartbeat/init.q +++ b/di/heartbeat/init.q @@ -5,4 +5,4 @@ version:"0.1.0"; / public api - only the functions intended to be called externally are exported -export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;hbsubscriptions;gethb;setcp;version]) +export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;gethb;setcp;version]) diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 64796471..7f305120 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -72,6 +72,12 @@ run,0,0,q,caplog:`info`warn`error!({[m] .test.loginfo:m};{[m] .test.logwarn:m};{ run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(caplog;timerdep;psdep)],1,1,init with the capturing logger true,0,0,q,.test.loginfo~"heartbeat: di.heartbeat initialised",1,1,init message routed through the injected logger not the no-op fallback +comment,,,,,,,only info/warn/error are mandated - a richer logger is accepted and kept whole +run,0,0,q,caplog2:`info`warn`error`debug!({[m]};{[m]};{[m]};{[m]}),1,1,logger providing an extra debug level beyond the mandatory three +run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(caplog2;timerdep;psdep)],1,1,init with the richer logger +true,0,0,q,`debug in key .m.di.0heartbeat.log,1,1,extra level retained - the whole logger is kept not stripped to three functions +true,0,0,q,all `info`warn`error in key .m.di.0heartbeat.log,1,1,mandatory levels present + comment,,,,,,,init is idempotent - re-running clears jobs first instead of colliding on ids run,0,0,q,.test.jobids:(`$())!`boolean$(),1,1,track registered job ids run,0,0,q,timerdep2:`addjob`deletejobs!({[id;func;params;period;mode;opts] if[id in key .test.jobids;'"duplicate job id"]; .test.jobids[id]:1b};{[ids] .test.jobids:(key[.test.jobids] except ids)#.test.jobids}),1,1,stateful timer mock that rejects duplicate job ids like di.timer From 78c2eb781feda4744b17b294ca11a9a626894d23 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 23 Jun 2026 10:53:50 +0100 Subject: [PATCH 11/16] Modifications to follow code formatting guidelines --- di/heartbeat/heartbeat.q | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index f66a8a3e..bea7a895 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -28,25 +28,27 @@ setcp:{[f] / replace the current-time function (used by tests and simulation) .z.m.cp:f; }; + / configuration defaults - overridden by the config dictionary passed to init -enabled:1b; / whether heartbeat publishing / checking is enabled -subenabled:0b; / whether this process monitors (subscribes to) other heartbeats -debug:1b; / whether to log warning / error transitions +enabled:1b; / whether heartbeat publishing / checking is enabled +subenabled:0b; / whether this process monitors (subscribes to) other heartbeats +debug:1b; / whether to log warning / error transitions publishinterval:0D00:00:30; / how often heartbeats are published checkinterval:0D00:00:10; / how often received heartbeats are checked -warningtolerance:1.5; / warning after warningtolerance*publishinterval without a beat -errortolerance:2f; / error after errortolerance*publishinterval without a beat -proctype:`unknown; / this process's type (published as sym) -procname:.z.h; / this process's name -pid:.z.i; / this process's pid -host:.z.h; / this process's host -port:`int$system"p"; / this process's port -connections:`$(); / process types this monitor should subscribe to -onwarning:{[procs]}; / callback fired with the rows entering warning state -onerror:{[procs]}; / callback fired with the rows entering error state +warningtolerance:1.5; / warning after warningtolerance*publishinterval without a beat +errortolerance:2f; / error after errortolerance*publishinterval without a beat +proctype:`unknown; / this process's type (published as sym) +procname:.z.h; / this process's name +pid:.z.i; / this process's pid +host:.z.h; / this process's host +port:`int$system"p"; / this process's port +connections:`$(); / process types this monitor should subscribe to +onwarning:{[procs]}; / callback fired with the rows entering warning state +onerror:{[procs]}; / callback fired with the rows entering error state / recognised configuration keys - anything else passed via config is ignored -configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolerance`errortolerance`proctype`procname`pid`host`port`connections`onwarning`onerror; +configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolerance`errortolerance, + `proctype`procname`pid`host`port`connections`onwarning`onerror; / warning / error grace periods - vary by process type if required warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; @@ -65,7 +67,7 @@ requiredep:{[deps;name] }; setdeps:{[deps] - / extract and store injected dependencies, flattened for access via .z.m + / extract and store injected dependencies under .z.m (log kept whole; the rest as the functions used) / log is optional (a kx.log logger; no-op fallback if absent or missing a mandatory level); / only info/warn/error are mandated - the whole logger is kept, so any extra levels the / user provides (debug/fatal/custom, kx.log format controls) remain available From 3714cabc867652d6681c2b685ae61fc6dcc078a2 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 23 Jun 2026 11:07:06 +0100 Subject: [PATCH 12/16] Split setdeps conditional into helper function to avoid if statements. Following code guidelines --- di/heartbeat/heartbeat.q | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index bea7a895..156a2785 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -80,11 +80,19 @@ setdeps:{[deps] pubsubdict:requiredep[deps;`pubsub]; .z.m.pubsubpublish:pubsubdict`publish; .z.m.pubsubsubscribe:pubsubdict`subscribe; - if[subenabled; - serversdict:requiredep[deps;`servers]; - .z.m.serversgetservers:serversdict`getservers; - handlersdict:requiredep[deps;`handlers]; - .z.m.handlersregister:handlersdict`register]; + / monitor-only deps are wired via a separate function so the conditional stays a + / single statement - the style guide says avoid block statements within conditionals + if[subenabled;setmonitordeps deps]; + }; + +setmonitordeps:{[deps] + / wire the monitor-only dependencies, required only when subenabled (this process + / monitors others' heartbeats); split out of setdeps to keep that conditional a + / single statement per the coding standards + serversdict:requiredep[deps;`servers]; + .z.m.serversgetservers:serversdict`getservers; + handlersdict:requiredep[deps;`handlers]; + .z.m.handlersregister:handlersdict`register; }; setconfig:{[config] From 6fd2a650e88c9f7d0e7a18897a0ff610c74c2310 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 23 Jun 2026 14:08:51 +0100 Subject: [PATCH 13/16] Changed to follow modularisation.md --- di/heartbeat/heartbeat.md | 32 ++++++++++++++++---------------- di/heartbeat/heartbeat.q | 26 +++++++++++++------------- di/heartbeat/test.csv | 10 ++++++---- 3 files changed, 35 insertions(+), 33 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 4175f993..2b17f9bb 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -14,29 +14,28 @@ It covers both sides: ## Dependencies All runtime dependencies are **injected** via `init` as dictionaries of functions -(`` `dependency!(dict of functions) ``). The functional dependencies are **required** - -`init` errors immediately with a clear message if one is missing. `log` is **optional** -and falls back to a no-op logger. There is no hard dependency on any other module: -any module exporting the contracted function signatures can be supplied. +(`` `dependency!(dict of functions) ``) and are **required** - `init` errors immediately +with a clear message if `deps` is not a dictionary or a required dependency is missing or +malformed. There is no hard dependency on any other module: any module exporting the +contracted function signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| -| `log` | a `kx.log` logger - **must** provide unary `info` `warn` `error` (`{[msg]}`); extra levels allowed | optional (no-op fallback) | logging | +| `log` | a `kx.log` logger - **must** provide unary `info` `warn` `error` (`{[msg]}`); extra levels allowed | always | logging | | `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | -`log` is a `kx.log` logger. Only `info`/`warn`/`error` are **mandated** (each unary -`{[msg]}` - the context tag is folded into the message, e.g. `"heartbeat: ..."`); the -**whole logger is retained**, so any extra levels or controls it provides (`debug`, -`fatal`, custom levels, `kx.log` format/level setters) remain available and are not -stripped. It is optional: if absent, or missing any of `info`/`warn`/`error`, the -module logs to a silent no-op. +`log` is a `kx.log` logger and is **required**. Only `info`/`warn`/`error` are mandated +(each unary `{[msg]}` - the context tag is folded into the message, e.g. `"heartbeat: ..."`); +the **whole logger is retained**, so any extra levels or controls it provides (`debug`, +`fatal`, custom levels, `kx.log` format/level setters) remain available and are not stripped. +`init` errors immediately if `log` is absent, not a dict, or missing any of `info`/`warn`/`error`. `timer` and `handlers` otherwise follow the standard kdb-x core dependency contracts; `pubsub` and `servers` are heartbeat-specific. `servers` and `handlers` are only required when `subenabled` is set (i.e. this process monitors other heartbeats); -a pure publisher needs only `timer` and `pubsub`. +a pure publisher needs `log`, `timer` and `pubsub`. Only the functions the module actually calls are accessed (`timer`'s `addjob` and `deletejobs`, `handlers`' `register`), but supplying the full contracted dictionary @@ -104,9 +103,10 @@ recognised key may be supplied; unset keys keep their defaults. // load the module heartbeat: use `di.heartbeat -// log is an optional kx.log logger instance (its info/warn/error are unary) +// log is a required kx.log logger instance (its info/warn/error are unary) +// bound as kxlog, not log, since log is a reserved q word logger: use `kx.log -log: logger.createLog[] +kxlog: logger.createLog[] timer: use `di.timer timer.init[()!()] @@ -118,8 +118,8 @@ timerdep: `addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) pubsub: use `di.pubsub psdep: `publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) -// initialise as a publishing RDB - timer and pubsub required, log optional -heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(log;timerdep;psdep)] +// initialise as a publishing RDB - log, timer and pubsub all required +heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(kxlog;timerdep;psdep)] // publish a heartbeat immediately (normally the timer does this) heartbeat.publishheartbeat[] diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 156a2785..eebf1d28 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -4,8 +4,8 @@ / blocked - even when the underlying connection is still valid / the module handles both publishing heartbeats and, on the monitoring side, / storing received heartbeats and raising warnings / errors when they stop -/ runtime dependencies are injected via init - timer and pubsub are required (init errors -/ if missing); log is an optional kx.log-style logger with a no-op fallback - see heartbeat.md +/ runtime dependencies are injected via init and are required - log, timer and pubsub always +/ (servers and handlers when monitoring); init errors immediately if one is missing - see heartbeat.md / module-local state convention: read config bare, mutate via .z.m, and access / injected dependencies via .z.m at every call site @@ -54,10 +54,6 @@ configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolera warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; -/ no-op fallback logger - used when no (or an incomplete) log dependency is injected -/ functions are unary {[msg]} to match the kx.log logger contract -defaultlog:`info`warn`error!({[m]};{[m]};{[m]}); - requiredep:{[deps;name] / extract a required dependency dictionary, erroring immediately if absent or null d:$[99h=type deps;$[(name in key deps) and not (::)~deps name;deps name;()!()];()!()]; @@ -68,12 +64,16 @@ requiredep:{[deps;name] setdeps:{[deps] / extract and store injected dependencies under .z.m (log kept whole; the rest as the functions used) - / log is optional (a kx.log logger; no-op fallback if absent or missing a mandatory level); - / only info/warn/error are mandated - the whole logger is kept, so any extra levels the - / user provides (debug/fatal/custom, kx.log format controls) remain available - / timer and pubsub are always required; servers and handlers only when monitoring - lograw:$[99h=type deps;$[`log in key deps;deps`log;(::)];(::)]; - .z.m.log:$[99h=type lograw;$[all `info`warn`error in key lograw;lograw;defaultlog];defaultlog]; + / log, timer and pubsub are required; servers and handlers only when monitoring + / init errors immediately if deps is not a dictionary or a required dependency is missing/malformed + / nested if guards (not and) - and evaluates both sides eagerly and key would throw on a non-dict + if[99h<>type deps;'"di.heartbeat: deps must be a dictionary of injected dependencies - see heartbeat.md"]; + / log - required: a dict providing info/warn/error (e.g. a kx.log logger); only those three are + / mandated but the whole logger is kept, so extra levels (debug/fatal/custom) remain available + if[not `log in key deps;'"di.heartbeat: log dependency is required; pass a logger keyed on `log - see kx.log"]; + if[99h<>type deps`log;'"di.heartbeat: log value must be a dict of info/warn/error functions"]; + if[not all `info`warn`error in key deps`log;'"di.heartbeat: log dict must have `info`warn`error keys; got: ",", " sv string key deps`log]; + .z.m.log:deps`log; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; .z.m.timerdeletejobs:timerdict`deletejobs; @@ -130,7 +130,7 @@ init:{[config;deps] / initialise the module with configuration and injected dependencies - see heartbeat.md / config - dictionary of configuration overrides (see configkeys), or (::) for defaults / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: - / `log - a kx.log logger - optional; must provide unary info/warn/error, may provide more + / `log - a kx.log logger - required; must provide unary info/warn/error, may provide more / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required / `pubsub - `publish`subscribe - required / `servers - `getservers - required when subenabled diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index 7f305120..f3150ab9 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -56,12 +56,14 @@ true,0,0,q,5i~.test.subh,1,1,subscribe dependency was invoked with the handle run,0,0,q,.m.di.0heartbeat.closeconnection[5i],1,1,simulate connection close true,0,0,q,not 5i in .m.di.0heartbeat.subscribedhandles,1,1,handle removed on close -comment,,,,,,,log is optional - omitting it falls back to a no-op logger (init still logs at the end) -run,0,0,q,heartbeat.init[`proctype`procname!(`rdb;`rdb1);`timer`pubsub!(timerdep;psdep)],1,1,init without a log dep falls back to no-op logging without error -run,0,0,q,heartbeat.checkheartbeat[],1,1,a logging code path runs safely under the no-op logger +comment,,,,,,,log is required - init errors clearly when deps or log is missing or malformed +fail,0,0,q,heartbeat.init[()!();(::)],1,1,errors when deps is not a dictionary +fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when log dependency missing +fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist 42],1,1,errors when log value is not a dictionary +fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist `info`warn!(logdep`info;logdep`warn)],1,1,errors when log dict is missing the error key comment,,,,,,,timer and pubsub are required - init errors when they are missing -fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when timer dependency missing (no deps supplied) +fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when timer dependency missing fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);deps],1,1,re-init with full mock deps and subenabled 0b to restore clean module state (prior fail test left subenabled 1b) From 8ccee9b5e43f85d5c691ebe50a28f335f954114e Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Thu, 25 Jun 2026 14:51:55 +0100 Subject: [PATCH 14/16] Restructure according to Group PR Review --- di/heartbeat/heartbeat.md | 48 ++++++++++++++++++-------------- di/heartbeat/heartbeat.q | 57 +++++++++++++++++++++++--------------- di/heartbeat/init.q | 6 ++-- di/heartbeat/test.csv | 58 +++++++++++++++++++++------------------ 4 files changed, 97 insertions(+), 72 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 2b17f9bb..6414d079 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -13,29 +13,33 @@ It covers both sides: ## Dependencies -All runtime dependencies are **injected** via `init` as dictionaries of functions -(`` `dependency!(dict of functions) ``) and are **required** - `init` errors immediately -with a clear message if `deps` is not a dictionary or a required dependency is missing or -malformed. There is no hard dependency on any other module: any module exporting the -contracted function signatures can be supplied. +Config and dependencies are passed together in a **single dictionary** to `init` (see +Configuration). The dependencies below are **required** - `init` errors immediately with a +clear message if `deps` is not a dictionary or a required dependency is missing or malformed. +There is no hard dependency on any other module: any module exporting the contracted function +signatures can be supplied. | Dependency | Keys | Required | Purpose | |------------|------|----------|---------| -| `log` | a `kx.log` logger - **must** provide unary `info` `warn` `error` (`{[msg]}`); extra levels allowed | always | logging | +| `log` | a logger providing `info` `warn` `error`; a `kx.log` instance is accepted directly | always | logging | | `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | | `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | | `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | | `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | -`log` is a `kx.log` logger and is **required**. Only `info`/`warn`/`error` are mandated -(each unary `{[msg]}` - the context tag is folded into the message, e.g. `"heartbeat: ..."`); -the **whole logger is retained**, so any extra levels or controls it provides (`debug`, -`fatal`, custom levels, `kx.log` format/level setters) remain available and are not stripped. -`init` errors immediately if `log` is absent, not a dict, or missing any of `info`/`warn`/`error`. -`timer` and `handlers` otherwise follow the standard kdb-x core dependency contracts; -`pubsub` and `servers` are heartbeat-specific. `servers` and `handlers` are only -required when `subenabled` is set (i.e. this process monitors other heartbeats); -a pure publisher needs `log`, `timer` and `pubsub`. +**Logging contract.** Internally the module calls the logger as **binary** `.z.m.log[\`info][\`heartbeat;"msg"]` +(`{[c;m]}` - context symbol + message). Only `info`/`warn`/`error` are mandated (heartbeat uses all +three). You may pass either: +- a **`kx.log` instance** (`(use\`kx.log).createLog[]`) - its unary `{[msg]}` functions are detected + (by the `getlvl`/`sinks`/`fmts` keys) and auto-wrapped to the binary contract by the internal + `normlog`, folding the context tag into the message (`"heartbeat: ..."`); or +- a **custom binary logger** - an `` `info`warn`error `` dict of `{[c;m]}` functions, used as-is + (extra keys are passed through untouched). + +`timer` and `handlers` follow the standard kdb-x core dependency contracts; `pubsub` and +`servers` are heartbeat-specific. `servers` and `handlers` are only required when `subenabled` +is set (i.e. this process monitors other heartbeats); a pure publisher needs `log`, `timer` +and `pubsub`. Only the functions the module actually calls are accessed (`timer`'s `addjob` and `deletejobs`, `handlers`' `register`), but supplying the full contracted dictionary @@ -48,8 +52,10 @@ e.g. `heartbeat.setcp[{2025.01.01D00:00:00.000}]`. ## Configuration -`init[config;deps]` takes a configuration dictionary as its first argument. Any -recognised key may be supplied; unset keys keep their defaults. +`init[deps]` takes a **single dictionary** carrying both config overrides and the injected +dependencies. The recognised config keys below are all optional - omit any and the module +falls back to the default; unrecognised keys are ignored. The dependency keys (`log`, `timer`, +`pubsub`, and `servers`/`handlers` when monitoring) live in the same dictionary. | Key | Default | Description | |-----|---------|-------------| @@ -71,13 +77,12 @@ recognised key may be supplied; unset keys keep their defaults. | Function | Description | |----------|-------------| -| `init[config;deps]` | wire dependencies and configuration, and schedule the timer jobs | +| `init[deps]` | wire config + dependencies (one dict) and schedule the timer jobs | | `publishheartbeat[]` | publish a single heartbeat row and increment the counter | | `checkheartbeat[]` | flag processes that have not heartbeated in time | | `storeheartbeat[batch]` | store incoming heartbeat(s); call from `upd` on the monitor | | `addprocs[proctypes;procnames]` | seed expected processes so a never-seen process is flagged | | `subscribe[handles]` | subscribe to heartbeats on the given remote handle(s) | -| `hbsubscriptions[]` | subscribe to all configured publishers (by `connections` process type) | | `gethb[]` | return the heartbeat store | | `setcp[f]` | replace the current-time function (for tests / simulation) | @@ -118,8 +123,9 @@ timerdep: `addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) pubsub: use `di.pubsub psdep: `publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) -// initialise as a publishing RDB - log, timer and pubsub all required -heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(kxlog;timerdep;psdep)] +// initialise from a single dict - config keys + the (required) log/timer/pubsub dependencies +// the kx.log instance is passed straight through; normlog wraps it to the binary contract +heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] // publish a heartbeat immediately (normally the timer does this) heartbeat.publishheartbeat[] diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index eebf1d28..986d42e8 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -4,8 +4,9 @@ / blocked - even when the underlying connection is still valid / the module handles both publishing heartbeats and, on the monitoring side, / storing received heartbeats and raising warnings / errors when they stop -/ runtime dependencies are injected via init and are required - log, timer and pubsub always -/ (servers and handlers when monitoring); init errors immediately if one is missing - see heartbeat.md +/ config and dependencies are passed to init in a single dictionary: config keys (see configkeys) +/ are optional with defaults; log/timer/pubsub are required (servers/handlers when monitoring) and +/ init errors immediately if a required dependency is missing - see heartbeat.md / module-local state convention: read config bare, mutate via .z.m, and access / injected dependencies via .z.m at every call site @@ -62,18 +63,31 @@ requiredep:{[deps;name] d }; +normlog:{[logdict] + / detect kx.log instance by presence of kx.log-specific keys (getlvl, sinks, fmts) + / kx.log functions are monadic - wrap each into binary {[c;m]} and embed context in the message + / plain {[c;m]} log dicts (info`warn`error only) pass through unchanged + $[any `getlvl`sinks`fmts in key logdict; + `info`warn`error!( + {[fn;c;m] fn[string[c],": ",m]}[logdict`info;]; + {[fn;c;m] fn[string[c],": ",m]}[logdict`warn;]; + {[fn;c;m] fn[string[c],": ",m]}[logdict`error;]); + logdict] + }; + setdeps:{[deps] - / extract and store injected dependencies under .z.m (log kept whole; the rest as the functions used) + / extract injected dependencies from the single deps dict (which also carries config keys) / log, timer and pubsub are required; servers and handlers only when monitoring / init errors immediately if deps is not a dictionary or a required dependency is missing/malformed / nested if guards (not and) - and evaluates both sides eagerly and key would throw on a non-dict - if[99h<>type deps;'"di.heartbeat: deps must be a dictionary of injected dependencies - see heartbeat.md"]; - / log - required: a dict providing info/warn/error (e.g. a kx.log logger); only those three are - / mandated but the whole logger is kept, so extra levels (debug/fatal/custom) remain available - if[not `log in key deps;'"di.heartbeat: log dependency is required; pass a logger keyed on `log - see kx.log"]; - if[99h<>type deps`log;'"di.heartbeat: log value must be a dict of info/warn/error functions"]; - if[not all `info`warn`error in key deps`log;'"di.heartbeat: log dict must have `info`warn`error keys; got: ",", " sv string key deps`log]; - .z.m.log:deps`log; + if[99h<>type deps;'"di.heartbeat: deps must be a dictionary of config and injected dependencies - see heartbeat.md"]; + / log - required: info/warn/error (heartbeat uses all three); a kx.log instance is auto-wrapped + / to the binary {[c;m]} contract by normlog, so it can be passed directly + if[not `log in key deps;'"di.heartbeat: log dependency is required; pass `info`warn`error (or a kx.log logger) keyed on `log"]; + if[99h<>type deps`log;'"di.heartbeat: log must be a dict of info/warn/error functions (or a kx.log logger)"]; + lg:normlog deps`log; + if[not all `info`warn`error in key lg;'"di.heartbeat: log must provide info/warn/error; got: ",", " sv string key lg]; + .z.m.log:lg; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; .z.m.timerdeletejobs:timerdict`deletejobs; @@ -126,23 +140,22 @@ registerhandlers:{ .z.m.handlersregister[`.z.pc;`heartbeat;closeconnection]]; }; -init:{[config;deps] - / initialise the module with configuration and injected dependencies - see heartbeat.md - / config - dictionary of configuration overrides (see configkeys), or (::) for defaults - / deps - dictionary of injected dependencies keyed by name, each a dictionary of functions: - / `log - a kx.log logger - required; must provide unary info/warn/error, may provide more +init:{[deps] + / initialise from a single dictionary holding config overrides and injected dependencies - see heartbeat.md + / config keys (see configkeys) are optional and fall back to defaults; dependencies are required: + / `log - a logger - required; must provide info/warn/error (a kx.log instance is auto-wrapped) / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required / `pubsub - `publish`subscribe - required / `servers - `getservers - required when subenabled / `handlers - `register`remove`list - required when subenabled / note: the module keeps its own clock (cp, default .z.p) - override via setcp / example: - / heartbeat.init[`proctype`procname!(`rdb;`rdb1); `log`timer`pubsub!(logdep;timerdep;psdep)] - setconfig config; + / heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] + setconfig deps; setdeps deps; registertimers[]; registerhandlers[]; - .z.m.log[`info]["heartbeat: di.heartbeat initialised"]; + .z.m.log[`info][`heartbeat;"di.heartbeat initialised"]; }; publishheartbeat:{ @@ -167,12 +180,12 @@ addprocs:{[proctypes;procnames] logwarnproc:{[r] / log a single process moving into warning state - .z.m.log[`warn]["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.log[`warn][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; logerrproc:{[r] / log a single process moving into error state - .z.m.log[`error]["heartbeat: process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; + .z.m.log[`error][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; warn:{[procs] @@ -206,7 +219,7 @@ checkheartbeat:{ subscribeone:{[h] / subscribe to a single remote heartbeat publisher, logging and skipping on failure - ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.log[`error]["heartbeat: failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.log[`error][`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; }; @@ -219,7 +232,7 @@ getheartbeats:{[proctype] / subscribe to publishers of the given process type(s) that are not yet subscribed handles:(.z.m.serversgetservers proctype) except subscribedhandles; if[count handles; - .z.m.log[`info]["heartbeat: subscribing to new heartbeat handle(s) ",", " sv string handles]; + .z.m.log[`info][`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; subscribe handles]; }; diff --git a/di/heartbeat/init.q b/di/heartbeat/init.q index e4d90ee8..a004ce22 100644 --- a/di/heartbeat/init.q +++ b/di/heartbeat/init.q @@ -1,8 +1,8 @@ / load core functionality into the module \l ::heartbeat.q -/ module version - compared against dependants' minimum requirements by di.depcheck -version:"0.1.0"; +/ module version - compared against dependants' minimum requirements by di.depcheck. Removed for now until PR merged +/version:"0.1.0"; / public api - only the functions intended to be called externally are exported -export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;gethb;setcp;version]) +export:([init;publishheartbeat;checkheartbeat;storeheartbeat;addprocs;subscribe;gethb;setcp]) diff --git a/di/heartbeat/test.csv b/di/heartbeat/test.csv index f3150ab9..27396a11 100644 --- a/di/heartbeat/test.csv +++ b/di/heartbeat/test.csv @@ -1,13 +1,13 @@ action,ms,bytes,lang,code,repeat,minver,comment -comment,,,,,,,Setup - load module and inject mock dependencies +comment,,,,,,,Setup - load module and inject mock dependencies (single deps dict holds config + deps) before,0,0,q,heartbeat:use`di.heartbeat,1,1,load the heartbeat module -before,0,0,q,logdep:`info`warn`error!({[m]};{[m]};{[m]}),1,1,silent log mock - unary {[m]} matches the kx.log logger contract -before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the timer mock +before,0,0,q,logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]}),1,1,silent binary log mock - {[c;m]} is the internal contract +before,0,0,q,.test.now:2025.01.01D00:00:00.000,1,1,controllable current time for the clock before,0,0,q,timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]}),1,1,timer mock - heartbeat requires addjob and deletejobs before,0,0,q,psdep:`publish`subscribe!({[t;x] .test.pubt:t;.test.pubx:x;.test.pubcount:.test.pubcount+1};{[h] .test.subh:h}),1,1,pubsub mock capturing publish and subscribe before,0,0,q,.test.pubcount:0,1,1,initialise publish counter -before,0,0,q,deps:`log`timer`pubsub!(logdep;timerdep;psdep),1,1,assemble dependency dictionary -before,0,0,q,heartbeat.init[`proctype`procname!(`rdb;`rdb1);deps],1,1,initialise with identity config and mocks +before,0,0,q,deps:`log`timer`pubsub!(logdep;timerdep;psdep),1,1,dependency-only dict (reused by the required-dependency fail tests) +before,0,0,q,heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;logdep;timerdep;psdep)],1,1,initialise from a single dict of config + deps before,0,0,q,heartbeat.setcp[{[].test.now}],1,1,point heartbeat clock at the controllable test time comment,,,,,,,publishheartbeat publishes a row and bumps the counter @@ -57,50 +57,56 @@ run,0,0,q,.m.di.0heartbeat.closeconnection[5i],1,1,simulate connection close true,0,0,q,not 5i in .m.di.0heartbeat.subscribedhandles,1,1,handle removed on close comment,,,,,,,log is required - init errors clearly when deps or log is missing or malformed -fail,0,0,q,heartbeat.init[()!();(::)],1,1,errors when deps is not a dictionary -fail,0,0,q,heartbeat.init[()!();()!()],1,1,errors when log dependency missing -fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist 42],1,1,errors when log value is not a dictionary -fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist `info`warn!(logdep`info;logdep`warn)],1,1,errors when log dict is missing the error key +fail,0,0,q,heartbeat.init[(::)],1,1,errors when deps is not a dictionary +fail,0,0,q,heartbeat.init[()!()],1,1,errors when log dependency missing +fail,0,0,q,heartbeat.init[enlist[`log]!enlist 42],1,1,errors when log value is not a dictionary +fail,0,0,q,heartbeat.init[enlist[`log]!enlist `info`warn!(logdep`info;logdep`warn)],1,1,errors when log dict is missing the error key comment,,,,,,,timer and pubsub are required - init errors when they are missing -fail,0,0,q,heartbeat.init[()!();enlist[`log]!enlist logdep],1,1,errors when timer dependency missing -fail,0,0,q,heartbeat.init[()!();`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing -fail,0,0,q,heartbeat.init[enlist[`subenabled]!enlist 1b;`log`timer`pubsub!(logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing -run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);deps],1,1,re-init with full mock deps and subenabled 0b to restore clean module state (prior fail test left subenabled 1b) +fail,0,0,q,heartbeat.init[enlist[`log]!enlist logdep],1,1,errors when timer dependency missing +fail,0,0,q,heartbeat.init[`log`timer!(logdep;timerdep)],1,1,errors when pubsub dependency missing +fail,0,0,q,heartbeat.init[`subenabled`log`timer`pubsub!(1b;logdep;timerdep;psdep)],1,1,monitor errors when servers dependency missing +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;logdep;timerdep;psdep)],1,1,re-init with full deps and subenabled 0b to restore clean module state -comment,,,,,,,an injected logger is actually invoked (guards against silent no-op fallback) +comment,,,,,,,an injected (binary) logger is actually invoked run,0,0,q,.test.loginfo:"",1,1,reset the capture -run,0,0,q,caplog:`info`warn`error!({[m] .test.loginfo:m};{[m] .test.logwarn:m};{[m] .test.logerr:m}),1,1,capturing unary logger - records the last message at each level -run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(caplog;timerdep;psdep)],1,1,init with the capturing logger -true,0,0,q,.test.loginfo~"heartbeat: di.heartbeat initialised",1,1,init message routed through the injected logger not the no-op fallback +run,0,0,q,caplog:`info`warn`error!({[c;m] .test.loginfo:m};{[c;m] .test.logwarn:m};{[c;m] .test.logerr:m}),1,1,capturing binary logger - records the last message at each level +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;caplog;timerdep;psdep)],1,1,init with the capturing logger +true,0,0,q,.test.loginfo~"di.heartbeat initialised",1,1,init message routed through the injected logger -comment,,,,,,,only info/warn/error are mandated - a richer logger is accepted and kept whole -run,0,0,q,caplog2:`info`warn`error`debug!({[m]};{[m]};{[m]};{[m]}),1,1,logger providing an extra debug level beyond the mandatory three -run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(caplog2;timerdep;psdep)],1,1,init with the richer logger -true,0,0,q,`debug in key .m.di.0heartbeat.log,1,1,extra level retained - the whole logger is kept not stripped to three functions +comment,,,,,,,a kx.log instance (getlvl/sinks/fmts + unary fns) is auto-wrapped to the binary contract by normlog +run,0,0,q,.test.kxmsg:"",1,1,reset the capture +run,0,0,q,fakekx:`info`warn`error`getlvl`sinks`fmts!({[m] .test.kxmsg:m};{[m]};{[m]};`info;()!();()!()),1,1,fake kx.log instance - unary level fns plus the marker keys +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;fakekx;timerdep;psdep)],1,1,init with the kx.log-style instance +true,0,0,q,.test.kxmsg~"heartbeat: di.heartbeat initialised",1,1,unary fn wrapped to binary - context folded into the message +true,0,0,q,`info`warn`error~key .m.di.0heartbeat.log,1,1,wrapped logger exposes exactly the binary info/warn/error + +comment,,,,,,,a binary logger with extra levels is accepted and passed through unchanged +run,0,0,q,caplog2:`info`warn`error`debug!({[c;m]};{[c;m]};{[c;m]};{[c;m]}),1,1,binary logger providing an extra debug level +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;caplog2;timerdep;psdep)],1,1,init with the richer binary logger +true,0,0,q,`debug in key .m.di.0heartbeat.log,1,1,extra level retained - a non-kx.log dict is passed through unchanged true,0,0,q,all `info`warn`error in key .m.di.0heartbeat.log,1,1,mandatory levels present comment,,,,,,,init is idempotent - re-running clears jobs first instead of colliding on ids run,0,0,q,.test.jobids:(`$())!`boolean$(),1,1,track registered job ids run,0,0,q,timerdep2:`addjob`deletejobs!({[id;func;params;period;mode;opts] if[id in key .test.jobids;'"duplicate job id"]; .test.jobids[id]:1b};{[ids] .test.jobids:(key[.test.jobids] except ids)#.test.jobids}),1,1,stateful timer mock that rejects duplicate job ids like di.timer -run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(logdep;timerdep2;psdep)],1,1,first init registers the jobs -run,0,0,q,heartbeat.init[`proctype`procname`subenabled!(`rdb;`rdb1;0b);`log`timer`pubsub!(logdep;timerdep2;psdep)],1,1,second init must not collide - clears then re-adds +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;logdep;timerdep2;psdep)],1,1,first init registers the jobs +run,0,0,q,heartbeat.init[`proctype`procname`subenabled`log`timer`pubsub!(`rdb;`rdb1;0b;logdep;timerdep2;psdep)],1,1,second init must not collide - clears then re-adds true,0,0,q,`hbcheck`hbpublish~asc key .test.jobids,1,1,exactly the publish and check jobs are registered after re-init comment,,,,,,,real-process integration - a background publisher beats over real di.pubsub into this process run,0,0,q,system"q -p 5050 -q &",1,,start a background heartbeat publisher run,0,0,q,system"sleep 1",1,,wait for the publisher to come up run,0,0,q,h:hopen`::5050,1,,connect to the publisher - true,0,0,q,h>0,1,,connected to the publisher run,0,0,q,h"hbmod:use`di.heartbeat",1,,load heartbeat on the publisher run,0,0,q,h"pubsub:use`di.pubsub",1,,load pubsub on the publisher run,0,0,q,h"heartbeat:.m.di.0heartbeat.heartbeat",1,,expose the heartbeat schema at root for pubsub run,0,0,q,h"pubsub.init[]",1,,initialise the pubsub registry on the publisher -run,0,0,q,h"logdep:`info`warn`error!({[m]};{[m]};{[m]})",1,,silent unary log dep on the publisher (kx.log contract) +run,0,0,q,h"logdep:`info`warn`error!({[c;m]};{[c;m]};{[c;m]})",1,,silent binary log dep on the publisher run,0,0,q,h"timerdep:`addjob`deletejobs!({[id;func;params;period;mode;opts]};{[ids]})",1,,no-op timer dep - publish is triggered manually for determinism run,0,0,q,h"psdep:`publish`subscribe!(pubsub.publish;{[x]})",1,,inject the real pubsub publish into heartbeat -run,0,0,q,h"hbmod.init[`proctype`procname`port!(`rdb;`pub1;5050i);`log`timer`pubsub!(logdep;timerdep;psdep)]",1,,init the publisher as an rdb named pub1 +run,0,0,q,h"hbmod.init[`proctype`procname`port`log`timer`pubsub!(`rdb;`pub1;5050i;logdep;timerdep;psdep)]",1,,init the publisher as an rdb named pub1 from a single dict run,0,0,q,upd:{[t;x] if[`heartbeat~t; heartbeat.storeheartbeat x]},1,,route incoming beats into the local store run,0,0,q,.m.di.0heartbeat.hb:0#.m.di.0heartbeat.hb,1,,clear the local store before subscribing run,0,0,q,h(`pubsub.subscribe;`heartbeat;`),1,,subscribe this process to the publisher's heartbeat feed From 4d1caf4688e7cc7c6940b41cddb61559e7c1b416 Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Thu, 25 Jun 2026 15:48:04 +0100 Subject: [PATCH 15/16] Final changes --- di/heartbeat/heartbeat.md | 251 +++++++++++++++++++++++--------------- di/heartbeat/heartbeat.q | 237 ++++++++++++++++++----------------- 2 files changed, 278 insertions(+), 210 deletions(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 6414d079..9e8803a1 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -1,97 +1,126 @@ -# Heartbeat +# di.heartbeat -This module lets every process publish a periodic heartbeat over pub/sub, and lets -monitoring processes detect when a process has stopped beating - i.e. it is stalled -or blocked - even when the underlying connection is still valid. +Periodic process-liveness signalling over pub/sub for kdb-x. Every process can publish a regular heartbeat so that downstream monitors detect when a process has stopped beating - i.e. it is stalled or blocked - even when the underlying connection is still valid. The module covers both sides: publishing heartbeats, and on the monitoring side storing received beats and raising a warning then an error when a process stops within configurable grace periods. -It covers both sides: +--- -* **Publishing** - a process periodically publishes a heartbeat row over pub/sub. -* **Monitoring** - a process subscribes to other processes' heartbeats, stores the - latest beat per process, and raises a *warning* then an *error* when a process - stops heartbeating within the configured grace periods. +## Features + +- Publish a periodic heartbeat row over pub/sub so downstream monitors can detect a stalled or blocked process even while its connection is still open +- Monitor side: subscribe to other processes' heartbeats and store the latest beat per process as inspectable module state +- Escalate from healthy to *warning* to *error* when a process stops heartbeating, using per-process-type grace periods (`warningtolerance`/`errortolerance` x `publishinterval`) +- Fire user-supplied `onwarning`/`onerror` callbacks with the rows entering each state +- Seed expected processes with `addprocs` so a never-seen process is flagged immediately +- Owns its own clock (`cp`, default `.z.p`), overridable via `setcp` for deterministic tests or simulation +- Idempotent `init` - re-running clears and re-registers its timer jobs safely +- No hard module dependencies - log, timer, pubsub (and servers/handlers when monitoring) are all injected via `init` + +--- ## Dependencies -Config and dependencies are passed together in a **single dictionary** to `init` (see -Configuration). The dependencies below are **required** - `init` errors immediately with a -clear message if `deps` is not a dictionary or a required dependency is missing or malformed. -There is no hard dependency on any other module: any module exporting the contracted function -signatures can be supplied. - -| Dependency | Keys | Required | Purpose | -|------------|------|----------|---------| -| `log` | a logger providing `info` `warn` `error`; a `kx.log` instance is accepted directly | always | logging | -| `timer` | `addjob` `deletejobs` (the full `di.timer` dict may be passed) | always | scheduling the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely) | -| `pubsub` | `publish` (`{[table;data]}`) `subscribe` (`{[handle]}`) | always | publishing heartbeats / subscribing to publishers | -| `servers` | `getservers` (`{[proctype]}` returning handles) | when `subenabled` | discovering heartbeat publishers by process type | -| `handlers` | `register` `remove` `list` | when `subenabled` | registering the connection-close (`.z.pc`) cleanup | - -**Logging contract.** Internally the module calls the logger as **binary** `.z.m.log[\`info][\`heartbeat;"msg"]` -(`{[c;m]}` - context symbol + message). Only `info`/`warn`/`error` are mandated (heartbeat uses all -three). You may pass either: -- a **`kx.log` instance** (`(use\`kx.log).createLog[]`) - its unary `{[msg]}` functions are detected - (by the `getlvl`/`sinks`/`fmts` keys) and auto-wrapped to the binary contract by the internal - `normlog`, folding the context tag into the message (`"heartbeat: ..."`); or -- a **custom binary logger** - an `` `info`warn`error `` dict of `{[c;m]}` functions, used as-is - (extra keys are passed through untouched). - -`timer` and `handlers` follow the standard kdb-x core dependency contracts; `pubsub` and -`servers` are heartbeat-specific. `servers` and `handlers` are only required when `subenabled` -is set (i.e. this process monitors other heartbeats); a pure publisher needs `log`, `timer` -and `pubsub`. - -Only the functions the module actually calls are accessed (`timer`'s `addjob` and -`deletejobs`, `handlers`' `register`), but supplying the full contracted dictionary -keeps the dependency interchangeable with the real `di.*` modules. - -The module keeps its **own** current-time function rather than taking it from the -timer dependency (so it doesn't rely on the timer exporting a clock getter). It -defaults to `.z.p`; override it with `setcp` for deterministic tests or simulation, -e.g. `heartbeat.setcp[{2025.01.01D00:00:00.000}]`. - -## Configuration - -`init[deps]` takes a **single dictionary** carrying both config overrides and the injected -dependencies. The recognised config keys below are all optional - omit any and the module -falls back to the default; unrecognised keys are ignored. The dependency keys (`log`, `timer`, -`pubsub`, and `servers`/`handlers` when monitoring) live in the same dictionary. +| Dependency | Key | Required | Description | +|---|---|---|---| +| logger | `` `log `` | yes | `info`/`warn`/`error` - binary `{[c;m]}` where `c` is a symbol context and `m` is a string. Heartbeat uses all three. A `kx.log` instance is accepted directly | +| timer | `` `timer `` | yes | `addjob`, `deletejobs` - schedules the publish / check / subscribe jobs (`deletejobs` lets `init` be re-run safely). The full `di.timer` dict may be passed | +| pubsub | `` `pubsub `` | yes | `publish` (`{[table;data]}`), `subscribe` (`{[handle]}`) - publishing heartbeats / subscribing to publishers | +| servers | `` `servers `` | when `subenabled` | `getservers` (`{[proctype]}` returning handles) - discovering heartbeat publishers by process type | +| handlers | `` `handlers `` | when `subenabled` | `register` - wiring the connection-close (`.z.pc`) cleanup | + +**Hard dependency:** none - all runtime dependencies are injected via `init`, so any module exporting the contracted signatures can be supplied. + +The dependencies are passed to `init` inside the single `deps` dict alongside any configuration. They are **required**: `init` throws immediately if `deps` is not a dictionary or a required dependency is missing or malformed. `servers` and `handlers` are only required when `subenabled` is set (i.e. this process monitors others); a pure publisher needs only `log`, `timer` and `pubsub`. + +**Logging contract.** Internally the module calls the logger as binary `.z.m.log[\`info][\`heartbeat;"msg"]` (`{[c;m]}` - context symbol + message). `info`/`warn`/`error` are all mandated because heartbeat uses all three. You may pass either a `kx.log` instance (`(use\`kx.log).createLog[]`) - its monadic `{[msg]}` functions are detected and auto-wrapped to the binary contract by the internal `normlog`, folding the context tag into the message (`"heartbeat: ..."`) - or a custom `` `info`warn`error `` dict of `{[c;m]}` functions, used as-is. + +The module keeps its **own** current-time function rather than taking it from the timer dependency. It defaults to `.z.p`; override it with `setcp` for deterministic tests or simulation, e.g. `heartbeat.setcp[{2025.01.01D00:00:00.000}]`. + +--- + +## Initialisation + +`init[deps]` takes a single dictionary combining the injected dependencies (above) with any configuration overrides. All config keys are optional - omit any and the module falls back to the default; unrecognised keys are ignored. | Key | Default | Description | -|-----|---------|-------------| -| `enabled` | `1b` | publish and check heartbeats | -| `subenabled` | `0b` | act as a monitor: subscribe to other heartbeats and register the disconnect handler | -| `debug` | `1b` | log warning / error transitions | -| `publishinterval` | `0D00:00:30` | how often heartbeats are published | -| `checkinterval` | `0D00:00:10` | how often received heartbeats are checked | -| `warningtolerance` | `1.5` | warning after `warningtolerance*publishinterval` without a beat | -| `errortolerance` | `2f` | error after `errortolerance*publishinterval` without a beat | -| `proctype` | `` `unknown `` | this process's type (published as `sym`) | -| `procname` | `.z.h` | this process's name | -| `pid` `host` `port` | from `.z` | this process's identity | -| `connections` | `` `$() `` | process types this monitor subscribes to (used by `hbsubscriptions`) | -| `onwarning` | no-op | callback invoked with the rows entering warning state | -| `onerror` | no-op | callback invoked with the rows entering error state | - -## Public API - -| Function | Description | -|----------|-------------| -| `init[deps]` | wire config + dependencies (one dict) and schedule the timer jobs | -| `publishheartbeat[]` | publish a single heartbeat row and increment the counter | -| `checkheartbeat[]` | flag processes that have not heartbeated in time | -| `storeheartbeat[batch]` | store incoming heartbeat(s); call from `upd` on the monitor | -| `addprocs[proctypes;procnames]` | seed expected processes so a never-seen process is flagged | -| `subscribe[handles]` | subscribe to heartbeats on the given remote handle(s) | -| `gethb[]` | return the heartbeat store | -| `setcp[f]` | replace the current-time function (for tests / simulation) | +|---|---|---| +| `` `enabled `` | `1b` | publish and check heartbeats | +| `` `subenabled `` | `0b` | act as a monitor: subscribe to other heartbeats and register the disconnect handler | +| `` `debug `` | `1b` | log warning / error transitions (callbacks still fire when off) | +| `` `publishinterval `` | `0D00:00:30` | how often heartbeats are published | +| `` `checkinterval `` | `0D00:00:10` | how often received heartbeats are checked | +| `` `warningtolerance `` | `1.5` | warning after `warningtolerance*publishinterval` without a beat | +| `` `errortolerance `` | `2f` | error after `errortolerance*publishinterval` without a beat | +| `` `proctype `` | `` `unknown `` | this process's type (published as `sym`) | +| `` `procname `` | `.z.h` | this process's name | +| `` `pid `` `` `host `` `` `port `` | from `.z` | this process's identity | +| `` `connections `` | `` `$() `` | process types this monitor subscribes to (used by the subscribe job) | +| `` `onwarning `` | no-op | callback invoked with the rows entering warning state | +| `` `onerror `` | no-op | callback invoked with the rows entering error state | + +`init` must be called before any other function. It applies config, wires dependencies, and schedules the timer jobs. Re-running `init` resets unset config keys to their defaults and re-registers the timer jobs. + +--- + +## Exported Functions + +### `init[deps]` +Wire config + dependencies (one dict) and schedule the timer jobs. Must be called before anything else. +```q +heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] +/ or as a monitor: +heartbeat.init[`subenabled`connections`log`timer`pubsub`servers`handlers!(1b;`rdb`hdb;kxlog;timerdep;psdep;serversdep;handlersdep)] +``` + +### `publishheartbeat[]` +Publish a single heartbeat row over pub/sub and increment the counter. Normally driven by the timer; a no-op when `enabled` is `0b`. +```q +heartbeat.publishheartbeat[] +``` + +### `checkheartbeat[]` +Flag processes that have not heartbeated within the warning / error grace periods, firing `onwarning`/`onerror` on transitions. Driven by the timer on the monitor. +```q +heartbeat.checkheartbeat[] +``` + +### `storeheartbeat[batch]` +Store one or more incoming heartbeats, keeping the latest per process and clearing warning / error state. Call from `upd` on the monitor. +```q +upd:{[t;x] if[t~`heartbeat; heartbeat.storeheartbeat[x]]; } +``` + +### `addprocs[proctypes;procnames]` +Seed the store with expected processes so a never-seen process is flagged. Real heartbeats arriving later override the seeded rows. +```q +heartbeat.addprocs[`rdb`hdb; `rdb1`hdb1] +``` + +### `subscribe[handles]` +Subscribe to heartbeats on the given remote handle(s), tracking successful subscriptions and skipping any that fail. +```q +heartbeat.subscribe hopen `:remotehost:5050 +``` + +### `gethb[]` +Return the current heartbeat store (keyed on `sym`,`procname`) for inspection. +```q +heartbeat.gethb[] +``` + +### `setcp[f]` +Replace the current-time function (for deterministic tests / simulation). Defaults to `.z.p`. +```q +heartbeat.setcp[{2025.01.01D00:00:00.000}] +``` + +--- ## Heartbeat store schema `gethb[]` returns the store, keyed on `sym`,`procname`: | Column | Type | Description | -|--------|------|-------------| +|---|---|---| | sym | `symbol` | process type | | procname | `symbol` | process name | | time | `timestamp` | time of the last received heartbeat | @@ -102,39 +131,59 @@ falls back to the default; unrecognised keys are ignored. The dependency keys (` | warning | `boolean` | process is in warning state | | error | `boolean` | process is in error state | -## Example +--- + +## Usage Example ```q -// load the module -heartbeat: use `di.heartbeat +/ --- publisher --- +kxlog:use`kx.log -// log is a required kx.log logger instance (its info/warn/error are unary) -// bound as kxlog, not log, since log is a reserved q word -logger: use `kx.log -kxlog: logger.createLog[] +heartbeat:use`di.heartbeat -timer: use `di.timer +timer:use`di.timer timer.init[()!()] -// heartbeat needs addjob and deletejobs from the timer - it keeps its own clock (see setcp) -// note: di.timer's addjob is a namespace; addjob.custom has the [id;func;params;period;mode;opts] signature heartbeat calls -timerdep: `addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) +/ heartbeat needs addjob and deletejobs - it keeps its own clock (see setcp) +/ note: di.timer's addjob is a namespace; addjob.custom has the [id;func;params;period;mode;opts] signature heartbeat calls +timerdep:`addjob`deletejobs!(timer.addjob.custom; timer.deletejobs) -// a pubsub dependency must provide publish[table;data] and subscribe[handle] -pubsub: use `di.pubsub -psdep: `publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) +/ pubsub must provide publish[table;data] and subscribe[handle] +pubsub:use`di.pubsub +psdep:`publish`subscribe!(pubsub.publish; {[h] h(`.m.di.0pubsub.subscribe;`heartbeat;`)}) -// initialise from a single dict - config keys + the (required) log/timer/pubsub dependencies -// the kx.log instance is passed straight through; normlog wraps it to the binary contract -heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] +/ initialise from a single dict - config keys + the (required) log/timer/pubsub dependencies +/ the kx.log instance is passed straight through; normlog wraps it to the binary contract +heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog.createLog[];timerdep;psdep)] -// publish a heartbeat immediately (normally the timer does this) +/ publish a heartbeat immediately (normally the timer does this) heartbeat.publishheartbeat[] ``` -On the monitoring side, route received heartbeats into the store from `upd` and let -the scheduled `checkheartbeat` raise warnings / errors: +On the monitoring side, route received heartbeats into the store from `upd` and let the scheduled `checkheartbeat` raise warnings / errors: ```q -upd: {[t;x] if[t~`heartbeat; heartbeat.storeheartbeat[x]]; } +upd:{[t;x] if[t~`heartbeat; heartbeat.storeheartbeat[x]]; } heartbeat.gethb[] ``` + +--- + +## Running Tests + +```q +k4unit:use`di.k4unit +k4unit.moduletest`di.heartbeat +``` + +The test suite injects no-op binary mock loggers and a capturing logger that records messages for assertion. It covers: dependency validation (non-dict deps throws; missing/non-dict `log` throws; `log` missing a level throws; missing `timer`/`pubsub` throws; missing monitor `servers`/`handlers` throws); the capturing-logger test confirming `init` logs `"di.heartbeat initialised"`; `normlog` wrapping a fake `kx.log` instance end-to-end (and extra log levels passing through untouched); idempotency of re-running `init`; and a real-process integration test driving a background publisher over a live handle. + +--- + +## Notes + +- The module keeps its own clock (`cp`, default `.z.p`) rather than taking one from the timer dependency - override it with `setcp` for deterministic tests or simulation +- Timer jobs use mode 2 (period after the previous actual start) so missed beats are not replayed as a catch-up storm +- `init` is idempotent: it deletes its jobs (`hbpublish`/`hbcheck`/`hbsubscribe`) before re-registering, so it is safe to call again +- `servers` and `handlers` are only required when `subenabled` is set; a pure publisher needs only `log`, `timer` and `pubsub` +- A `kx.log` instance is accepted directly - `normlog` wraps its monadic functions to the binary `{[c;m]}` contract and folds the context tag into the message +- `debug` toggles whether warning / error transitions are logged; the `onwarning`/`onerror` callbacks still fire either way diff --git a/di/heartbeat/heartbeat.q b/di/heartbeat/heartbeat.q index 986d42e8..201e9cb5 100644 --- a/di/heartbeat/heartbeat.q +++ b/di/heartbeat/heartbeat.q @@ -4,12 +4,16 @@ / blocked - even when the underlying connection is still valid / the module handles both publishing heartbeats and, on the monitoring side, / storing received heartbeats and raising warnings / errors when they stop -/ config and dependencies are passed to init in a single dictionary: config keys (see configkeys) +/ config and dependencies are passed to init in a single dictionary: config keys (see heartbeat.md) / are optional with defaults; log/timer/pubsub are required (servers/handlers when monitoring) and / init errors immediately if a required dependency is missing - see heartbeat.md / module-local state convention: read config bare, mutate via .z.m, and access / injected dependencies via .z.m at every call site +/ ============================================================ +/ module state and defaults +/ ============================================================ + / table used to publish heartbeats - sym holds the publishing process type heartbeat:([] time:`timestamp$(); sym:`symbol$(); procname:`symbol$(); counter:`long$(); pid:`int$(); host:`symbol$(); port:`int$()); @@ -19,17 +23,12 @@ hb:update warning:0b,error:0b from `sym`procname xkey heartbeat; / remote handles we have already subscribed to for heartbeats subscribedhandles:`int$(); -/ heartbeat counter +/ heartbeat counter - bumped on each publish hbcounter:0; / current-time function - heartbeat owns its clock; override via setcp for testing / simulation cp:{.z.p}; -setcp:{[f] - / replace the current-time function (used by tests and simulation) - .z.m.cp:f; - }; - / configuration defaults - overridden by the config dictionary passed to init enabled:1b; / whether heartbeat publishing / checking is enabled subenabled:0b; / whether this process monitors (subscribes to) other heartbeats @@ -47,21 +46,9 @@ connections:`$(); / process types this monitor should subscribe to onwarning:{[procs]}; / callback fired with the rows entering warning state onerror:{[procs]}; / callback fired with the rows entering error state -/ recognised configuration keys - anything else passed via config is ignored -configkeys:`enabled`subenabled`debug`publishinterval`checkinterval`warningtolerance`errortolerance, - `proctype`procname`pid`host`port`connections`onwarning`onerror; - -/ warning / error grace periods - vary by process type if required -warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; -errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; - -requiredep:{[deps;name] - / extract a required dependency dictionary, erroring immediately if absent or null - d:$[99h=type deps;$[(name in key deps) and not (::)~deps name;deps name;()!()];()!()]; - if[not count d; - '"di.heartbeat: ",(string name)," dependency is required; pass it via init deps - see di.",string name]; - d - }; +/ ============================================================ +/ internal helpers +/ ============================================================ normlog:{[logdict] / detect kx.log instance by presence of kx.log-specific keys (getlvl, sinks, fmts) @@ -75,18 +62,37 @@ normlog:{[logdict] logdict] }; +/ extract a required dependency dictionary, erroring immediately if absent or null +requiredep:{[deps;name] + d:$[99h=type deps;$[(name in key deps) and not (::)~deps name;deps name;()!()];()!()]; + if[not count d; + '"di.heartbeat: ",(string name)," dependency is required; pass it via init deps - see di.",string name]; + d + }; + +/ warning / error grace periods - vary by process type if required +warningperiod:{[processtype] `timespan$warningtolerance*publishinterval}; +errorperiod:{[processtype] `timespan$errortolerance*publishinterval}; + +/ convert a timespan into whole seconds for the timer period +tosecs:{[span] `int$span%0D00:00:01}; + +/ wire the injected dependencies from the single deps dict (which also carries config keys) setdeps:{[deps] - / extract injected dependencies from the single deps dict (which also carries config keys) / log, timer and pubsub are required; servers and handlers only when monitoring / init errors immediately if deps is not a dictionary or a required dependency is missing/malformed / nested if guards (not and) - and evaluates both sides eagerly and key would throw on a non-dict - if[99h<>type deps;'"di.heartbeat: deps must be a dictionary of config and injected dependencies - see heartbeat.md"]; + if[99h<>type deps; + '"di.heartbeat: deps must be a dictionary of config and injected dependencies - see heartbeat.md"]; / log - required: info/warn/error (heartbeat uses all three); a kx.log instance is auto-wrapped / to the binary {[c;m]} contract by normlog, so it can be passed directly - if[not `log in key deps;'"di.heartbeat: log dependency is required; pass `info`warn`error (or a kx.log logger) keyed on `log"]; - if[99h<>type deps`log;'"di.heartbeat: log must be a dict of info/warn/error functions (or a kx.log logger)"]; + if[not `log in key deps; + '"di.heartbeat: log dependency is required; pass `info`warn`error (or a kx.log logger) keyed on `log"]; + if[99h<>type deps`log; + '"di.heartbeat: log must be a dict of info/warn/error functions (or a kx.log logger)"]; lg:normlog deps`log; - if[not all `info`warn`error in key lg;'"di.heartbeat: log must provide info/warn/error; got: ",", " sv string key lg]; + if[not all `info`warn`error in key lg; + '"di.heartbeat: log must provide info/warn/error; got: ",", " sv string key lg]; .z.m.log:lg; timerdict:requiredep[deps;`timer]; .z.m.timeraddjob:timerdict`addjob; @@ -99,30 +105,38 @@ setdeps:{[deps] if[subenabled;setmonitordeps deps]; }; +/ wire the monitor-only dependencies, required only when subenabled (this process monitors others) setmonitordeps:{[deps] - / wire the monitor-only dependencies, required only when subenabled (this process - / monitors others' heartbeats); split out of setdeps to keep that conditional a - / single statement per the coding standards + / split out of setdeps to keep that conditional a single statement per the coding standards serversdict:requiredep[deps;`servers]; .z.m.serversgetservers:serversdict`getservers; handlersdict:requiredep[deps;`handlers]; .z.m.handlersregister:handlersdict`register; }; -setconfig:{[config] - / apply recognised configuration overrides (a dictionary) on top of current values - cfg:$[99h=type config;config;()!()]; - ks:configkeys inter key cfg; - (.Q.dd[.z.M] each ks) set' cfg ks; - }; - -tosecs:{[span] - / convert a timespan into whole seconds for the timer period - `int$span%0D00:00:01 +/ apply recognised config overrides from the deps dict, defaulting where a key is absent +setconfig:{[deps] + / explicit per-key .z.m assignment (like di.eodtime) - no .z.M, and reinit resets to defaults + cfg:$[99h=type deps;deps;()!()]; + .z.m.enabled:$[`enabled in key cfg;cfg`enabled;1b]; + .z.m.subenabled:$[`subenabled in key cfg;cfg`subenabled;0b]; + .z.m.debug:$[`debug in key cfg;cfg`debug;1b]; + .z.m.publishinterval:$[`publishinterval in key cfg;cfg`publishinterval;0D00:00:30]; + .z.m.checkinterval:$[`checkinterval in key cfg;cfg`checkinterval;0D00:00:10]; + .z.m.warningtolerance:$[`warningtolerance in key cfg;cfg`warningtolerance;1.5]; + .z.m.errortolerance:$[`errortolerance in key cfg;cfg`errortolerance;2f]; + .z.m.proctype:$[`proctype in key cfg;cfg`proctype;`unknown]; + .z.m.procname:$[`procname in key cfg;cfg`procname;.z.h]; + .z.m.pid:$[`pid in key cfg;cfg`pid;.z.i]; + .z.m.host:$[`host in key cfg;cfg`host;.z.h]; + .z.m.port:$[`port in key cfg;cfg`port;`int$system"p"]; + .z.m.connections:$[`connections in key cfg;cfg`connections;`$()]; + .z.m.onwarning:$[`onwarning in key cfg;cfg`onwarning;{[procs]}]; + .z.m.onerror:$[`onerror in key cfg;cfg`onerror;{[procs]}]; }; +/ schedule the periodic heartbeat jobs via the injected timer registertimers:{ - / schedule the periodic heartbeat jobs via the injected timer / mode 2 = period after previous actual start - a heartbeat says "alive now", so / missed beats must not be replayed as a catch-up storm (which mode 1 would do) / clear any previously-registered jobs first so init is safe to call again @@ -134,76 +148,73 @@ registertimers:{ .z.m.timeraddjob[`hbsubscribe;hbsubscriptions;();60;2;()!()]]; }; +/ wire the connection-close cleanup through the injected handler manager registerhandlers:{ - / wire the connection-close cleanup through the injected handler manager if[subenabled; .z.m.handlersregister[`.z.pc;`heartbeat;closeconnection]]; }; -init:{[deps] - / initialise from a single dictionary holding config overrides and injected dependencies - see heartbeat.md - / config keys (see configkeys) are optional and fall back to defaults; dependencies are required: - / `log - a logger - required; must provide info/warn/error (a kx.log instance is auto-wrapped) - / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required - / `pubsub - `publish`subscribe - required - / `servers - `getservers - required when subenabled - / `handlers - `register`remove`list - required when subenabled - / note: the module keeps its own clock (cp, default .z.p) - override via setcp - / example: - / heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] - setconfig deps; - setdeps deps; - registertimers[]; - registerhandlers[]; - .z.m.log[`info][`heartbeat;"di.heartbeat initialised"]; - }; - -publishheartbeat:{ - / publish a single heartbeat row over pub/sub and bump the counter - if[not enabled;:()]; - .z.m.pubsubpublish[`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(cp[];proctype;procname;hbcounter;pid;host;port)]; - .z.m.hbcounter:hbcounter+1; - }; - -storeheartbeat:{[batch] - / store one or more incoming heartbeats, keeping the latest per process and - / clearing warning / error state - call this from upd when a heartbeat arrives - .z.m.hb:hb upsert update warning:0b,error:0b from select by sym,procname from batch; - }; - -addprocs:{[proctypes;procnames] - / seed the store with expected processes so a never-seen process is flagged - / real heartbeats arriving later override these seeded rows - seed:2!([]sym:proctypes,();procname:procnames,();time:cp[];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); - .z.m.hb:seed,hb; - }; - +/ log a single process moving into warning state logwarnproc:{[r] - / log a single process moving into warning state .z.m.log[`warn][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; +/ log a single process moving into error state logerrproc:{[r] - / log a single process moving into error state .z.m.log[`error][`heartbeat;"process ",(string r`procname)," (type ",(string r`sym),") has not heartbeated since ",string r`time]; }; +/ move processes into warning state, log and fire the warning callback warn:{[procs] - / move processes into warning state, log and fire the warning callback if[debug;logwarnproc each 0!procs]; .z.m.hb:hb upsert select sym,procname,warning:1b from procs; onwarning procs; }; +/ move processes into error state, log and fire the error callback err:{[procs] - / move processes into error state, log and fire the error callback if[debug;logerrproc each 0!procs]; .z.m.hb:hb upsert select sym,procname,error:1b from procs; onerror procs; }; +/ subscribe to a single remote heartbeat publisher, logging and skipping on failure +subscribeone:{[h] + ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.log[`error][`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; + if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; + }; + +/ subscribe to publishers of the given process type(s) that are not yet subscribed +getheartbeats:{[proctype] + handles:(.z.m.serversgetservers proctype) except subscribedhandles; + if[count handles; + .z.m.log[`info][`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; + subscribe handles]; + }; + +/ subscribe to all configured heartbeat publishers (by configured process type) - timer job +hbsubscriptions:{ + getheartbeats connections; + }; + +/ drop a closed handle from the tracked subscriptions - registered against .z.pc +closeconnection:{[h] + .z.m.subscribedhandles:subscribedhandles except h; + }; + +/ ============================================================ +/ public api +/ ============================================================ + +/ publish a single heartbeat row over pub/sub and bump the counter +publishheartbeat:{ + if[not enabled;:()]; + .z.m.pubsubpublish[`heartbeat;enlist `time`sym`procname`counter`pid`host`port!(cp[];proctype;procname;hbcounter;pid;host;port)]; + .z.m.hbcounter:hbcounter+1; + }; + +/ flag processes that have not heartbeated within the warning / error grace periods - timer job checkheartbeat:{ - / flag processes that have not heartbeated within the warning / error grace periods / status: 0 healthy, 1 warning, 2+ error / grace periods are computed as locals first - module functions do not resolve inside qsql now:cp[]; @@ -217,36 +228,44 @@ checkheartbeat:{ if[count newerr;err newerr]; }; -subscribeone:{[h] - / subscribe to a single remote heartbeat publisher, logging and skipping on failure - ok:@[{.z.m.pubsubsubscribe x;1b};h;{[h;e] .z.m.log[`error][`heartbeat;"failed to subscribe to heartbeats on handle ",(string h),": ",e];0b}[h]]; - if[ok;.z.m.subscribedhandles:distinct subscribedhandles,h]; +/ store one or more incoming heartbeats, keeping the latest per process and clearing warning / error state +storeheartbeat:{[batch] + / call this from upd when a heartbeat arrives + .z.m.hb:hb upsert update warning:0b,error:0b from select by sym,procname from batch; }; -subscribe:{[handles] - / subscribe to heartbeats on the given remote handle(s), tracking successful subscriptions - subscribeone each (),handles; +/ seed the store with expected processes so a never-seen process is flagged +addprocs:{[proctypes;procnames] + / real heartbeats arriving later override these seeded rows + seed:2!([]sym:proctypes,();procname:procnames,();time:cp[];counter:0N;pid:0Ni;host:`;port:0Ni;warning:0b;error:0b); + .z.m.hb:seed,hb; }; -getheartbeats:{[proctype] - / subscribe to publishers of the given process type(s) that are not yet subscribed - handles:(.z.m.serversgetservers proctype) except subscribedhandles; - if[count handles; - .z.m.log[`info][`heartbeat;"subscribing to new heartbeat handle(s) ",", " sv string handles]; - subscribe handles]; +/ subscribe to heartbeats on the given remote handle(s), tracking successful subscriptions +subscribe:{[handles] + subscribeone each (),handles; }; -hbsubscriptions:{ - / subscribe to all configured heartbeat publishers (by configured process type) - getheartbeats connections; - }; +/ return the current heartbeat store for inspection +gethb:{hb}; -closeconnection:{[h] - / drop a closed handle from the tracked subscriptions - registered against .z.pc - .z.m.subscribedhandles:subscribedhandles except h; - }; +/ replace the current-time function (used by tests and simulation) +setcp:{[f] .z.m.cp:f}; -gethb:{ - / return the current heartbeat store for inspection - hb +init:{[deps] + / initialise from a single dictionary holding config overrides and injected dependencies - see heartbeat.md + / config keys (see heartbeat.md) are optional and fall back to defaults; dependencies are required: + / `log - a logger - required; must provide info/warn/error (a kx.log instance is auto-wrapped) + / `timer - `addjob`deletejobs (full di.timer dict may be passed) - required + / `pubsub - `publish`subscribe - required + / `servers - `getservers - required when subenabled + / `handlers - `register`remove`list - required when subenabled + / note: the module keeps its own clock (cp, default .z.p) - override via setcp + / example: + / heartbeat.init[`proctype`procname`log`timer`pubsub!(`rdb;`rdb1;kxlog;timerdep;psdep)] + setconfig deps; + setdeps deps; + registertimers[]; + registerhandlers[]; + .z.m.log[`info][`heartbeat;"di.heartbeat initialised"]; }; From dccdef7ba22d96b4f4b97d763fe7973cf6d4569e Mon Sep 17 00:00:00 2001 From: Olly99999 Date: Tue, 30 Jun 2026 16:38:02 +0100 Subject: [PATCH 16/16] Blank commit --- di/heartbeat/heartbeat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/di/heartbeat/heartbeat.md b/di/heartbeat/heartbeat.md index 9e8803a1..f578f826 100644 --- a/di/heartbeat/heartbeat.md +++ b/di/heartbeat/heartbeat.md @@ -2,7 +2,7 @@ Periodic process-liveness signalling over pub/sub for kdb-x. Every process can publish a regular heartbeat so that downstream monitors detect when a process has stopped beating - i.e. it is stalled or blocked - even when the underlying connection is still valid. The module covers both sides: publishing heartbeats, and on the monitoring side storing received beats and raising a warning then an error when a process stops within configurable grace periods. ---- +--- ## Features