From 9b092f9f643f0d9ab35be7dcb5d8d7f75be08d40 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 19:47:48 -0700 Subject: [PATCH 1/6] chore(deps): bump datafusion to 55 and arrow to 59 --- Cargo.lock | 498 ++++++++++-------- Cargo.toml | 24 +- src/core/src/cache/policies/squeeze.rs | 6 +- src/datafusion-client/src/client_exec.rs | 11 +- src/datafusion-client/src/lib.rs | 3 +- src/datafusion-client/src/optimizer.rs | 16 +- src/datafusion-local/src/lib.rs | 3 +- .../src/admin_server/handlers.rs | 26 +- src/datafusion-server/src/lib.rs | 3 +- src/datafusion/src/optimizers/mod.rs | 7 +- src/datafusion/src/reader/plantime/opener.rs | 23 +- src/datafusion/src/reader/plantime/source.rs | 42 +- 12 files changed, 362 insertions(+), 300 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89e315fa8..62643e0cf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -167,9 +167,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471" +checksum = "61d285d16bce7d0be61912f7928342b673067b6b7d7ef6cc179258ba7de1fecf" dependencies = [ "arrow-arith", "arrow-array", @@ -188,9 +188,9 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738" +checksum = "757ef1836251e88222542a7da2623bc1c9cb9e20afefa6db2c41e79991cd91d4" dependencies = [ "arrow-array", "arrow-buffer", @@ -202,9 +202,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e" +checksum = "bc9a4a4b2b5ecd0e04df03471661cb61f28bed3c7fd50994715129b01b2edb97" dependencies = [ "ahash", "arrow-buffer", @@ -221,21 +221,21 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0" +checksum = "c12b576ef18c1deb80925a248b25ad84f419198d791b8e293fc6aaa60441fe90" dependencies = [ "bytes", "half", - "num-bigint", + "num-bigint 0.5.1", "num-traits", ] [[package]] name = "arrow-cast" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f" +checksum = "68338a9096a5dc9bc11927c58c43a8526d96bf6abd2012ef6c0c9f505991cc79" dependencies = [ "arrow-array", "arrow-buffer", @@ -244,7 +244,7 @@ dependencies = [ "arrow-schema", "arrow-select", "atoi", - "base64 0.22.1", + "base64 0.23.1", "chrono", "comfy-table", "half", @@ -255,9 +255,9 @@ dependencies = [ [[package]] name = "arrow-csv" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de" +checksum = "25011b52b346407d497ef0030e12b45e4f2d0cc279efc09c4f3d09106db30e36" dependencies = [ "arrow-array", "arrow-cast", @@ -270,9 +270,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0" +checksum = "723fe4aeed7604e00b9883a465af4ff0a0e6c44c03e41a68c3d1cbc403e0e44d" dependencies = [ "arrow-buffer", "arrow-schema", @@ -283,9 +283,9 @@ dependencies = [ [[package]] name = "arrow-flight" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28abfe8bf9f124e5fc83b334af4fa58f8d0323ad25312ccb2d1da50178415704" +checksum = "2bebfacc9d71f0728f6774164e4d4254b5e504d2b46812d0512d8290ec119a64" dependencies = [ "arrow-arith", "arrow-array", @@ -298,11 +298,10 @@ dependencies = [ "arrow-schema", "arrow-select", "arrow-string", - "base64 0.22.1", + "base64 0.23.1", "bytes", "futures", "once_cell", - "paste", "prost", "prost-types", "tonic", @@ -311,9 +310,9 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f" +checksum = "149437b14371f5b9ec60f5ddc751483ae99d7a7072653c0075e5e469156eea7b" dependencies = [ "arrow-array", "arrow-buffer", @@ -327,9 +326,9 @@ dependencies = [ [[package]] name = "arrow-json" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b" +checksum = "f18b9123ccfec418a663f821c9a034af339711678c11ffe00d3ec07da5ff9f7e" dependencies = [ "arrow-array", "arrow-buffer", @@ -352,9 +351,9 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0" +checksum = "e6c08dff0686cf23ca4f562803f191ccbeb726dbae6309cd4b4aaf65e0f2c979" dependencies = [ "arrow-array", "arrow-buffer", @@ -365,9 +364,9 @@ dependencies = [ [[package]] name = "arrow-row" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c" +checksum = "bbec439386df71ad570e6758a946111322b9e9dc8db83b5527321f0b4c9119c2" dependencies = [ "arrow-array", "arrow-buffer", @@ -378,9 +377,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed" +checksum = "e6fed2ca0d1eade57e811cbe73b98ad50cc08a1183e13b2d2aa43a7df593f40e" dependencies = [ "serde", "serde_core", @@ -389,9 +388,9 @@ dependencies = [ [[package]] name = "arrow-select" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222" +checksum = "466b19cf75130b891dc1b23a84b343c714c62c64c9c62e365c76aa0ff90a53fb" dependencies = [ "ahash", "arrow-array", @@ -403,9 +402,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42" +checksum = "c838a25bb3691e919e0f617616ac51a4ff8517a952e29ca133cf0c22b2ce65b1" dependencies = [ "arrow-array", "arrow-buffer", @@ -461,7 +460,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -472,7 +471,7 @@ checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -600,7 +599,7 @@ checksum = "7aa268c23bfbbd2c4363b9cd302a4f504fb2a9dfe7e3451d66f35dd392e20aca" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -636,6 +635,12 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + [[package]] name = "bigdecimal" version = "0.4.10" @@ -644,7 +649,7 @@ checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" dependencies = [ "autocfg", "libm", - "num-bigint", + "num-bigint 0.4.6", "num-integer", "num-traits", ] @@ -925,7 +930,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1061,7 +1066,7 @@ checksum = "4f160aad86b4343e8d4e261fee9965c3005b2fd6bc117d172ab65948779e4acf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1072,7 +1077,7 @@ checksum = "42571ed01eb46d2e1adcf99c8ca576f081e46f2623d13500eba70d1d99a4c439" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1347,7 +1352,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -1358,14 +1363,14 @@ checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", - "syn", + "syn 2.0.117", ] [[package]] name = "dashmap" -version = "6.1.0" +version = "6.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" dependencies = [ "cfg-if", "crossbeam-utils", @@ -1383,9 +1388,9 @@ checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8" [[package]] name = "datafusion" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "997a31e15872606a49478e670c58302094c97cb96abb0a7d60720f8e92170040" +checksum = "96f76f0167ed0842b29a3d1e41be3c034c0a46409a3a703cc4cc84ee8c24abf4" dependencies = [ "arrow", "arrow-schema", @@ -1420,7 +1425,7 @@ dependencies = [ "flate2", "futures", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1436,9 +1441,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dd61161508f8f5fa1107774ea687bd753c22d83a32eebf963549f89de14139" +checksum = "d79ec3460f6ed5c58f9b3f2d873fbc77748b82653bff1b4cdaf06de33bb4e05f" dependencies = [ "arrow", "async-trait", @@ -1452,7 +1457,7 @@ dependencies = [ "datafusion-physical-plan", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1461,9 +1466,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "897c70f871277f9ce99aa38347be0d679bbe3e617156c4d2a8378cec8a2a0891" +checksum = "b48cef241e2efcfd496fe05ae4d0d5de20793451862faefe406c397a467e12d4" dependencies = [ "arrow", "async-trait", @@ -1477,16 +1482,17 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", + "percent-encoding", ] [[package]] name = "datafusion-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c9ded5d87d9172319e006f2afdb9928d72dbacd6a90a458d8acb1e3b43a65" +checksum = "3f72810485975c258f1b4d00baab31728470676c60c5546f366ebd0d99f05ab6" dependencies = [ "arrow", "arrow-ipc", @@ -1496,9 +1502,10 @@ dependencies = [ "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "libc", "log", + "num-traits", "object_store", "parquet", "recursive", @@ -1510,9 +1517,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "981b9dae74f78ee3d9f714fb49b01919eab975461b56149510c3ba9ea11287d1" +checksum = "533c28e75dba52f41bde187d23a1cb24ab91c7c097966824fa471e67b60320ea" dependencies = [ "futures", "log", @@ -1521,9 +1528,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd7d295b2ec7c00d8a56562f41ed41062cf0af75549ed891c12a0a09eddfefe" +checksum = "5b00a1fa0da26f6087136a82fea7f13c76a672cbab452d4086952a7cf770a19b" dependencies = [ "arrow", "async-compression", @@ -1539,11 +1546,12 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "flate2", "futures", "glob", - "itertools", + "itertools 0.15.0", "liblzma", "log", "object_store", @@ -1557,9 +1565,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "552b0b3f342f7ec41b3fbd70f6339dc82a30cfd0349e7f280e7852528085349f" +checksum = "5ad17ec881bff2ed7768b4bfe971d3efbf3473f2fd1f9d365447bccbdf908678" dependencies = [ "arrow", "arrow-ipc", @@ -1572,18 +1580,19 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "object_store", "tokio", ] [[package]] name = "datafusion-datasource-csv" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68850aa426b897e879c8b87e512ea8124f1d0a2869a4e51808ddaaddf1bc0ada" +checksum = "b5345285b0c3eaab412e7539b706973c083bd7e5bce575de5e0a3da488d08d1d" dependencies = [ "arrow", "async-trait", @@ -1595,6 +1604,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1604,9 +1614,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "402f93242ae08ef99139ee2c528a49d087efe88d5c7b2c3ff5480855a40ce54f" +checksum = "da02fb9324f56bd8c53f1ee2e949547425cb66f76adc6832b10d44f80a1221d2" dependencies = [ "arrow", "async-trait", @@ -1618,6 +1628,7 @@ dependencies = [ "datafusion-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-session", "futures", "object_store", @@ -1627,11 +1638,12 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffd2499c1bee0eeccf6a57156105700eeeb17bc701899ac719183c4e74231450" +checksum = "3c0b0dc1453952952fd5c69ad1c7f6042176e69ed233011d47e07cf74ed0949e" dependencies = [ "arrow", + "arrow-schema", "async-trait", "bytes", "datafusion-common", @@ -1645,10 +1657,11 @@ dependencies = [ "datafusion-physical-expr-adapter", "datafusion-physical-expr-common", "datafusion-physical-plan", + "datafusion-proto-models", "datafusion-pruning", "datafusion-session", "futures", - "itertools", + "itertools 0.15.0", "log", "object_store", "parking_lot", @@ -1658,19 +1671,20 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb9e7e5d11130c48c8bd4e80c79a9772dd28ce6dc330baca9246205d245b9e2e" +checksum = "a88fd985bc0550c36f557db69543cc9d6393b1509783520b30e902f23c555da6" [[package]] name = "datafusion-execution" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37a8643ab852eb68864e1b72ae789e8066282dce48eea6347ffb0aee33d1ccc0" +checksum = "a98f1052f91b4991f0bf2ce1e4e36dfbdcda454a956b8c8d562c7c845e8fce1d" dependencies = [ "arrow", "arrow-buffer", "async-trait", + "bytes", "dashmap", "datafusion-common", "datafusion-expr", @@ -1679,16 +1693,19 @@ dependencies = [ "log", "object_store", "parking_lot", + "pin-project-lite", "rand 0.9.4", "tempfile", + "tokio", + "tokio-util", "url", ] [[package]] name = "datafusion-expr" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6932f4d71eed9c8d9341476a2b845aadfabde5495d08dbcd8fc23881f49fa7a0" +checksum = "464625a1f0e4b9df552d894fafcc8aac953ebbc8b0fa0acdaf20975fd615040e" dependencies = [ "arrow", "arrow-schema", @@ -1700,8 +1717,10 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "recursive", "serde_json", "sqlparser", @@ -1709,25 +1728,25 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0225491839a31b1f7d2cb8092c2d50792e2fe1c1724e4e6d08e011f5feaf4ed2" +checksum = "2604994999d5aeca1d1df645ffc98bc787447aaff05dde27aad0342b48fc1fe0" dependencies = [ "arrow", "datafusion-common", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-functions" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14872c47bfc3d21e53ec82f57074e6987a15941c1e2f43cde4ac6ae2746634e3" +checksum = "051e97533e6af53e4aa0a0667cadc886abcaf36c4a5925019c55c0aa4c218fde" dependencies = [ "arrow", "arrow-buffer", - "base64 0.22.1", + "base64 0.23.1", "blake2", "blake3", "chrono", @@ -1740,7 +1759,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hex", - "itertools", + "itertools 0.15.0", "log", "md-5 0.11.0", "memchr", @@ -1753,9 +1772,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75a2ca14e1b609be21e657e2d3130b2f446456b08393b377bb721a33952d2e09" +checksum = "2d0f1bb166d3572b6ed40e1afb2faaacade962abc08c2fcf04babee74681c56b" dependencies = [ "arrow", "datafusion-common", @@ -1766,17 +1785,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", - "foldhash 0.2.0", "half", + "hashbrown 0.17.1", "log", "num-traits", ] [[package]] name = "datafusion-functions-aggregate-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ece74ba09092d2ef9c9b54a38445450aea292a1f8b04faf531936b723a24b3c" +checksum = "7ed756770f5f98369e181d692fd5ee6b1127ffd7322caba92f3730f9f5c92333" dependencies = [ "arrow", "datafusion-common", @@ -1786,9 +1805,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f3e3f9ee8ca59bf70518802107de6f1b88a9509efdc629fadc5de9d6b2d5ef5" +checksum = "91173fdb5c0ff2a41169a8ffa1b385b8844f18728747bb0a37e35ad7d5772a4f" dependencies = [ "arrow", "arrow-ord", @@ -1803,7 +1822,7 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr-common", "hashbrown 0.17.1", - "itertools", + "itertools 0.15.0", "itoa", "log", "memchr", @@ -1811,9 +1830,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89161dffc22cf2b50f9f4b1bee83b5221d3b4ed7c2e37fd7aa2b22a5297b3a26" +checksum = "b1bcdfb286a745461b126719c32700777e83df4f17cc44db5d71ebce5731e840" dependencies = [ "arrow", "async-trait", @@ -1827,9 +1846,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7339345b226b3874037708bf5023ba1c2de705128f8457a095aae5ae9cb9c78" +checksum = "9ec4b508f1f93f00038ba3e737e894ec6c775528b4369413386655ae6125f0fc" dependencies = [ "arrow", "datafusion-common", @@ -1844,9 +1863,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa84836dc2392df6f43d6a29d37fb56a8ebdc8b3f4e10ae8dc15861fd20278fb" +checksum = "0b352020834140073fbf5b46ee0ceb926e5074a9d0bcae1dbd91d0586d999cde" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -1854,20 +1873,20 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "587164e03ad68732aa9e7bfe5686e3f25970d4c64fd4bd80790749840892dae5" +checksum = "15192effab05d38cce10e92a6fb48c967b5f166b27b7195a165a72b232569c58" dependencies = [ "datafusion-doc", "quote", - "syn", + "syn 3.0.4", ] [[package]] name = "datafusion-optimizer" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77f20e8cf9e8654d92f4c16b24c487353ee5bf153ffc12d5772cd399ab8cd281" +checksum = "854445d9f7847e1e46089cf61b8d341a64382f14484e912c83a0f23b31216896" dependencies = [ "arrow", "chrono", @@ -1876,7 +1895,7 @@ dependencies = [ "datafusion-expr-common", "datafusion-physical-expr", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "recursive", "regex", @@ -1885,9 +1904,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f015a4a82f6f7ff7e1d8d4bf3870a936752fa38b17705dfcc14adef95aa8922c" +checksum = "671558dad1d2aa253c39c0a4c52515958b99eb91abf649f4b88d5e69cc55282f" dependencies = [ "arrow", "datafusion-common", @@ -1895,10 +1914,11 @@ dependencies = [ "datafusion-expr-common", "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", + "datafusion-proto-models", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", "petgraph", "recursive", @@ -1907,9 +1927,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "51e6ffff8acdfe54e0ea15ccf38115c4a9184433b0439f42907637928d00a235" +checksum = "ffae3d78c2da80ecc829cb58536cc5aca2e99cf1365eda694fc75bfe288861e0" dependencies = [ "arrow", "datafusion-common", @@ -1917,31 +1937,32 @@ dependencies = [ "datafusion-functions", "datafusion-physical-expr", "datafusion-physical-expr-common", - "itertools", + "itertools 0.15.0", ] [[package]] name = "datafusion-physical-expr-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7967a3e171c6a4bf09474b3f7a14f1a3db13ed1714ba12156f33fcce2bba54e8" +checksum = "3d9092ed15e7203fbd0903215172f7c9d18f10d94cba35137f3b3836f7c46f16" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", + "datafusion-proto-models", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "parking_lot", "pin-project", ] [[package]] name = "datafusion-physical-optimizer" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ff803e2a96054cb6d83f35f9e60fd4f42eac515e1932bd1b2dbc91d5fcbf36" +checksum = "9005b6cf50b57b72d476c6ed4662b04be7ca6be5320ba9127c6d0b7e4218095b" dependencies = [ "arrow", "datafusion-common", @@ -1952,15 +1973,16 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", - "itertools", + "datafusion-session", + "itertools 0.15.0", "recursive", ] [[package]] name = "datafusion-physical-plan" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "776ee54d47d15bdb126452f9ca17b03761e3b004682914beaedd3f86eb507fbc" +checksum = "5787e4fcff4adc4fce8948441103a99705018b49c8dff0720b650bd7a15da112" dependencies = [ "arrow", "arrow-data", @@ -1968,6 +1990,7 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", + "bytes", "datafusion-common", "datafusion-common-runtime", "datafusion-execution", @@ -1977,26 +2000,28 @@ dependencies = [ "datafusion-functions-window-common", "datafusion-physical-expr", "datafusion-physical-expr-common", + "datafusion-proto-common", + "datafusion-proto-models", "futures", "half", "hashbrown 0.17.1", "indexmap 2.14.0", - "itertools", + "itertools 0.15.0", "log", "num-traits", "parking_lot", "pin-project-lite", + "serde_json", "tokio", ] [[package]] name = "datafusion-proto" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9dd15a1ba5d3af93808241065c6c44dbca8296a189845e8a587c45c07bf0ffae" +checksum = "0df0504eb9028d5e01f481af3519cde3f97ab650fd50ce7b787e94b69a7a193c" dependencies = [ "arrow", - "chrono", "datafusion-catalog", "datafusion-catalog-listing", "datafusion-common", @@ -2012,26 +2037,38 @@ dependencies = [ "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-proto-common", + "datafusion-proto-models", "object_store", "prost", ] [[package]] name = "datafusion-proto-common" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90042982cf9462eb06a0b81f92efa4188dae871e7ea3ab8dc61aa9c9349b2530" +checksum = "6b92415a2442964f180d39cdcd8ff1edd99f9260c1499ba80a396499c2154d11" dependencies = [ "arrow", "datafusion-common", "prost", ] +[[package]] +name = "datafusion-proto-models" +version = "55.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62e4c0bd6af4fcabdbe201ee86fbeef2ac423a44e1d8fda56d994c9e2d1d3ad2" +dependencies = [ + "datafusion-common", + "datafusion-proto-common", + "prost", +] + [[package]] name = "datafusion-pruning" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fb9e5774660aa69c3ba93c610f175f75b65cb8c3776edb3626de8f3a4f4ee3" +checksum = "9e651c8df0b90daed6a7be5921ec0ee379e6909705f063eeff70fd4e35010e4c" dependencies = [ "arrow", "datafusion-common", @@ -2045,10 +2082,11 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15ce715fa2a61f4623cc234bcc14a3ef6a91f189128d5b14b468a6a17cdfc417" +checksum = "fb56667ee38217efab19b895d9a936052cfb47ed438a19663351bdc42a6214a1" dependencies = [ + "arrow-schema", "async-trait", "datafusion-common", "datafusion-execution", @@ -2059,9 +2097,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "54.0.0" +version = "55.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6094ad36a3ed6d7ac87b20b479b2d0b118250f66cf997603829fdc65b44a7099" +checksum = "9c29067cb9d32f8e603c45e15d61ea18f1069f96ceafeceb4e18466b8e5b31d9" dependencies = [ "arrow", "bigdecimal", @@ -2074,6 +2112,7 @@ dependencies = [ "recursive", "regex", "sqlparser", + "stacker", ] [[package]] @@ -2102,7 +2141,7 @@ checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2124,7 +2163,7 @@ dependencies = [ "proc-macro2", "quote", "rustc_version", - "syn", + "syn 2.0.117", "unicode-xid", ] @@ -2268,7 +2307,7 @@ dependencies = [ "dioxus-rsx", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2430,7 +2469,7 @@ dependencies = [ "convert_case 0.8.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "xxhash-rust", ] @@ -2496,7 +2535,7 @@ dependencies = [ "convert_case 0.8.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2591,7 +2630,7 @@ dependencies = [ "quote", "sha2 0.10.9", "slab", - "syn", + "syn 2.0.117", ] [[package]] @@ -2604,7 +2643,7 @@ dependencies = [ "proc-macro2-diagnostics", "quote", "rustversion", - "syn", + "syn 2.0.117", ] [[package]] @@ -2714,7 +2753,7 @@ dependencies = [ "convert_case 0.8.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2759,7 +2798,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2784,7 +2823,7 @@ checksum = "9556bc800956545d6420a640173e5ba7dfa82f38d3ea5a167eb555bc69ac3323" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2866,7 +2905,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2886,7 +2925,7 @@ checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -2993,7 +3032,7 @@ dependencies = [ "proc-macro-error2", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3170,7 +3209,7 @@ checksum = "e835b70203e41293343137df5c0664546da5745f82ec9b84d40be8336958447b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3806,12 +3845,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "integer-encoding" -version = "3.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" - [[package]] name = "inventory" version = "0.3.24" @@ -3864,6 +3897,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -3893,7 +3935,7 @@ checksum = "e000de030ff8022ea1da3f466fbb0f3a809f5e51ed31f6dd931c35181ad8e6d7" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -3952,7 +3994,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4469,9 +4511,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db9a0d582c2874f68138a16ce1867e0ffde6c0bb0a0df85e1f36d04146db488a" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" dependencies = [ "twox-hash", ] @@ -4484,7 +4526,7 @@ checksum = "1b27834086c65ec3f9387b096d66e99f221cf081c2b738042aa252bcd41204e3" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4528,7 +4570,7 @@ dependencies = [ "manganis-core", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4568,9 +4610,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.8.0" +version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memfd" @@ -4747,6 +4789,16 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-bigint" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93e7820bc0a80a0238e650327316f929ba18d5be054b647490a3a6a339f3e7c0" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -4810,7 +4862,7 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -4874,7 +4926,7 @@ dependencies = [ "http-body-util", "humantime", "hyper", - "itertools", + "itertools 0.14.0", "md-5 0.10.6", "parking_lot", "percent-encoding", @@ -4986,15 +5038,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "ordered-float" -version = "2.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68f19d67e5a2795c94e73e0bb1cc1a7edeb2e28efd39e2e1c9b7a40c1108b11c" -dependencies = [ - "num-traits", -] - [[package]] name = "owo-colors" version = "3.5.0" @@ -5026,9 +5069,9 @@ dependencies = [ [[package]] name = "parquet" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908" +checksum = "7065842956a20c2a536924ce8e4d9955f7422451511b9eb7500d7bfe5077e59c" dependencies = [ "ahash", "arrow-array", @@ -5037,7 +5080,7 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64 0.22.1", + "base64 0.23.1", "brotli", "bytes", "chrono", @@ -5046,18 +5089,16 @@ dependencies = [ "half", "hashbrown 0.17.1", "lz4_flex", - "num-bigint", + "num-bigint 0.5.1", "num-integer", "num-traits", "object_store", "parquet-variant", "parquet-variant-compute", "parquet-variant-json", - "paste", "seq-macro", "simdutf8", "snap", - "thrift", "tokio", "twox-hash", "zstd", @@ -5065,9 +5106,9 @@ dependencies = [ [[package]] name = "parquet-variant" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74c8db065291f088a2aad8ab831853eae1871c0d311c8d0b83bbc3b7e735d0fc" +checksum = "3f7e5fff3ed0c07514a7fb8bee3f2ea5a53f36939410ecac4a466620213539a8" dependencies = [ "arrow", "arrow-schema", @@ -5081,9 +5122,9 @@ dependencies = [ [[package]] name = "parquet-variant-compute" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a530e8d5b5e14efcb39c9a6ec55432ad11f6afb7dc4455a79be0dc615fe3cc31" +checksum = "ba4d3de89dab8d1aaaf601ae8d71bd07ea88cfca9efc1df5815b982c30f631e1" dependencies = [ "arrow", "arrow-schema", @@ -5098,12 +5139,12 @@ dependencies = [ [[package]] name = "parquet-variant-json" -version = "58.3.0" +version = "59.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00ed89908289f67caa2ca078f9ff9aacd6229a313ec92b12bf4f48f613dc2b97" +checksum = "fb19dfe1bd24c17addd761ba4f7000f615e2fa12525871c7baa835dbb3d7f147" dependencies = [ "arrow-schema", - "base64 0.22.1", + "base64 0.23.1", "chrono", "parquet-variant", "serde_json", @@ -5187,7 +5228,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5247,7 +5288,7 @@ checksum = "d9b20ed30f105399776b9c883e68e536ef602a16ae6f596d2c473591d6ad64c6" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5348,7 +5389,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" dependencies = [ "proc-macro2", - "syn", + "syn 2.0.117", ] [[package]] @@ -5379,7 +5420,7 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5399,7 +5440,7 @@ checksum = "af066a9c399a26e020ada66a034357a868728e72cd426f3adcd35f80d88d88c8" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "version_check", ] @@ -5420,10 +5461,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5670,7 +5711,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76009fbe0614077fc1a2ce255e3a1881a2e3a3527097d5dc6d8212c585e7e38b" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -5970,7 +6011,7 @@ checksum = "ed76efe62313ab6610570951494bdaa81568026e0318eaa55f167de70eeea67d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6064,7 +6105,7 @@ checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6082,6 +6123,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ + "indexmap 2.14.0", "itoa", "memchr", "serde", @@ -6119,7 +6161,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6131,7 +6173,7 @@ dependencies = [ "proc-macro2", "quote", "serde", - "syn", + "syn 2.0.117", ] [[package]] @@ -6262,7 +6304,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb251b407f50028476a600541542b605bb864d35d9ee1de4f6cab45d88475e6d" dependencies = [ "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6346,7 +6388,7 @@ checksum = "a6dd45d8fc1c79299bfbb7190e42ccbbdf6a5f52e4a6ad98d92357ea965bd289" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6526,6 +6568,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -6543,7 +6596,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6661,7 +6714,7 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6672,7 +6725,7 @@ checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -6694,17 +6747,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "thrift" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" -dependencies = [ - "byteorder", - "integer-encoding", - "ordered-float", -] - [[package]] name = "time" version = "0.3.47" @@ -6793,7 +6835,7 @@ checksum = "385a6cb71ab9ab790c5fe8d67f1645e6c450a7ce006a33de03daa956cf70a496" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7016,7 +7058,7 @@ checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7232,7 +7274,7 @@ dependencies = [ "proc-macro2", "quote", "serde_tokenstream", - "syn", + "syn 2.0.117", "usdt-impl", ] @@ -7250,7 +7292,7 @@ dependencies = [ "quote", "serde", "serde_json", - "syn", + "syn 2.0.117", "thiserror 2.0.18", "thread-id", ] @@ -7265,7 +7307,7 @@ dependencies = [ "proc-macro2", "quote", "serde_tokenstream", - "syn", + "syn 2.0.117", "usdt-impl", ] @@ -7361,7 +7403,7 @@ checksum = "7ef946d78c84f284991d59035ca252f29cd8071526003e794be80f0beee18f2d" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", "verus_prettyplease", "verus_syn", @@ -7449,7 +7491,7 @@ checksum = "59195a1db0e95b920366d949ba5e0d3fc0e70b67c09be15ce5abb790106b0571" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7518,7 +7560,7 @@ dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wasm-bindgen-shared", ] @@ -7691,7 +7733,7 @@ checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7702,7 +7744,7 @@ checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -7970,7 +8012,7 @@ dependencies = [ "heck", "indexmap 2.14.0", "prettyplease", - "syn", + "syn 2.0.117", "wasm-metadata", "wit-bindgen-core", "wit-component", @@ -7986,7 +8028,7 @@ dependencies = [ "prettyplease", "proc-macro2", "quote", - "syn", + "syn 2.0.117", "wit-bindgen-core", "wit-bindgen-rust", ] @@ -8068,7 +8110,7 @@ checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -8089,7 +8131,7 @@ checksum = "70e3cd084b1788766f53af483dd21f93881ff30d7320490ec3ef7526d203bad4" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] @@ -8109,7 +8151,7 @@ checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", "synstructure", ] @@ -8149,7 +8191,7 @@ checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.117", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9fcc379c5..ade6052ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,25 +30,25 @@ liquid-cache-datafusion = { path = "src/datafusion", version = "0.1.13" } liquid-cache-common = { path = "src/common", version = "0.1.13" } liquid-cache = { path = "src/core", version = "0.1.13" } liquid-cache-datafusion-local = { path = "src/datafusion-local", version = "0.1.13" } -arrow = { version = "58.3.0", default-features = false, features = [ +arrow = { version = "59.2", default-features = false, features = [ "prettyprint", "ipc", ] } -arrow-flight = { version = "58.3.0", features = ["flight-sql-experimental"] } -arrow-schema = { version = "58.3.0", features = ["serde"] } -parquet = { version = "58.3.0", features = [ +arrow-flight = { version = "59.2", features = ["flight-sql-experimental"] } +arrow-schema = { version = "59.2", features = ["serde"] } +parquet = { version = "59.2", features = [ "async", "experimental", "variant_experimental", ] } -parquet-variant-json = { version = "58.3.0" } -parquet-variant-compute = { version = "58.3.0" } -datafusion = { version = "54.0.0" } -datafusion-common = { version = "54.0.0" } -datafusion-expr-common = { version = "54.0.0" } -datafusion-physical-expr = { version = "54.0.0" } -datafusion-physical-expr-common = { version = "54.0.0" } -datafusion-proto = { version = "54.0.0" } +parquet-variant-json = { version = "59.2" } +parquet-variant-compute = { version = "59.2" } +datafusion = { version = "55" } +datafusion-common = { version = "55" } +datafusion-expr-common = { version = "55" } +datafusion-physical-expr = { version = "55" } +datafusion-physical-expr-common = { version = "55" } +datafusion-proto = { version = "55" } async-trait = "0.1.89" futures = { version = "0.3.32", default-features = false, features = ["std"] } tokio = { version = "1.52.3", features = ["rt-multi-thread"] } diff --git a/src/core/src/cache/policies/squeeze.rs b/src/core/src/cache/policies/squeeze.rs index 53f489135..fd9a2e511 100644 --- a/src/core/src/cache/policies/squeeze.rs +++ b/src/core/src/cache/policies/squeeze.rs @@ -244,7 +244,7 @@ pub(crate) fn try_variant_squeeze( shredded_array = Some(shredded_struct); } - let typed_root = variant_array.typed_value_field()?; + let typed_root = variant_array.typed_value_column()?; let typed_root = typed_root.as_any().downcast_ref::()?; let mut collected = Vec::new(); @@ -676,7 +676,9 @@ mod tests { inner .column_by_name("metadata") .cloned() - .unwrap_or_else(|| Arc::new(base_variant.metadata_field().clone()) as ArrayRef), + .unwrap_or_else(|| { + Arc::new(base_variant.metadata_column().clone()) as ArrayRef + }), inner.column_by_name("value").cloned().unwrap_or_else(|| { Arc::new(BinaryViewArray::from(vec![None::<&[u8]>; inner.len()])) as ArrayRef }), diff --git a/src/datafusion-client/src/client_exec.rs b/src/datafusion-client/src/client_exec.rs index 658c86dc1..76b63c441 100644 --- a/src/datafusion-client/src/client_exec.rs +++ b/src/datafusion-client/src/client_exec.rs @@ -10,7 +10,7 @@ use arrow_flight::flight_service_client::FlightServiceClient; use arrow_schema::{Schema, SchemaRef}; use datafusion::catalog::memory::DataSourceExec; use datafusion::common::internal_err; -use datafusion::common::tree_node::{Transformed, TreeNode}; +use datafusion::common::tree_node::{Transformed, TreeNode, TreeNodeRecursion}; use datafusion::config::ConfigOptions; use datafusion::datasource::physical_plan::{FileSource, ParquetSource}; use datafusion::execution::object_store::ObjectStoreUrl; @@ -140,6 +140,15 @@ impl ExecutionPlan for LiquidCacheClientExec { vec![&self.remote_plan] } + /// The client node holds no expressions of its own; the wrapped remote plan + /// is visited as a child. + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn with_new_children( self: Arc, children: Vec>, diff --git a/src/datafusion-client/src/lib.rs b/src/datafusion-client/src/lib.rs index 079c7611a..02d8aef9b 100644 --- a/src/datafusion-client/src/lib.rs +++ b/src/datafusion-client/src/lib.rs @@ -9,6 +9,7 @@ mod metrics; mod optimizer; pub use client_exec::LiquidCacheClientExec; use datafusion::{ + common::config::ConfigNonZeroUsize, error::{DataFusionError, Result}, execution::{SessionStateBuilder, object_store::ObjectStoreUrl, runtime_env::RuntimeEnv}, prelude::*, @@ -91,7 +92,7 @@ impl LiquidCacheClientBuilder { .execution .parquet .binary_as_string = true; - session_config.options_mut().execution.batch_size = 8192 * 2; + session_config.options_mut().execution.batch_size = ConfigNonZeroUsize::try_new(8192 * 2)?; // Dynamic filters (e.g. a hash join's runtime build-side filter) are pushed // into scan predicates by DataFusion. In distributed mode those scans are // serialized and executed on a remote server that can never receive the diff --git a/src/datafusion-client/src/optimizer.rs b/src/datafusion-client/src/optimizer.rs index 228e75270..a7a406427 100644 --- a/src/datafusion-client/src/optimizer.rs +++ b/src/datafusion-client/src/optimizer.rs @@ -5,6 +5,7 @@ use datafusion::{ execution::object_store::ObjectStoreUrl, physical_optimizer::PhysicalOptimizerRule, physical_plan::ExecutionPlan, physical_plan::aggregates::AggregateExec, physical_plan::aggregates::AggregateMode, physical_plan::repartition::RepartitionExec, + physical_plan::replace_children_if_necessary, }; use liquid_cache_datafusion::optimizers::SqueezeHintMap; @@ -69,22 +70,13 @@ impl PushdownOptimizer { // Otherwise, recurse into children let mut new_children = Vec::with_capacity(plan.children().len()); - let mut children_changed = false; for child in plan.children() { - let new_child = self.optimize_plan(child.clone(), hints)?; - if !Arc::ptr_eq(child, &new_child) { - children_changed = true; - } - new_children.push(new_child); + new_children.push(self.optimize_plan(child.clone(), hints)?); } - // If any children were changed, create a new plan with the updated children - if children_changed { - plan.with_new_children(new_children) - } else { - Ok(plan) - } + // Returns the plan untouched when every child is unchanged. + replace_children_if_necessary(plan, new_children) } } diff --git a/src/datafusion-local/src/lib.rs b/src/datafusion-local/src/lib.rs index 82cc7ad7b..27254ea73 100644 --- a/src/datafusion-local/src/lib.rs +++ b/src/datafusion-local/src/lib.rs @@ -7,6 +7,7 @@ mod tests; use std::path::PathBuf; use std::sync::Arc; +use datafusion::common::config::ConfigNonZeroUsize; use datafusion::error::Result; use datafusion::logical_expr::ScalarUDF; use datafusion::prelude::{SessionConfig, SessionContext}; @@ -183,7 +184,7 @@ impl LiquidCacheLocalBuilder { .schema_force_view_types = false; config.options_mut().execution.parquet.skip_arrow_metadata = false; config.options_mut().execution.parquet.skip_metadata = false; - config.options_mut().execution.batch_size = self.batch_size; + config.options_mut().execution.batch_size = ConfigNonZeroUsize::try_new(self.batch_size)?; let store = liquid_cache::store::mount(self.cache_dir.join("liquid_cache.t4")) .await diff --git a/src/datafusion-server/src/admin_server/handlers.rs b/src/datafusion-server/src/admin_server/handlers.rs index 8fcdf9c5b..1be5b0be7 100644 --- a/src/datafusion-server/src/admin_server/handlers.rs +++ b/src/datafusion-server/src/admin_server/handlers.rs @@ -15,7 +15,7 @@ use datafusion::{ tree_node::{TreeNode, TreeNodeRecursion}, }, datasource::physical_plan::FileScanConfig, - physical_plan::ExecutionPlan, + physical_plan::{ExecutionPlan, StatisticsArgs, StatisticsContext}, }; use liquid_cache_common::rpc::ExecutionMetricsResponse; use liquid_cache_datafusion::LiquidParquetSource; @@ -323,14 +323,12 @@ impl From<&Arc> for ExecutionPlanWithStats { }); } + let stats = StatisticsContext::new() + .compute(plan.as_ref(), &StatisticsArgs::new()) + .unwrap(); + let mut column_statistics = Vec::new(); - for (i, cs) in plan - .partition_statistics(None) - .unwrap() - .column_statistics - .iter() - .enumerate() - { + for (i, cs) in stats.column_statistics.iter().enumerate() { let min = if cs.min_value != Precision::Absent { Some(cs.min_value.to_string()) } else { @@ -378,16 +376,8 @@ impl From<&Arc> for ExecutionPlanWithStats { }) .collect(), statistics: Statistics { - num_rows: plan - .partition_statistics(None) - .unwrap() - .num_rows - .to_string(), - total_byte_size: plan - .partition_statistics(None) - .unwrap() - .total_byte_size - .to_string(), + num_rows: stats.num_rows.to_string(), + total_byte_size: stats.total_byte_size.to_string(), column_statistics, }, metrics: metric_values, diff --git a/src/datafusion-server/src/lib.rs b/src/datafusion-server/src/lib.rs index 4da84881f..a12b6dd4d 100644 --- a/src/datafusion-server/src/lib.rs +++ b/src/datafusion-server/src/lib.rs @@ -28,6 +28,7 @@ use arrow_flight::{ }, }; use datafusion::{ + common::config::ConfigNonZeroUsize, error::DataFusionError, execution::{SessionStateBuilder, object_store::ObjectStoreUrl}, prelude::{SessionConfig, SessionContext}, @@ -167,7 +168,7 @@ impl LiquidCacheService { let mut session_config = SessionConfig::from_env()?; let options_mut = session_config.options_mut(); options_mut.execution.parquet.pushdown_filters = true; - options_mut.execution.batch_size = 8192 * 2; + options_mut.execution.batch_size = ConfigNonZeroUsize::try_new(8192 * 2)?; { // view types cause excessive memory usage because they are not gced. diff --git a/src/datafusion/src/optimizers/mod.rs b/src/datafusion/src/optimizers/mod.rs index daf21fa65..00113a59c 100644 --- a/src/datafusion/src/optimizers/mod.rs +++ b/src/datafusion/src/optimizers/mod.rs @@ -21,7 +21,7 @@ use datafusion::{ source::DataSource, }, physical_expr::utils::collect_columns, - physical_optimizer::{PhysicalOptimizerRule, pruning::PruningPredicate}, + physical_optimizer::{PhysicalOptimizerRule, pruning::PruningPredicateBuilder}, physical_plan::ExecutionPlan, }; @@ -486,7 +486,10 @@ fn surviving_files( let Some(pred) = src.filter() else { return vec![true; files.len()]; }; - let pruning = match PruningPredicate::try_new(pred, table_schema.clone()) { + let pruning = match PruningPredicateBuilder::new() + .with_file_schema(table_schema.clone()) + .try_build(pred) + { Ok(p) => p, Err(_) => return vec![true; files.len()], }; diff --git a/src/datafusion/src/reader/plantime/opener.rs b/src/datafusion/src/reader/plantime/opener.rs index f1c2804e6..386e35735 100644 --- a/src/datafusion/src/reader/plantime/opener.rs +++ b/src/datafusion/src/reader/plantime/opener.rs @@ -24,7 +24,6 @@ use datafusion::{ physical_expr::projection::ProjectionExprs, physical_expr::utils::reassign_expr_columns, physical_expr_adapter::{PhysicalExprAdapterFactory, replace_columns_with_literals}, - physical_expr_common::physical_expr::is_dynamic_physical_expr, physical_optimizer::pruning::{FilePruner, PruningPredicate, build_pruning_predicate}, physical_plan::{ PhysicalExpr, @@ -151,17 +150,17 @@ impl FileOpener for LiquidParquetOpener { // We'll also check this after every record batch we read, // and if at some point we are able to prove we can prune the file using just the file level statistics // we can end the stream early. - let mut file_pruner = predicate - .as_ref() - .filter(|p| is_dynamic_physical_expr(p) || partitioned_file.has_statistics()) - .and_then(|p| { - FilePruner::try_new( - Arc::clone(p), - &logical_file_schema, - &partitioned_file, - predicate_creation_errors.clone(), - ) - }); + // `FilePruner::try_new` itself decides whether a pruner is worth + // building: it returns `None` for a purely static predicate over a + // file with no usable column statistics. + let mut file_pruner = predicate.as_ref().and_then(|p| { + FilePruner::try_new( + Arc::clone(p), + &logical_file_schema, + &partitioned_file, + predicate_creation_errors.clone(), + ) + }); if let Some(file_pruner) = &mut file_pruner && file_pruner.should_prune()? diff --git a/src/datafusion/src/reader/plantime/source.rs b/src/datafusion/src/reader/plantime/source.rs index 356072430..b9af3c9e2 100644 --- a/src/datafusion/src/reader/plantime/source.rs +++ b/src/datafusion/src/reader/plantime/source.rs @@ -4,6 +4,7 @@ use ahash::{HashMap, HashMapExt}; use arrow_schema::Schema; use bytes::Bytes; use datafusion::{ + common::tree_node::TreeNodeRecursion, config::TableParquetOptions, datasource::{ listing::PartitionedFile, @@ -16,21 +17,25 @@ use datafusion::{ error::Result, physical_expr::projection::ProjectionExprs, physical_expr_adapter::DefaultPhysicalExprAdapterFactory, - physical_optimizer::pruning::PruningPredicate, + physical_optimizer::pruning::{PruningPredicate, PruningPredicateBuilder}, physical_plan::{ - PhysicalExpr, + PhysicalExpr, apply_expression_roots, metrics::{ExecutionPlanMetricsSet, MetricBuilder}, }, }; use futures::{FutureExt, future::BoxFuture}; use object_store::{ObjectStore, path::Path}; -use parquet::{ - arrow::{ - arrow_reader::ArrowReaderOptions, - async_reader::{AsyncFileReader, ParquetObjectReader}, - }, - file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader}, -}; +use parquet::arrow::arrow_reader::ArrowReaderOptions; +use parquet::arrow::async_reader::AsyncFileReader; +// `ParquetObjectReader` is deprecated in arrow-rs 59 in favour of implementing +// `AsyncFileReader` against the object store directly +// (https://github.com/apache/arrow-rs/issues/10308). We keep it for now: it is the +// only readily available reader that coalesces byte ranges, and DataFusion's +// `ParquetFileReader` — the suggested replacement — has a `pub(crate)` +// constructor and does no coalescing. +#[allow(deprecated)] +use parquet::arrow::async_reader::ParquetObjectReader; +use parquet::file::metadata::{PageIndexPolicy, ParquetMetaData, ParquetMetaDataReader}; use std::{ ops::Range, sync::{Arc, LazyLock}, @@ -58,6 +63,7 @@ impl CachedMetaReaderFactory { ) -> ParquetMetadataCacheReader { let path = partitioned_file.object_meta.location.clone(); let store = Arc::clone(&self.store); + #[allow(deprecated)] let mut inner = ParquetObjectReader::new(store, path.clone()) .with_file_size(partitioned_file.object_meta.size); @@ -106,6 +112,7 @@ impl MetadataCache { #[derive(Clone)] pub struct ParquetMetadataCacheReader { file_metrics: ParquetFileMetrics, + #[allow(deprecated)] inner: ParquetObjectReader, path: Path, } @@ -225,7 +232,10 @@ impl LiquidParquetSource { self.metrics = metrics; self.predicate = Some(Arc::clone(&predicate)); - match PruningPredicate::try_new(Arc::clone(&predicate), Arc::clone(&file_schema)) { + match PruningPredicateBuilder::new() + .with_file_schema(Arc::clone(&file_schema)) + .try_build(Arc::clone(&predicate)) + { Ok(pruning_predicate) => { if !pruning_predicate.always_true() { self.pruning_predicate = Some(Arc::new(pruning_predicate)); @@ -363,4 +373,16 @@ impl FileSource for LiquidParquetSource { fn file_type(&self) -> &str { "liquid_parquet" } + + fn apply_expressions( + &self, + f: &mut dyn FnMut(&Arc) -> Result, + ) -> Result { + apply_expression_roots( + self.predicate + .iter() + .chain(self.projection.iter().map(|proj_expr| &proj_expr.expr)), + f, + ) + } } From b7b4391bebec0c6e5383be51afa0aceb9a5d5130 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 19:47:48 -0700 Subject: [PATCH 2/6] test: keep cache-test scans single-partition --- src/datafusion-local/src/tests/mod.rs | 22 +++++++++++++++++++--- src/datafusion-local/src/tests/squeeze.rs | 11 +++++------ 2 files changed, 24 insertions(+), 9 deletions(-) diff --git a/src/datafusion-local/src/tests/mod.rs b/src/datafusion-local/src/tests/mod.rs index 2feff9141..697c2d83f 100644 --- a/src/datafusion-local/src/tests/mod.rs +++ b/src/datafusion-local/src/tests/mod.rs @@ -99,12 +99,28 @@ impl fmt::Display for CacheStatsSummary { } } +/// Session config for the tests that read [`TEST_FILE`] and pin cache traces, +/// entry counts or IO counts. +/// +/// DataFusion 55 lowered `repartition_file_min_size` from 10 MiB to 1 MiB, so the +/// 2.3 MB test file is now split into one scan partition per `target_partitions` +/// instead of being read by a single one. Several scan partitions hit the shared +/// cache concurrently, which makes admission and eviction order — and with it +/// every trace and byte count these tests assert — depend on scheduling and on +/// the host's core count. Raise the threshold back above the file size so the +/// scan stays single-partition and the snapshots stay reproducible. +pub(super) fn cache_test_config() -> SessionConfig { + let mut config = SessionConfig::new(); + config.options_mut().optimizer.repartition_file_min_size = 16 * 1024 * 1024; + config +} + async fn create_session_context_with_liquid_cache( squeeze_policy: Box, cache_size_bytes: usize, cache_dir: &Path, ) -> Result<(SessionContext, LiquidCacheParquetRef)> { - let mut config = SessionConfig::new(); + let mut config = cache_test_config(); config.options_mut().execution.target_partitions = 4; let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(cache_size_bytes) @@ -385,7 +401,7 @@ async fn test_provide_schema2() { let cache_dir = TempDir::new().unwrap(); let df_ctx = SessionContext::new(); - let mut config = SessionConfig::new(); + let mut config = cache_test_config(); config.options_mut().execution.target_partitions = 4; let (liquid_ctx, cache) = LiquidCacheLocalBuilder::new() .with_cache_dir(cache_dir.path().to_path_buf()) @@ -560,7 +576,7 @@ async fn test_provide_schema_with_filter() { let (ctx, _) = LiquidCacheLocalBuilder::new() .with_squeeze_policy(Box::new(TranscodeSqueezeEvict)) - .build(SessionConfig::new()) + .build(cache_test_config()) .await .unwrap(); diff --git a/src/datafusion-local/src/tests/squeeze.rs b/src/datafusion-local/src/tests/squeeze.rs index 89b2eba80..e32d21a38 100644 --- a/src/datafusion-local/src/tests/squeeze.rs +++ b/src/datafusion-local/src/tests/squeeze.rs @@ -1,5 +1,4 @@ use arrow::{array::AsArray, datatypes::Int64Type, util::pretty::pretty_format_batches}; -use datafusion::prelude::SessionConfig; use tempfile::TempDir; use crate::LiquidCacheLocalBuilder; @@ -12,7 +11,7 @@ async fn basic_squeeze() { let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(1024 * 128) .with_cache_dir(cache_dir.path().to_path_buf()) - .build(SessionConfig::new()) + .build(super::cache_test_config()) .await .unwrap(); ctx.register_parquet("hits", TEST_FILE, Default::default()) @@ -39,7 +38,7 @@ async fn squeeze_strings() { let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(1024 * 1024) .with_cache_dir(cache_dir.path().to_path_buf()) - .build(SessionConfig::new()) + .build(super::cache_test_config()) .await .unwrap(); ctx.register_parquet("hits", TEST_FILE, Default::default()) @@ -66,7 +65,7 @@ async fn squeeze_substrings_search() { let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(1024 * 256) .with_cache_dir(cache_dir.path().to_path_buf()) - .build(SessionConfig::new()) + .build(super::cache_test_config()) .await .unwrap(); ctx.register_parquet("hits", TEST_FILE, Default::default()) @@ -90,7 +89,7 @@ async fn squeeze_substrings_search_title() { let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(1024 * 1024 * 4) .with_cache_dir(cache_dir.path().to_path_buf()) - .build(SessionConfig::new()) + .build(super::cache_test_config()) .await .unwrap(); ctx.register_parquet("hits", TEST_FILE, Default::default()) @@ -115,7 +114,7 @@ async fn squeeze_distinct_search_phase() { let (ctx, cache) = LiquidCacheLocalBuilder::new() .with_max_memory_bytes(1024 * 256) .with_cache_dir(cache_dir.path().to_path_buf()) - .build(SessionConfig::new()) + .build(super::cache_test_config()) .await .unwrap(); ctx.register_parquet("hits", TEST_FILE, Default::default()) From 9338d47e59e09f13eaf1228ea1a0a9a485e2c8ae Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 19:47:48 -0700 Subject: [PATCH 3/6] test: refresh plan snapshots for datafusion 55 --- ...che_datafusion_client__tests__tpch_q1.snap | 48 +++-- ...he_datafusion_client__tests__tpch_q10.snap | 37 ++-- ...he_datafusion_client__tests__tpch_q11.snap | 18 +- ...he_datafusion_client__tests__tpch_q12.snap | 10 +- ...he_datafusion_client__tests__tpch_q13.snap | 12 +- ...he_datafusion_client__tests__tpch_q14.snap | 12 +- ...he_datafusion_client__tests__tpch_q16.snap | 31 ++-- ...he_datafusion_client__tests__tpch_q17.snap | 6 +- ...he_datafusion_client__tests__tpch_q18.snap | 2 +- ...he_datafusion_client__tests__tpch_q19.snap | 70 ++++---- ...che_datafusion_client__tests__tpch_q2.snap | 6 +- ...he_datafusion_client__tests__tpch_q20.snap | 6 +- ...he_datafusion_client__tests__tpch_q21.snap | 168 +++++++++--------- ...he_datafusion_client__tests__tpch_q22.snap | 24 +-- ...che_datafusion_client__tests__tpch_q3.snap | 41 +++-- ...che_datafusion_client__tests__tpch_q4.snap | 12 +- ...che_datafusion_client__tests__tpch_q5.snap | 33 ++-- ...che_datafusion_client__tests__tpch_q6.snap | 7 +- ...che_datafusion_client__tests__tpch_q7.snap | 20 +-- ...che_datafusion_client__tests__tpch_q8.snap | 28 ++- ...che_datafusion_client__tests__tpch_q9.snap | 16 +- ...al__tests__provide_schema_with_filter.snap | 7 + 22 files changed, 311 insertions(+), 303 deletions(-) diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q1.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q1.snap index 7590f16a0..f112cc27d 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q1.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q1.snap @@ -10,13 +10,6 @@ expression: displayable.tree_render().to_string() │ ASC NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ l_returnflag@0 ASC NULLS │ -│ LAST, l_linestatus@1 │ -│ ASC NULLS LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ avg_disc: │ @@ -51,6 +44,13 @@ expression: displayable.tree_render().to_string() │ ... │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ l_returnflag@0 ASC NULLS │ +│ LAST, l_linestatus@1 │ +│ ASC NULLS LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ @@ -62,14 +62,13 @@ expression: displayable.tree_render().to_string() │ .l_extendedprice * │ │ Int64(1) - lineitem │ │ .l_discount), sum │ -│ (__common_expr_1 * │ -│ Some(1),20,0 + │ -│ lineitem.l_tax) as │ -│ sum(lineitem │ -│ .l_extendedpri │ -│ ce * Int64(1) - lineitem │ -│ .l_discount * Int64(1) │ -│ + lineitem.l_tax), avg │ +│ (__common_expr_1 * 1 + │ +│ lineitem.l_tax) as sum │ +│ (lineitem │ +│ .l_extendedp │ +│ rice * Int64(1) - lineitem│ +│ .l_discount * Int64(1) + │ +│ lineitem.l_tax), avg │ │ (lineitem.l_quantity), │ │ avg(lineitem │ │ .l_extendedpri │ @@ -105,14 +104,13 @@ expression: displayable.tree_render().to_string() │ .l_extendedprice * │ │ Int64(1) - lineitem │ │ .l_discount), sum │ -│ (__common_expr_1 * │ -│ Some(1),20,0 + │ -│ lineitem.l_tax) as │ -│ sum(lineitem │ -│ .l_extendedpri │ -│ ce * Int64(1) - lineitem │ -│ .l_discount * Int64(1) │ -│ + lineitem.l_tax), avg │ +│ (__common_expr_1 * 1 + │ +│ lineitem.l_tax) as sum │ +│ (lineitem │ +│ .l_extendedp │ +│ rice * Int64(1) - lineitem│ +│ .l_discount * Int64(1) + │ +│ lineitem.l_tax), avg │ │ (lineitem.l_quantity), │ │ avg(lineitem │ │ .l_extendedpri │ @@ -129,8 +127,8 @@ expression: displayable.tree_render().to_string() │ ProjectionExec │ │ -------------------- │ │ __common_expr_1: │ -│ l_extendedprice * (Some(1)│ -│ ,20,0 - l_discount) │ +│ l_extendedprice * (1 - │ +│ l_discount) │ │ │ │ l_discount: │ │ l_discount │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q10.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q10.snap index 8643f5637..2f6a455e4 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q10.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q10.snap @@ -10,13 +10,6 @@ expression: displayable.tree_render().to_string() │ revenue DESC │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec(TopK) │ -│ -------------------- │ -│ limit: 20 │ -│ │ -│ revenue@2 DESC │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ c_acctbal: c_acctbal │ @@ -34,16 +27,26 @@ expression: displayable.tree_render().to_string() │ .l_discount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec(TopK) │ +│ -------------------- │ +│ limit: 20 │ +│ │ +│ sum(lineitem │ +│ .l_extendedp │ +│ rice * Int64(1) - lineitem│ +│ .l_discount)@7 DESC │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: │ @@ -73,11 +76,11 @@ expression: displayable.tree_render().to_string() │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q11.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q11.snap index cbc05809d..b12a8e0db 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q11.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q11.snap @@ -19,23 +19,21 @@ expression: displayable.tree_render().to_string() │ │ │ Decimal128(38, 15)) │ └─────────────┬─────────────┘ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ -│ SortExec │ │ AggregateExec │ +│ ProjectionExec │ │ AggregateExec │ │ -------------------- │ │ -------------------- │ -│ value@1 DESC │ │ aggr: │ -│ │ │ sum(partsupp.ps_supplycost│ +│ ps_partkey: │ │ aggr: │ +│ ps_partkey │ │ sum(partsupp.ps_supplycost│ │ │ │ * partsupp.ps_availqty) │ -│ │ │ │ -│ │ │ mode: Final │ +│ value: │ │ │ +│ sum(partsupp.ps_supplycost│ │ mode: Final │ +│ * partsupp.ps_availqty) │ │ │ └─────────────┬─────────────┘ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ -│ ProjectionExec │ │ CoalescePartitionsExec │ +│ SortExec │ │ CoalescePartitionsExec │ │ -------------------- │ │ │ -│ ps_partkey: │ │ │ -│ ps_partkey │ │ │ -│ │ │ │ -│ value: │ │ │ │ sum(partsupp.ps_supplycost│ │ │ │ * partsupp.ps_availqty) │ │ │ +│ @1 DESC │ │ │ └─────────────┬─────────────┘ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ │ FilterExec │ │ AggregateExec │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q12.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q12.snap index 97026caa9..708c98429 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q12.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q12.snap @@ -8,11 +8,6 @@ expression: displayable.tree_render().to_string() │ l_shipmode ASC NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│l_shipmode@0 ASC NULLS LAST│ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ high_line_count: │ @@ -37,6 +32,11 @@ expression: displayable.tree_render().to_string() │ END) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│l_shipmode@0 ASC NULLS LAST│ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q13.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q13.snap index 3002872f9..14919ac99 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q13.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q13.snap @@ -8,12 +8,6 @@ expression: displayable.tree_render().to_string() │custdist DESC, c_count DESC│ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ custdist@1 DESC, c_count@0│ -│ DESC │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ c_count: c_count │ @@ -22,6 +16,12 @@ expression: displayable.tree_render().to_string() │ count(Int64(1)) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ count(Int64(1))@1 DESC, │ +│ c_count@0 DESC │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: count(1) │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q14.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q14.snap index 9f8cd9219..ec8ab519e 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q14.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q14.snap @@ -26,8 +26,8 @@ expression: displayable.tree_render().to_string() │ sum(CASE WHEN part.p_type │ │ LIKE PROMO% THEN │ │ __common_expr_1 │ -│ ELSE Some(0),38,4 END) │ -│ as sum(CASE WHEN part │ +│ ELSE 0.0000 END) as │ +│ sum(CASE WHEN part │ │ .p_type LIKE Utf8( │ │ "PROMO%") THEN │ │ lineitem │ @@ -53,8 +53,8 @@ expression: displayable.tree_render().to_string() │ sum(CASE WHEN part.p_type │ │ LIKE PROMO% THEN │ │ __common_expr_1 │ -│ ELSE Some(0),38,4 END) │ -│ as sum(CASE WHEN part │ +│ ELSE 0.0000 END) as │ +│ sum(CASE WHEN part │ │ .p_type LIKE Utf8( │ │ "PROMO%") THEN │ │ lineitem │ @@ -74,8 +74,8 @@ expression: displayable.tree_render().to_string() │ ProjectionExec │ │ -------------------- │ │ __common_expr_1: │ -│ l_extendedprice * (Some(1)│ -│ ,20,0 - l_discount) │ +│ l_extendedprice * (1 - │ +│ l_discount) │ │ │ │ p_type: p_type │ └─────────────┬─────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q16.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q16.snap index 6c2e06f76..b7c8c7f59 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q16.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q16.snap @@ -11,16 +11,6 @@ expression: displayable.tree_render().to_string() │ ASC NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ supplier_cnt@3 DESC, │ -│ p_brand@0 ASC │ -│ NULLS LAST, p_type │ -│ @1 ASC NULLS LAST, │ -│ p_size@2 ASC NULLS │ -│ LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ p_brand: p_brand │ @@ -31,6 +21,15 @@ expression: displayable.tree_render().to_string() │ count(alias1) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ count(alias1)@3 DESC, │ +│ p_brand@0 ASC NULLS │ +│ LAST, p_type@1 ASC NULLS │ +│ LAST, p_size@2 ASC NULLS │ +│ LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: count(alias1) │ @@ -94,7 +93,9 @@ expression: displayable.tree_render().to_string() │ HashJoinExec │ │ -------------------- │ │ join_type: LeftAnti │ -│ ├───────────────────────────────────────────┐ +│ │ +│ null_aware ├───────────────────────────────────────────┐ +│ │ │ │ on: │ │ │ (ps_suppkey = s_suppkey) │ │ └─────────────┬─────────────┘ │ @@ -122,8 +123,8 @@ expression: displayable.tree_render().to_string() │ ││ ││ │ │ predicate: ││ ││ predicate: │ │ p_brand != Brand#45 AND ││ ││ s_comment LIKE %Customer │ -│ p_type NOT LIKE ││ ││ %Complaints% │ -│ MEDIUM POLISHED% ││ ││ │ -│ AND p_size IN (49, 14, ││ ││ │ -│ 23, 45, 19, 3, 36, 9) ││ ││ │ +│ p_size IN (49, 14, 23 ││ ││ %Complaints% │ +│ , 45, 19, 3, 36, 9) AND ││ ││ │ +│ p_type NOT LIKE MEDIUM ││ ││ │ +│ POLISHED% ││ ││ │ └───────────────────────────┘└───────────────────────────┘└───────────────────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q17.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q17.snap index 78d6cade3..2c05107f6 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q17.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q17.snap @@ -39,8 +39,10 @@ expression: displayable.tree_render().to_string() │ filter: │ │ CAST(l_quantity AS │ │ Decimal128(30, │ -│ 15)) < Float64(0.2) * ├──────────────┐ -│ avg(lineitem.l_quantity) │ │ +│ 15)) < Float64(0.2) * │ +│ avg(lineitem.l_quantity) ├──────────────┐ +│ │ │ +│ join_type: RightSemi │ │ │ │ │ │ on: │ │ │ (l_partkey = p_partkey) │ │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q18.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q18.snap index f6b222226..a03daf342 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q18.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q18.snap @@ -79,7 +79,7 @@ expression: displayable.tree_render().to_string() │ -------------------- ││ -------------------- │ │ predicate: ││ on: ├───────────────────────────────────────────┐ │ sum(lineitem.l_quantity) >││ (o_orderkey = l_orderkey) │ │ -│ Some(30000),25,2 ││ │ │ +│ 300.00 ││ │ │ └─────────────┬─────────────┘└─────────────┬─────────────┘ │ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ │ AggregateExec ││ HashJoinExec │ │ LiquidCacheClientExec │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q19.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q19.snap index 080a4932c..b65eaaf77 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q19.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q19.snap @@ -17,11 +17,11 @@ expression: displayable.tree_render().to_string() │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ mode: Final │ @@ -35,11 +35,11 @@ expression: displayable.tree_render().to_string() │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ mode: Partial │ @@ -52,26 +52,21 @@ expression: displayable.tree_render().to_string() │ p_container IN (SM │ │ CASE, SM BOX, SM PACK, │ │ SM PKG) AND l_quantity > │ -│ = Some(100),15,2 AND │ -│ l_quantity <= Some │ -│ (1100),15,2 AND p_size < │ -│ = 5 OR p_brand = Brand#23 │ +│ = 1.00 AND l_quantity <= │ +│ 11.00 AND p_size <= 5 │ +│ OR p_brand = Brand#23 │ │ AND p_container IN (MED │ │ BAG, MED BOX, MED PKG, │ -│ MED PACK) AND │ -│ l_quantity >= │ -│ Some(1000),15,2 AND ├──────────────┐ -│ l_quantity <= Some │ │ -│ (2000),15,2 AND p_size │ │ -│ <= 10 OR p_brand = Brand │ │ -│ #34 AND p_container IN │ │ -│ (LG CASE, LG BOX, LG │ │ -│ PACK, LG PKG) AND │ │ -│ l_quantity >= Some │ │ -│ (2000),15,2 AND │ │ -│ l_quantity <= │ │ -│ Some(3000),15,2 AND │ │ -│ p_size <= 15 │ │ +│ MED PACK) AND ├──────────────┐ +│ l_quantity >= │ │ +│ 10.00 AND l_quantity < │ │ +│ = 20.00 AND p_size <= 10 │ │ +│ OR p_brand = Brand#34 │ │ +│ AND p_container IN (LG │ │ +│ CASE, LG BOX, LG PACK, │ │ +│ LG PKG) AND l_quantity │ │ +│ >= 20.00 AND l_quantity │ │ +│ <= 30.00 AND p_size <= 15 │ │ │ │ │ │ on: │ │ │ (p_partkey = l_partkey) │ │ @@ -118,16 +113,11 @@ expression: displayable.tree_render().to_string() │ REG) AND │ │ l_shipinstr │ │ uct = DELIVER IN PERSON │ - │ AND (l_quantity >= │ - │ Some(100),15,2 AND │ - │ l_quantity <= Some │ - │ (1100),15,2 OR │ - │ l_quantity >= │ - │ Some(1000),15,2 AND │ - │ l_quantity <= Some │ - │ (2000),15,2 OR │ - │ l_quantity >= │ - │ Some(2000),15,2 AND │ - │ l_quantity <= Some(3000 │ - │ ),15,2) │ + │ AND (l_quantity >= 1 │ + │ .00 AND l_quantity <= 11 │ + │ .00 OR l_quantity >= 10 │ + │ .00 AND l_quantity <= │ + │ 20.00 OR l_quantity >= │ + │ 20.00 AND l_quantity <= │ + │ 30.00) │ └───────────────────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q2.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q2.snap index 633cea2c9..1cc295cdd 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q2.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q2.snap @@ -22,9 +22,11 @@ expression: displayable.tree_render().to_string() ┌─────────────┴─────────────┐ │ HashJoinExec │ │ -------------------- │ +│ join_type: LeftSemi │ +│ │ │ on: │ -│ (p_partkey = ps_partkey), │ -│ (ps_supplycost = min ├──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ (p_partkey = ps_partkey), ├──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ (ps_supplycost = min │ │ │ (partsupp │ │ │ .ps_supplycost │ │ │ )) │ │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q20.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q20.snap index 7a145d7b9..9f69961d4 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q20.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q20.snap @@ -27,8 +27,10 @@ expression: displayable.tree_render().to_string() │(n_nationkey = s_nationkey)│ │ CAST(ps_availqty AS │ │ │ │ Float64) > │ │ │ │ Float64(0.5) * │ -│ ├──────────────┐ │ sum(lineitem ├───────────────────────────────────────────┐ -│ │ │ │ .l_quantity) │ │ +│ │ │ sum(lineitem │ +│ ├──────────────┐ │ .l_quantity) ├───────────────────────────────────────────┐ +│ │ │ │ │ │ +│ │ │ │ join_type: LeftSemi │ │ │ │ │ │ │ │ │ │ │ │ on: │ │ │ │ │ │ (ps_partkey = l_partkey), │ │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q21.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q21.snap index d2a6f6f5b..859314c85 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q21.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q21.snap @@ -9,12 +9,6 @@ expression: displayable.tree_render().to_string() │ NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ numwait@1 DESC, s_name@0 │ -│ ASC NULLS LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ numwait: │ @@ -23,6 +17,13 @@ expression: displayable.tree_render().to_string() │ s_name: s_name │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ count(Int64(1))@1 DESC, │ +│ s_name@0 ASC NULLS │ +│ LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: count(1) │ @@ -52,83 +53,86 @@ expression: displayable.tree_render().to_string() │ -------------------- │ │ filter: │ │ l_suppkey != l_suppkey │ -│ ├──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ -│ join_type: LeftAnti │ │ -│ │ │ -│ on: │ │ -│ (l_orderkey = l_orderkey) │ │ -└─────────────┬─────────────┘ │ -┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ -│ HashJoinExec │ │ LiquidCacheClientExec │ -│ -------------------- │ │ -------------------- │ -│ filter: │ │ server: │ -│ l_suppkey != l_suppkey │ │ http://localhost:50051, │ -│ ├─────────────────────────────────────────────────────────────────────────────────────────────────────┐ │ object_stores=[] │ -│ join_type: LeftSemi │ │ │ │ -│ │ │ │ │ -│ on: │ │ │ │ -│ (l_orderkey = l_orderkey) │ │ │ │ -└─────────────┬─────────────┘ │ └─────────────┬─────────────┘ -┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ CoalescePartitionsExec │ │ LiquidCacheClientExec ││ RepartitionExec │ -│ │ │ -------------------- ││ -------------------- │ -│ │ │ server: ││ partition_count(in->out): │ -│ │ │ http://localhost:50051, ││ 1 -> 8 │ -│ │ │ object_stores=[] ││ │ -│ │ │ ││ partitioning_scheme: │ -│ │ │ ││ RoundRobinBatch(8) │ -└─────────────┬─────────────┘ └─────────────┬─────────────┘└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ HashJoinExec │ │ DataSourceExec ││ DataSourceExec │ -│ -------------------- │ │ -------------------- ││ -------------------- │ -│ on: │ │ files: 1 ││ files: 1 │ -│(n_nationkey = s_nationkey)│ │ format: parquet ││ format: parquet │ -│ ├──────────────┐ │ ││ │ -│ │ │ │ ││ predicate: │ -│ │ │ │ ││ l_receiptdate > │ -│ │ │ │ ││ l_commitdate │ -└─────────────┬─────────────┘ │ └───────────────────────────┘└───────────────────────────┘ +│ ├──────────────┐ +│ join_type: RightAnti │ │ +│ │ │ +│ on: │ │ +│ (l_orderkey = l_orderkey) │ │ +└─────────────┬─────────────┘ │ +┌─────────────┴─────────────┐┌─────────────┴─────────────┐ +│ LiquidCacheClientExec ││ RepartitionExec │ +│ -------------------- ││ -------------------- │ +│ server: ││ partition_count(in->out): │ +│ http://localhost:50051, ││ 1 -> 8 │ +│ object_stores=[] ││ │ +│ ││ partitioning_scheme: │ +│ ││ RoundRobinBatch(8) │ +└─────────────┬─────────────┘└─────────────┬─────────────┘ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ LiquidCacheClientExec ││ HashJoinExec │ +│ DataSourceExec ││ HashJoinExec │ │ -------------------- ││ -------------------- │ -│ server: ││ on: ├──────────────┐ -│ http://localhost:50051, ││ (o_orderkey = l_orderkey) │ │ -│ object_stores=[] ││ │ │ -└─────────────┬─────────────┘└─────────────┬─────────────┘ │ -┌─────────────┴─────────────┐┌─────────────┴─────────────┐┌─────────────┴─────────────┐ -│ DataSourceExec ││ LiquidCacheClientExec ││ HashJoinExec │ -│ -------------------- ││ -------------------- ││ -------------------- │ -│ files: 1 ││ server: ││ on: │ -│ format: parquet ││ http://localhost:50051, ││ (s_suppkey = l_suppkey) ├──────────────┐ -│ ││ object_stores=[] ││ │ │ -│ predicate: ││ ││ │ │ -│ n_name = SAUDI ARABIA ││ ││ │ │ -└───────────────────────────┘└─────────────┬─────────────┘└─────────────┬─────────────┘ │ +│ files: 1 ││ filter: │ +│ format: parquet ││ l_suppkey != l_suppkey │ +│ ││ ├─────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ predicate: ││ join_type: LeftSemi │ │ +│ l_receiptdate > ││ │ │ +│ l_commitdate ││ on: │ │ +│ ││ (l_orderkey = l_orderkey) │ │ +└───────────────────────────┘└─────────────┬─────────────┘ │ + ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ + │ CoalescePartitionsExec │ │ LiquidCacheClientExec │ + │ │ │ -------------------- │ + │ │ │ server: │ + │ │ │ http://localhost:50051, │ + │ │ │ object_stores=[] │ + └─────────────┬─────────────┘ └─────────────┬─────────────┘ + ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ + │ HashJoinExec │ │ DataSourceExec │ + │ -------------------- │ │ -------------------- │ + │ on: ├──────────────┐ │ files: 1 │ + │(n_nationkey = s_nationkey)│ │ │ format: parquet │ + └─────────────┬─────────────┘ │ └───────────────────────────┘ + ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ + │ LiquidCacheClientExec ││ HashJoinExec │ + │ -------------------- ││ -------------------- │ + │ server: ││ on: ├──────────────┐ + │ http://localhost:50051, ││ (o_orderkey = l_orderkey) │ │ + │ object_stores=[] ││ │ │ + └─────────────┬─────────────┘└─────────────┬─────────────┘ │ ┌─────────────┴─────────────┐┌─────────────┴─────────────┐┌─────────────┴─────────────┐ - │ DataSourceExec ││ LiquidCacheClientExec ││ LiquidCacheClientExec │ + │ DataSourceExec ││ LiquidCacheClientExec ││ HashJoinExec │ │ -------------------- ││ -------------------- ││ -------------------- │ - │ files: 1 ││ server: ││ server: │ - │ format: parquet ││ http://localhost:50051, ││ http://localhost:50051, │ - │ ││ object_stores=[] ││ object_stores=[] │ - │ predicate: ││ ││ │ - │ o_orderstatus = F ││ ││ │ - └───────────────────────────┘└─────────────┬─────────────┘└─────────────┬─────────────┘ - ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ - │ DataSourceExec ││ RepartitionExec │ - │ -------------------- ││ -------------------- │ - │ files: 1 ││ partition_count(in->out): │ - │ format: parquet ││ 1 -> 8 │ - │ ││ │ - │ ││ partitioning_scheme: │ - │ ││ RoundRobinBatch(8) │ - └───────────────────────────┘└─────────────┬─────────────┘ - ┌─────────────┴─────────────┐ - │ DataSourceExec │ - │ -------------------- │ - │ files: 1 │ - │ format: parquet │ - │ │ - │ predicate: │ - │ l_receiptdate > │ - │ l_commitdate │ - └───────────────────────────┘ + │ files: 1 ││ server: ││ on: │ + │ format: parquet ││ http://localhost:50051, ││ (s_suppkey = l_suppkey) ├──────────────┐ + │ ││ object_stores=[] ││ │ │ + │ predicate: ││ ││ │ │ + │ n_name = SAUDI ARABIA ││ ││ │ │ + └───────────────────────────┘└─────────────┬─────────────┘└─────────────┬─────────────┘ │ + ┌─────────────┴─────────────┐┌─────────────┴─────────────┐┌─────────────┴─────────────┐ + │ DataSourceExec ││ LiquidCacheClientExec ││ LiquidCacheClientExec │ + │ -------------------- ││ -------------------- ││ -------------------- │ + │ files: 1 ││ server: ││ server: │ + │ format: parquet ││ http://localhost:50051, ││ http://localhost:50051, │ + │ ││ object_stores=[] ││ object_stores=[] │ + │ predicate: ││ ││ │ + │ o_orderstatus = F ││ ││ │ + └───────────────────────────┘└─────────────┬─────────────┘└─────────────┬─────────────┘ + ┌─────────────┴─────────────┐┌─────────────┴─────────────┐ + │ DataSourceExec ││ RepartitionExec │ + │ -------------------- ││ -------------------- │ + │ files: 1 ││ partition_count(in->out): │ + │ format: parquet ││ 1 -> 8 │ + │ ││ │ + │ ││ partitioning_scheme: │ + │ ││ RoundRobinBatch(8) │ + └───────────────────────────┘└─────────────┬─────────────┘ + ┌─────────────┴─────────────┐ + │ DataSourceExec │ + │ -------------------- │ + │ files: 1 │ + │ format: parquet │ + │ │ + │ predicate: │ + │ l_receiptdate > │ + │ l_commitdate │ + └───────────────────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q22.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q22.snap index 740987243..89d275d25 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q22.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q22.snap @@ -14,22 +14,24 @@ expression: displayable.tree_render().to_string() │ │ │ mode: Final │ └─────────────┬─────────────┘ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ -│ SortExec │ │ CoalescePartitionsExec │ +│ ProjectionExec │ │ CoalescePartitionsExec │ │ -------------------- │ │ │ -│ cntrycode@0 ASC NULLS LAST│ │ │ -└─────────────┬─────────────┘ └─────────────┬─────────────┘ -┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ -│ ProjectionExec │ │ LiquidCacheClientExec │ -│ -------------------- │ │ -------------------- │ -│ cntrycode: cntrycode │ │ server: │ -│ │ │ http://localhost:50051, │ -│ numcust: │ │ object_stores=[] │ +│ cntrycode: cntrycode │ │ │ +│ │ │ │ +│ numcust: │ │ │ │ count(Int64(1)) │ │ │ │ │ │ │ │ totacctbal: │ │ │ │ sum(custsale.c_acctbal) │ │ │ └─────────────┬─────────────┘ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ +│ SortExec │ │ LiquidCacheClientExec │ +│ -------------------- │ │ -------------------- │ +│ cntrycode@0 ASC NULLS LAST│ │ server: │ +│ │ │ http://localhost:50051, │ +│ │ │ object_stores=[] │ +└─────────────┬─────────────┘ └─────────────┬─────────────┘ +┌─────────────┴─────────────┐ ┌─────────────┴─────────────┐ │ AggregateExec │ │ AggregateExec │ │ -------------------- │ │ -------------------- │ │ aggr: │ │ aggr: │ @@ -57,8 +59,8 @@ expression: displayable.tree_render().to_string() │ count(1), sum(custsale │ │ format: parquet │ │ .c_acctbal) │ │ │ │ │ │ predicate: │ -│ group_by: cntrycode │ │ c_acctbal > Some(0),15,2 │ -│ mode: Partial │ │ AND substr(c_phone, 1, │ +│ group_by: cntrycode │ │ c_acctbal > 0.00 AND │ +│ mode: Partial │ │ substr(c_phone, 1, │ │ │ │ 2) IN (13, 31, 23, 29, │ │ │ │ 30, 18, 17) │ └─────────────┬─────────────┘ └───────────────────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q3.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q3.snap index 9387f57b6..76f7ce9ed 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q3.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q3.snap @@ -11,15 +11,6 @@ expression: displayable.tree_render().to_string() │ ASC NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec(TopK) │ -│ -------------------- │ -│ limit: 10 │ -│ │ -│ revenue@1 DESC, │ -│ o_orderdate@2 │ -│ ASC NULLS LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ l_orderkey: │ @@ -38,16 +29,28 @@ expression: displayable.tree_render().to_string() │ .l_discount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec(TopK) │ +│ -------------------- │ +│ limit: 10 │ +│ │ +│ sum(lineitem │ +│ .l_extendedp │ +│ rice * Int64(1) - lineitem│ +│ .l_discount)@3 DESC, │ +│ o_orderdate@1 ASC │ +│ NULLS LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: │ @@ -74,11 +77,11 @@ expression: displayable.tree_render().to_string() │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q4.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q4.snap index 7974f5a26..a727ea5fc 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q4.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q4.snap @@ -9,12 +9,6 @@ expression: displayable.tree_render().to_string() │ LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ o_orderpriority@0 ASC │ -│ NULLS LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ o_orderpriority: │ @@ -24,6 +18,12 @@ expression: displayable.tree_render().to_string() │ count(Int64(1)) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ o_orderpriority@0 ASC │ +│ NULLS LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: count(1) │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q5.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q5.snap index 0c5df4436..bd87dd29b 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q5.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q5.snap @@ -8,11 +8,6 @@ expression: displayable.tree_render().to_string() │ revenue DESC │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ revenue@1 DESC │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ n_name: n_name │ @@ -24,16 +19,24 @@ expression: displayable.tree_render().to_string() │ .l_discount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ sum(lineitem │ +│ .l_extendedp │ +│ rice * Int64(1) - lineitem│ +│ .l_discount)@1 DESC │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: n_name │ @@ -56,11 +59,11 @@ expression: displayable.tree_render().to_string() │ aggr: │ │ sum(lineitem │ │ .l_extendedp │ -│ rice * Some(1),20,0 - │ -│ lineitem.l_discount │ -│ ) as sum(lineitem │ -│ .l_extendedprice │ -│ * Int64(1) - lineitem │ +│ rice * 1 - lineitem │ +│ .l_discount) as │ +│ sum(lineitem │ +│ .l_extendedprice │ +│ * Int64(1) - lineitem │ │ .l_discount) │ │ │ │ group_by: n_name │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q6.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q6.snap index 4e5df56ea..2b52c8267 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q6.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q6.snap @@ -59,9 +59,6 @@ expression: displayable.tree_render().to_string() │ l_shipdate >= 1994-01-01 │ │ AND l_shipdate < 1995 │ │ -01-01 AND l_discount >= │ -│ Some(5),15,2 AND │ -│ l_discount <= │ -│ Some(7),15,2 AND │ -│ l_quantity < Some │ -│ (2400),15,2 │ +│ 0.05 AND l_discount <= 0 │ +│ .07 AND l_quantity < 24.00│ └───────────────────────────┘ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q7.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q7.snap index 66132d2ea..dc6c9b082 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q7.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q7.snap @@ -11,14 +11,6 @@ expression: displayable.tree_render().to_string() │ LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ supp_nation@0 ASC NULLS │ -│ LAST, cust_nation@1 │ -│ ASC NULLS LAST, l_year │ -│ @2 ASC NULLS LAST │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ cust_nation: │ @@ -33,6 +25,14 @@ expression: displayable.tree_render().to_string() │ supp_nation │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ supp_nation@0 ASC NULLS │ +│ LAST, cust_nation@1 │ +│ ASC NULLS LAST, l_year │ +│ @2 ASC NULLS LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ @@ -79,8 +79,8 @@ expression: displayable.tree_render().to_string() │ supp_nation: n_name │ │ │ │ volume: │ -│ l_extendedprice * (Some(1)│ -│ ,20,0 - l_discount) │ +│ l_extendedprice * (1 - │ +│ l_discount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ │ HashJoinExec │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q8.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q8.snap index 7f2733a23..c133378a9 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q8.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q8.snap @@ -32,16 +32,14 @@ expression: displayable.tree_render().to_string() │ sum(CASE WHEN all_nations │ │ .nation = BRAZIL THEN │ │ all_nations.volume │ -│ ELSE Some(0),38,4 END │ -│ ) as sum(CASE WHEN │ +│ ELSE 0.0000 END) as │ +│ sum(CASE WHEN │ │ all_nations │ │ .nation = Utf8( │ │ "BRAZIL") THEN │ -│ all_nations │ -│ .volume ELSE Int64 │ -│ (0) END), sum │ -│ (all_nations │ -│ .volume) │ +│ all_nations.volume │ +│ ELSE Int64(0) END), sum │ +│ (all_nations.volume) │ │ │ │ group_by: o_year │ │ │ @@ -64,16 +62,14 @@ expression: displayable.tree_render().to_string() │ sum(CASE WHEN all_nations │ │ .nation = BRAZIL THEN │ │ all_nations.volume │ -│ ELSE Some(0),38,4 END │ -│ ) as sum(CASE WHEN │ +│ ELSE 0.0000 END) as │ +│ sum(CASE WHEN │ │ all_nations │ │ .nation = Utf8( │ │ "BRAZIL") THEN │ -│ all_nations │ -│ .volume ELSE Int64 │ -│ (0) END), sum │ -│ (all_nations │ -│ .volume) │ +│ all_nations.volume │ +│ ELSE Int64(0) END), sum │ +│ (all_nations.volume) │ │ │ │ group_by: o_year │ │ mode: Partial │ @@ -88,8 +84,8 @@ expression: displayable.tree_render().to_string() │ o_orderdate) │ │ │ │ volume: │ -│ l_extendedprice * (Some(1)│ -│ ,20,0 - l_discount) │ +│ l_extendedprice * (1 - │ +│ l_discount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ │ HashJoinExec │ diff --git a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q9.snap b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q9.snap index 54d9618e5..7957aaf08 100644 --- a/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q9.snap +++ b/src/datafusion-client/src/tests/snapshots/liquid_cache_datafusion_client__tests__tpch_q9.snap @@ -9,12 +9,6 @@ expression: displayable.tree_render().to_string() │ o_year DESC │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│ nation@0 ASC NULLS LAST, │ -│ o_year@1 DESC │ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ nation: nation │ @@ -24,6 +18,12 @@ expression: displayable.tree_render().to_string() │ sum(profit.amount) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ nation@0 ASC NULLS LAST, │ +│ o_year@1 DESC │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: │ @@ -60,8 +60,8 @@ expression: displayable.tree_render().to_string() │ ProjectionExec │ │ -------------------- │ │ amount: │ -│ l_extendedprice * (Some(1)│ -│ ,20,0 - l_discount) - │ +│ l_extendedprice * (1 - │ +│ l_discount) - │ │ ps_supplycost * │ │ l_quantity │ │ │ diff --git a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema_with_filter.snap b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema_with_filter.snap index e518b4d6c..0b214cc49 100644 --- a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema_with_filter.snap +++ b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema_with_filter.snap @@ -4,6 +4,13 @@ expression: "format!(\"plan: \\n{}\\nvalues: \\n{}\\nstats:\\n{}\", plan, values --- plan: ┌───────────────────────────┐ +│ ProjectionExec │ +│ -------------------- │ +│ EventTime: EventTime │ +│ OS: OS │ +│ WatchID: WatchID │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ SortExec(TopK) │ │ -------------------- │ │ WatchID@0 DESC │ From a9ceeb497067b380711053f123e11b3095f80282 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 20:12:24 -0700 Subject: [PATCH 4/6] test: refresh provide_schema2 plan snapshot --- ...afusion_local__tests__provide_schema2.snap | 21 ++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap index 3b21a3f31..0a535fb1a 100644 --- a/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap +++ b/src/datafusion-local/src/tests/snapshots/liquid_cache_datafusion_local__tests__provide_schema2.snap @@ -54,11 +54,6 @@ plan: │ zo_sql_key ASC NULLS LAST │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ -│ SortExec │ -│ -------------------- │ -│zo_sql_key@0 ASC NULLS LAST│ -└─────────────┬─────────────┘ -┌─────────────┴─────────────┐ │ ProjectionExec │ │ -------------------- │ │ zo_sql_key: │ @@ -78,6 +73,22 @@ plan: │ count(Int64(1)) │ └─────────────┬─────────────┘ ┌─────────────┴─────────────┐ +│ SortExec │ +│ -------------------- │ +│ date_bin │ +│ (Interva │ +│ lMonthDayNano( │ +│ "IntervalMonth │ +│ DayNano { months: 0, days:│ +│ 0, nanoseconds: │ +│ 10000000000 }") │ +│ ,to_timestamp_micros │ +│ (default._timestamp) │ +│ ,to_timestamp(Utf8("2001 │ +│ -01-01T00:00:00")))@0 │ +│ ASC NULLS LAST │ +└─────────────┬─────────────┘ +┌─────────────┴─────────────┐ │ AggregateExec │ │ -------------------- │ │ aggr: count(1) │ From 7563c2eba353602e5aa691b26afdca5e6dfdccc2 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 21:55:11 -0700 Subject: [PATCH 5/6] docs(reader): drop stale file pruning comment --- src/datafusion/src/reader/plantime/opener.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/datafusion/src/reader/plantime/opener.rs b/src/datafusion/src/reader/plantime/opener.rs index 386e35735..c34271ccd 100644 --- a/src/datafusion/src/reader/plantime/opener.rs +++ b/src/datafusion/src/reader/plantime/opener.rs @@ -145,8 +145,6 @@ impl FileOpener for LiquidParquetOpener { // Prune this file using the file level statistics and partition values. // Since dynamic filters may have been updated since planning it is possible that we are able // to prune files now that we couldn't prune at planning time. - // It is assumed that there is no point in doing pruning here if the predicate is not dynamic, - // as it would have been done at planning time. // We'll also check this after every record batch we read, // and if at some point we are able to prove we can prune the file using just the file level statistics // we can end the stream early. From a7ac25a769529ff43f66c47cef65fe8bd46da244 Mon Sep 17 00:00:00 2001 From: Zac Farrell Date: Thu, 27 Aug 2026 21:55:11 -0700 Subject: [PATCH 6/6] test(local): cover multi-partition cache scans --- src/datafusion-local/src/tests/mod.rs | 92 +++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/src/datafusion-local/src/tests/mod.rs b/src/datafusion-local/src/tests/mod.rs index 697c2d83f..eedfceec3 100644 --- a/src/datafusion-local/src/tests/mod.rs +++ b/src/datafusion-local/src/tests/mod.rs @@ -610,3 +610,95 @@ async fn test_provide_schema_with_filter() { } assert_eq!(formatted_results, reference); } + +/// Covers the multi-partition scan path against the shared cache. +/// +/// The tests above pin `repartition_file_min_size` above the test file size so +/// the scan stays single-partition and their traces stay reproducible. That pin +/// is deliberate, but it also means nothing else exercises several scan +/// partitions admitting into one cache concurrently — which is exactly what a +/// default DataFusion 55 deployment does for any file over 1 MiB, since DF 55 +/// lowered the threshold from 10 MiB to 1 MiB. +/// +/// So this test leaves `repartition_file_min_size` at the DF 55 default and +/// asserts only order-independent properties: the result rows compared as a +/// sorted multiset (against a single-partition run of the same query), and that +/// the warm run hits the cache. No trace, byte count or row order is pinned, so +/// it cannot reintroduce the snapshot flakiness the pin defends against. +#[tokio::test] +async fn test_multi_partition_scan_shares_cache() { + /// Rows as an order-independent multiset. + async fn sorted_rows(ctx: &SessionContext, sql: &str) -> Vec { + let plan = get_physical_plan(sql, ctx).await; + let batches = collect(plan, ctx.task_ctx()).await.unwrap(); + let mut rows = pretty_format_batches(&batches) + .unwrap() + .to_string() + .lines() + .map(str::to_string) + .collect::>(); + rows.sort(); + rows + } + + async fn build_ctx( + config: SessionConfig, + cache_dir: &Path, + ) -> (SessionContext, LiquidCacheParquetRef) { + let (ctx, cache) = LiquidCacheLocalBuilder::new() + .with_max_memory_bytes(1024 * 1024) + .with_cache_dir(cache_dir.to_path_buf()) + .with_squeeze_policy(Box::new(TranscodeSqueezeEvict)) + .with_cache_policy(Box::new(LiquidPolicy::new())) + .build(config) + .await + .unwrap(); + ctx.register_parquet("hits", TEST_FILE, ParquetReadOptions::default()) + .await + .unwrap(); + (ctx, cache) + } + + let sql = r#"select "OS", COUNT(*) from hits where "URL" like '%tours%' group by "OS""#; + + // Multi-partition: DF 55 default `repartition_file_min_size` (1 MiB) against + // the 2.3 MB test file, so `target_partitions` really does split the scan. + let multi_dir = TempDir::new().unwrap(); + let mut multi_config = SessionConfig::new(); + multi_config.options_mut().execution.target_partitions = 4; + let (multi_ctx, cache) = build_ctx(multi_config, multi_dir.path()).await; + + // Guard the premise: if a future default makes the scan single-partition + // again, this test would silently stop covering concurrent admission. + let scan_partitions = { + let mut node = get_physical_plan(sql, &multi_ctx).await; + while let Some(child) = node.children().first() { + node = Arc::clone(child); + } + node.properties().partitioning.partition_count() + }; + assert!( + scan_partitions > 1, + "expected a multi-partition scan, got {scan_partitions}" + ); + + // Clear historical counters, then warm the cache and read it back. + cache.storage().stats(); + let first_run = sorted_rows(&multi_ctx, sql).await; + let entries_after_first_run = cache.storage().stats().total_entries; + let second_run = sorted_rows(&multi_ctx, sql).await; + let stats = CacheStatsSummary::from_stats(cache.storage().stats(), entries_after_first_run); + + assert_eq!(first_run, second_run); + assert!( + stats.has_cache_hits(), + "warm multi-partition run did not read from the cache" + ); + + // Same answer as the single-partition path the snapshot tests pin. + let single_dir = TempDir::new().unwrap(); + let mut single_config = cache_test_config(); + single_config.options_mut().execution.target_partitions = 4; + let (single_ctx, _single_cache) = build_ctx(single_config, single_dir.path()).await; + assert_eq!(sorted_rows(&single_ctx, sql).await, second_run); +}