From 64e66feb3947ec16e9261fffef15f8fc3b347594 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 23 Jun 2026 10:05:56 -0700 Subject: [PATCH 1/5] feat(python): add non-recursive delimited list to LanceFileSession Expose a blob-only, non-recursive list on LanceFileSession that returns the immediate child prefixes (and objects) of a path, mirroring object_store's list_with_delimiter. Unlike the existing recursive list, this descends exactly one directory level and, on Azure, is served entirely from the blob endpoint (no hierarchical-namespace/DFS probe). - lance-io: add ObjectStore::list_with_delimiter wrapper exposing the underlying ListResult primitive. - python: add LanceFileSession.list_with_delimiter returning a ListResult dataclass (common_prefixes, objects) of session-relative paths. Co-Authored-By: Claude Opus 4.8 --- python/python/lance/file.py | 45 ++++++++++++++++++++++ python/python/lance/lance/__init__.pyi | 3 ++ python/python/tests/test_file.py | 38 ++++++++++++++++++ python/src/file.rs | 53 ++++++++++++++++++++++++++ rust/lance-io/src/object_store.rs | 14 ++++++- 5 files changed, 152 insertions(+), 1 deletion(-) diff --git a/python/python/lance/file.py b/python/python/lance/file.py index 011fbe4a01d..e1bc851e12a 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -1,6 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright The Lance Authors +from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Optional, Union @@ -207,6 +208,26 @@ def num_rows(self) -> int: return self._reader.num_rows() +@dataclass +class ListResult: + """ + Result of a non-recursive, delimited list (see + :meth:`LanceFileSession.list_with_delimiter`). + + Attributes + ---------- + common_prefixes : List[str] + The immediate child "directories" of the listed path, relative to the + session's base path. + objects : List[str] + The immediate child files of the listed path, relative to the session's + base path. + """ + + common_prefixes: List[str] + objects: List[str] + + class LanceFileSession: """ A file session for reading and writing Lance files. @@ -344,6 +365,30 @@ def list(self, path: Optional[str] = None) -> List[str]: """ return self._session.list(path) + def list_with_delimiter(self, path: Optional[str] = None) -> ListResult: + """ + Non-recursively list a single directory level (relative to this + session's base path). + + Unlike :meth:`list`, which recurses into the entire subtree, this + returns only the immediate children of ``path``: the child + "directories" as ``common_prefixes`` and the direct child files as + ``objects``. On Azure this is served entirely from the blob endpoint, so + it never probes the hierarchical-namespace (DFS) endpoint. + + Parameters + ---------- + path : str, optional + Path relative to `base_path` to list. If None, lists the base path. + + Returns + ------- + ListResult + The immediate child prefixes and objects of `path`. + """ + common_prefixes, objects = self._session.list_with_delimiter(path) + return ListResult(common_prefixes=common_prefixes, objects=objects) + def upload_file(self, local_path: Union[str, Path], remote_path: str) -> None: """ Upload a file from local filesystem to the object store. diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi index 26ad75a27b7..8a0a91efeaa 100644 --- a/python/python/lance/lance/__init__.pyi +++ b/python/python/lance/lance/__init__.pyi @@ -135,6 +135,9 @@ class LanceFileSession: ) -> LanceFileWriter: ... def contains(self, path: str) -> bool: ... def list(self, path: Optional[str] = None) -> List[str]: ... + def list_with_delimiter( + self, path: Optional[str] = None + ) -> tuple[List[str], List[str]]: ... def upload_file(self, local_path: str, remote_path: str) -> None: ... def download_file(self, remote_path: str, local_path: str) -> None: ... diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py index c71654769b8..d10643d10c3 100644 --- a/python/python/tests/test_file.py +++ b/python/python/tests/test_file.py @@ -736,6 +736,44 @@ def test_session_list_with_trailing_slash(tmp_path): assert files_no_slash == ["dir/file.lance"] +def test_session_list_with_delimiter(tmp_path): + """Test that LanceFileSession.list_with_delimiter() is non-recursive.""" + session = LanceFileSession(str(tmp_path)) + schema = pa.schema([pa.field("x", pa.int64())]) + + # Two top-level files and two nested subtrees. + with session.open_writer("file1.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [1]})) + with session.open_writer("file2.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [2]})) + with session.open_writer("subdir/file3.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [3]})) + with session.open_writer("subdir/nested/file4.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [4]})) + with session.open_writer("other/file5.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [5]})) + + # Listing the base path returns only the immediate children: the two + # top-level files and the two child directories (not their contents). + result = session.list_with_delimiter() + assert sorted(result.common_prefixes) == ["other", "subdir"] + assert sorted(result.objects) == ["file1.lance", "file2.lance"] + + # Listing a subdirectory descends exactly one level: the direct file and + # the nested directory, but not the file inside the nested directory. + subdir = session.list_with_delimiter("subdir") + assert subdir.common_prefixes == ["subdir/nested"] + assert subdir.objects == ["subdir/file3.lance"] + + # Trailing slash behaves the same as no trailing slash. + assert session.list_with_delimiter("subdir/") == subdir + + # A non-existent prefix yields empty results rather than erroring. + empty = session.list_with_delimiter("nonexistent") + assert empty.common_prefixes == [] + assert empty.objects == [] + + def test_session_contains(tmp_path): """Test that LanceFileSession.contains() works correctly""" session = LanceFileSession(str(tmp_path)) diff --git a/python/src/file.rs b/python/src/file.rs index b0bc20f9d0a..17633af22b3 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -574,6 +574,59 @@ impl LanceFileSession { })? } + /// Non-recursive, delimited list of a single directory level. + /// + /// Returns a tuple `(common_prefixes, objects)` of paths relative to the + /// session's `base_path`, where `common_prefixes` are the immediate child + /// "directories" and `objects` are the immediate child files. Unlike + /// `list`, this does not recurse into the subtree. On Azure it is served + /// entirely from the blob endpoint, so it never probes the + /// hierarchical-namespace (DFS) endpoint. + #[pyo3(signature=(path=None))] + pub fn list_with_delimiter( + &self, + path: Option, + ) -> PyResult<(Vec, Vec)> { + rt().block_on(None, async { + let list_path = if let Some(prefix) = path { + self.base_path.child_path(&Path::from(prefix)) + } else { + self.base_path.clone() + }; + + let result = self + .object_store + .list_with_delimiter(Some(&list_path)) + .await + .map_err(|e| PyErr::new::(format!("{}", e)))?; + + // Strip the base_path prefix to make each path relative to the session. + let relativize = |location: &Path| -> PyResult { + let relative_parts = location.prefix_match(&self.base_path).ok_or_else(|| { + PyErr::new::(format!( + "Path '{}' does not start with base path '{}'", + location.as_ref(), + self.base_path.as_ref() + )) + })?; + Ok(Path::from_iter(relative_parts).as_ref().to_string()) + }; + + let common_prefixes = result + .common_prefixes + .iter() + .map(relativize) + .collect::>>()?; + let objects = result + .objects + .iter() + .map(|meta| relativize(&meta.location)) + .collect::>>()?; + + Ok((common_prefixes, objects)) + })? + } + /// Upload a file from local filesystem to the object store /// /// Parameters diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs index 1761dc4b059..aecc2015f5a 100644 --- a/rust/lance-io/src/object_store.rs +++ b/rust/lance-io/src/object_store.rs @@ -25,7 +25,7 @@ use object_store::ObjectStoreExt as OSObjectStoreExt; use object_store::aws::AwsCredentialProvider; #[cfg(any(feature = "aws", feature = "azure", feature = "gcp"))] use object_store::{ClientOptions, HeaderMap, HeaderValue}; -use object_store::{ObjectMeta, ObjectStore as OSObjectStore, path::Path}; +use object_store::{ListResult, ObjectMeta, ObjectStore as OSObjectStore, path::Path}; use providers::local::FileStoreProvider; use providers::memory::MemoryStoreProvider; use tokio::io::AsyncWriteExt; @@ -849,6 +849,18 @@ impl ObjectStore { .collect()) } + /// Non-recursive, path-segment delimited list of a single directory level. + /// + /// Unlike [`Self::list`], which recurses into the entire subtree, this returns + /// only the immediate children of `prefix`: the child "directories" as + /// [`ListResult::common_prefixes`] and the direct child files as + /// [`ListResult::objects`]. On Azure this is served entirely from the blob + /// endpoint (`List Blobs` with `delimiter=/`), so it never probes the + /// hierarchical-namespace (DFS) endpoint. + pub async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { + Ok(self.inner.list_with_delimiter(prefix).await?) + } + pub fn list( &self, path: Option, From 53d0e71db1e64d21004e5d7f34fa94d5435adba4 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 23 Jun 2026 10:45:53 -0700 Subject: [PATCH 2/5] feat(python): add delete_file to LanceFileSession Add a delete primitive to LanceFileSession so callers can remove objects through the object_store session instead of going through PyArrow's AzureFileSystem. The underlying object_store crate already supports delete; this is a binding-exposure change. - python (Rust): add LanceFileSession.delete_file(path), routing to object_store delete with a session-relative path. Deleting a missing object is a no-op (idempotent) rather than an error. - python (wrapper): add the delete_file wrapper and type stub. On Azure this talks to the blob endpoint only and never probes the hierarchical-namespace (DFS) endpoint. Co-Authored-By: Claude Opus 4.8 --- python/python/lance/file.py | 16 +++++++++++++++ python/python/lance/lance/__init__.pyi | 1 + python/python/tests/test_file.py | 25 ++++++++++++++++++++++++ python/src/file.rs | 27 ++++++++++++++++++++++++++ 4 files changed, 69 insertions(+) diff --git a/python/python/lance/file.py b/python/python/lance/file.py index e1bc851e12a..eeb09087774 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -389,6 +389,22 @@ def list_with_delimiter(self, path: Optional[str] = None) -> ListResult: common_prefixes, objects = self._session.list_with_delimiter(path) return ListResult(common_prefixes=common_prefixes, objects=objects) + def delete_file(self, path: str) -> None: + """ + Delete a file (relative to this session's base path). + + Deleting a path that does not exist is a no-op (idempotent) rather than + an error, so this is safe to call on best-effort cleanup paths. On Azure + it talks to the blob endpoint only and never probes the + hierarchical-namespace (DFS) endpoint. + + Parameters + ---------- + path : str + Path relative to `base_path` to delete. + """ + self._session.delete_file(path) + def upload_file(self, local_path: Union[str, Path], remote_path: str) -> None: """ Upload a file from local filesystem to the object store. diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi index 8a0a91efeaa..c91d05c331e 100644 --- a/python/python/lance/lance/__init__.pyi +++ b/python/python/lance/lance/__init__.pyi @@ -138,6 +138,7 @@ class LanceFileSession: def list_with_delimiter( self, path: Optional[str] = None ) -> tuple[List[str], List[str]]: ... + def delete_file(self, path: str) -> None: ... def upload_file(self, local_path: str, remote_path: str) -> None: ... def download_file(self, remote_path: str, local_path: str) -> None: ... diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py index d10643d10c3..b830e7ca14b 100644 --- a/python/python/tests/test_file.py +++ b/python/python/tests/test_file.py @@ -797,6 +797,31 @@ def test_session_contains(tmp_path): assert not session.contains("subdir/nonexistent.lance") +def test_session_delete_file(tmp_path): + """Test that LanceFileSession.delete_file() removes files and is idempotent.""" + session = LanceFileSession(str(tmp_path)) + schema = pa.schema([pa.field("x", pa.int64())]) + + with session.open_writer("test.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [1]})) + with session.open_writer("subdir/nested.lance", schema=schema) as writer: + writer.write_batch(pa.table({"x": [2]})) + + # Deleting an existing file removes it. + assert session.contains("test.lance") + session.delete_file("test.lance") + assert not session.contains("test.lance") + + # Nested paths work too. + assert session.contains("subdir/nested.lance") + session.delete_file("subdir/nested.lance") + assert not session.contains("subdir/nested.lance") + + # Deleting a missing path is a no-op rather than an error (idempotent). + session.delete_file("test.lance") + session.delete_file("never_existed.lance") + + def test_struct_null_regression(): import lance diff --git a/python/src/file.rs b/python/src/file.rs index 17633af22b3..74baff06c2e 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -659,6 +659,33 @@ impl LanceFileSession { })? } + /// Delete a file from the object store. + /// + /// The path is interpreted relative to the session's `base_path`, matching + /// `contains`/`upload_file`/`download_file`. Deleting a missing object is a + /// no-op (idempotent) rather than an error, so this is safe to call on + /// best-effort cleanup paths. On Azure it talks to the blob endpoint only + /// and never probes the hierarchical-namespace (DFS) endpoint. + /// + /// Parameters + /// ---------- + /// path : str + /// Path relative to `base_path` to delete. + pub fn delete_file(&self, path: String) -> PyResult<()> { + rt().block_on(None, async { + let full_path = self.base_path.child_path(&Path::from(path)); + match self.object_store.inner.delete(&full_path).await { + Ok(()) => Ok(()), + // Deleting an object that does not exist is a no-op. + Err(object_store::Error::NotFound { .. }) => Ok(()), + Err(e) => Err(PyIOError::new_err(format!( + "Failed to delete remote file: {}", + e + ))), + } + })? + } + /// Download a file from object store to local filesystem /// /// Parameters From 9cf65b318e7c3a4380d78277704dc3dbfec0bd52 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 23 Jun 2026 11:17:15 -0700 Subject: [PATCH 3/5] feat(python): add read_range to LanceFileSession Add a byte-range random read to LanceFileSession so callers can read a slice of a data file through the object_store session instead of going through PyArrow's AzureFileSystem. The underlying object_store crate already supports ranged GETs; this is a binding-exposure change. - python (Rust): add LanceFileSession.read_range(path, offset, length) -> bytes, routing to a single ranged read with a session-relative path. Reading a missing object raises OSError, consistent with download_file. - python (wrapper): add the read_range wrapper and type stub. On Azure this talks to the blob endpoint only and never probes the hierarchical-namespace (DFS) endpoint. Co-Authored-By: Claude Opus 4.8 --- python/python/lance/file.py | 24 ++++++++++++++++++ python/python/lance/lance/__init__.pyi | 1 + python/python/tests/test_file.py | 23 +++++++++++++++++ python/src/file.rs | 35 ++++++++++++++++++++++++++ 4 files changed, 83 insertions(+) diff --git a/python/python/lance/file.py b/python/python/lance/file.py index eeb09087774..20ce5e20b9b 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -389,6 +389,30 @@ def list_with_delimiter(self, path: Optional[str] = None) -> ListResult: common_prefixes, objects = self._session.list_with_delimiter(path) return ListResult(common_prefixes=common_prefixes, objects=objects) + def read_range(self, path: str, offset: int, length: int) -> bytes: + """ + Read a byte range from a file (relative to this session's base path). + + Issues a single ranged read. On Azure it talks to the blob endpoint only + and never probes the hierarchical-namespace (DFS) endpoint. Reading a + missing object raises ``OSError``, consistent with ``download_file``. + + Parameters + ---------- + path : str + Path relative to `base_path` to read from. + offset : int + Byte offset at which to start reading. + length : int + Number of bytes to read. + + Returns + ------- + bytes + The requested byte range. + """ + return self._session.read_range(path, offset, length) + def delete_file(self, path: str) -> None: """ Delete a file (relative to this session's base path). diff --git a/python/python/lance/lance/__init__.pyi b/python/python/lance/lance/__init__.pyi index c91d05c331e..49e410459cc 100644 --- a/python/python/lance/lance/__init__.pyi +++ b/python/python/lance/lance/__init__.pyi @@ -138,6 +138,7 @@ class LanceFileSession: def list_with_delimiter( self, path: Optional[str] = None ) -> tuple[List[str], List[str]]: ... + def read_range(self, path: str, offset: int, length: int) -> bytes: ... def delete_file(self, path: str) -> None: ... def upload_file(self, local_path: str, remote_path: str) -> None: ... def download_file(self, remote_path: str, local_path: str) -> None: ... diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py index b830e7ca14b..7e7e174b103 100644 --- a/python/python/tests/test_file.py +++ b/python/python/tests/test_file.py @@ -797,6 +797,29 @@ def test_session_contains(tmp_path): assert not session.contains("subdir/nonexistent.lance") +def test_session_read_range(tmp_path): + """Test that LanceFileSession.read_range() returns the requested bytes.""" + session = LanceFileSession(str(tmp_path)) + + payload = bytes(range(256)) + local = tmp_path / "src.bin" + local.write_bytes(payload) + session.upload_file(str(local), "data/file.bin") + + # A range in the middle of the file. + assert session.read_range("data/file.bin", 10, 5) == payload[10:15] + # From the start. + assert session.read_range("data/file.bin", 0, 4) == payload[0:4] + # Up to the end. + assert session.read_range("data/file.bin", 250, 6) == payload[250:256] + # A zero-length read yields empty bytes. + assert session.read_range("data/file.bin", 100, 0) == b"" + + # Reading a missing object raises OSError (consistent with download_file). + with pytest.raises(OSError): + session.read_range("data/missing.bin", 0, 4) + + def test_session_delete_file(tmp_path): """Test that LanceFileSession.delete_file() removes files and is idempotent.""" session = LanceFileSession(str(tmp_path)) diff --git a/python/src/file.rs b/python/src/file.rs index 74baff06c2e..3e06d86c67c 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -686,6 +686,41 @@ impl LanceFileSession { })? } + /// Read a byte range from a file in the object store. + /// + /// The path is interpreted relative to the session's `base_path`, matching + /// the other session methods. This issues a single ranged GET; on Azure it + /// talks to the blob endpoint only and never probes the + /// hierarchical-namespace (DFS) endpoint. Reading a missing object raises + /// `OSError`, consistent with `download_file`. + /// + /// Parameters + /// ---------- + /// path : str + /// Path relative to `base_path` to read from. + /// offset : int + /// Byte offset at which to start reading. + /// length : int + /// Number of bytes to read. + /// + /// Returns + /// ------- + /// bytes + /// The requested byte range. + pub fn read_range(&self, path: String, offset: usize, length: usize) -> PyResult> { + rt().block_on(None, async { + let full_path = self.base_path.child_path(&Path::from(path)); + let bytes = self + .object_store + .read_one_range(&full_path, offset..offset + length) + .await + .map_err(|e| { + PyIOError::new_err(format!("Failed to read range from remote file: {}", e)) + })?; + Ok(bytes.to_vec()) + })? + } + /// Download a file from object store to local filesystem /// /// Parameters From c4c93f5eef5183992caab8a0ad8b8346fdb91e6f Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 23 Jun 2026 12:37:38 -0700 Subject: [PATCH 4/5] docs(python): drop backend-specific wording from LanceFileSession docs Address review feedback: the new session methods are generic to any object store, so remove the storage-backend-specific notes about the blob endpoint and hierarchical-namespace probing from the list_with_delimiter, delete_file, and read_range docs. Co-Authored-By: Claude Opus 4.8 --- python/python/lance/file.py | 12 ++++-------- python/src/file.rs | 13 ++++--------- rust/lance-io/src/object_store.rs | 4 +--- 3 files changed, 9 insertions(+), 20 deletions(-) diff --git a/python/python/lance/file.py b/python/python/lance/file.py index 20ce5e20b9b..027db3b32f1 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -373,8 +373,7 @@ def list_with_delimiter(self, path: Optional[str] = None) -> ListResult: Unlike :meth:`list`, which recurses into the entire subtree, this returns only the immediate children of ``path``: the child "directories" as ``common_prefixes`` and the direct child files as - ``objects``. On Azure this is served entirely from the blob endpoint, so - it never probes the hierarchical-namespace (DFS) endpoint. + ``objects``. Parameters ---------- @@ -393,9 +392,8 @@ def read_range(self, path: str, offset: int, length: int) -> bytes: """ Read a byte range from a file (relative to this session's base path). - Issues a single ranged read. On Azure it talks to the blob endpoint only - and never probes the hierarchical-namespace (DFS) endpoint. Reading a - missing object raises ``OSError``, consistent with ``download_file``. + Issues a single ranged read. Reading a missing object raises + ``OSError``, consistent with ``download_file``. Parameters ---------- @@ -418,9 +416,7 @@ def delete_file(self, path: str) -> None: Delete a file (relative to this session's base path). Deleting a path that does not exist is a no-op (idempotent) rather than - an error, so this is safe to call on best-effort cleanup paths. On Azure - it talks to the blob endpoint only and never probes the - hierarchical-namespace (DFS) endpoint. + an error, so this is safe to call on best-effort cleanup paths. Parameters ---------- diff --git a/python/src/file.rs b/python/src/file.rs index 3e06d86c67c..4bc28fa8736 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -579,9 +579,7 @@ impl LanceFileSession { /// Returns a tuple `(common_prefixes, objects)` of paths relative to the /// session's `base_path`, where `common_prefixes` are the immediate child /// "directories" and `objects` are the immediate child files. Unlike - /// `list`, this does not recurse into the subtree. On Azure it is served - /// entirely from the blob endpoint, so it never probes the - /// hierarchical-namespace (DFS) endpoint. + /// `list`, this does not recurse into the subtree. #[pyo3(signature=(path=None))] pub fn list_with_delimiter( &self, @@ -664,8 +662,7 @@ impl LanceFileSession { /// The path is interpreted relative to the session's `base_path`, matching /// `contains`/`upload_file`/`download_file`. Deleting a missing object is a /// no-op (idempotent) rather than an error, so this is safe to call on - /// best-effort cleanup paths. On Azure it talks to the blob endpoint only - /// and never probes the hierarchical-namespace (DFS) endpoint. + /// best-effort cleanup paths. /// /// Parameters /// ---------- @@ -689,10 +686,8 @@ impl LanceFileSession { /// Read a byte range from a file in the object store. /// /// The path is interpreted relative to the session's `base_path`, matching - /// the other session methods. This issues a single ranged GET; on Azure it - /// talks to the blob endpoint only and never probes the - /// hierarchical-namespace (DFS) endpoint. Reading a missing object raises - /// `OSError`, consistent with `download_file`. + /// the other session methods. This issues a single ranged GET. Reading a + /// missing object raises `OSError`, consistent with `download_file`. /// /// Parameters /// ---------- diff --git a/rust/lance-io/src/object_store.rs b/rust/lance-io/src/object_store.rs index aecc2015f5a..31d2e8f997e 100644 --- a/rust/lance-io/src/object_store.rs +++ b/rust/lance-io/src/object_store.rs @@ -854,9 +854,7 @@ impl ObjectStore { /// Unlike [`Self::list`], which recurses into the entire subtree, this returns /// only the immediate children of `prefix`: the child "directories" as /// [`ListResult::common_prefixes`] and the direct child files as - /// [`ListResult::objects`]. On Azure this is served entirely from the blob - /// endpoint (`List Blobs` with `delimiter=/`), so it never probes the - /// hierarchical-namespace (DFS) endpoint. + /// [`ListResult::objects`]. pub async fn list_with_delimiter(&self, prefix: Option<&Path>) -> Result { Ok(self.inner.list_with_delimiter(prefix).await?) } From 54ef4809e9def03d2d20edc094ec16286559d323 Mon Sep 17 00:00:00 2001 From: Jonathan M Hsieh Date: Tue, 23 Jun 2026 12:43:51 -0700 Subject: [PATCH 5/5] feat(python): propagate missing-object error from delete_file Address review feedback: rather than swallowing the not-found error to make delete idempotent, mirror the object_store API directly and let the error propagate. Deleting a missing path now raises OSError, consistent with download_file and read_range. Co-Authored-By: Claude Opus 4.8 --- python/python/lance/file.py | 4 ++-- python/python/tests/test_file.py | 8 +++++--- python/src/file.rs | 20 ++++++++------------ 3 files changed, 15 insertions(+), 17 deletions(-) diff --git a/python/python/lance/file.py b/python/python/lance/file.py index 027db3b32f1..7e2b2933aab 100644 --- a/python/python/lance/file.py +++ b/python/python/lance/file.py @@ -415,8 +415,8 @@ def delete_file(self, path: str) -> None: """ Delete a file (relative to this session's base path). - Deleting a path that does not exist is a no-op (idempotent) rather than - an error, so this is safe to call on best-effort cleanup paths. + Deleting a path that does not exist raises ``OSError``, consistent with + ``download_file``. Parameters ---------- diff --git a/python/python/tests/test_file.py b/python/python/tests/test_file.py index 7e7e174b103..d0f0fb0b8b4 100644 --- a/python/python/tests/test_file.py +++ b/python/python/tests/test_file.py @@ -840,9 +840,11 @@ def test_session_delete_file(tmp_path): session.delete_file("subdir/nested.lance") assert not session.contains("subdir/nested.lance") - # Deleting a missing path is a no-op rather than an error (idempotent). - session.delete_file("test.lance") - session.delete_file("never_existed.lance") + # Deleting a missing path raises OSError (consistent with download_file). + with pytest.raises(OSError): + session.delete_file("test.lance") + with pytest.raises(OSError): + session.delete_file("never_existed.lance") def test_struct_null_regression(): diff --git a/python/src/file.rs b/python/src/file.rs index 4bc28fa8736..e1906b9f4f2 100644 --- a/python/src/file.rs +++ b/python/src/file.rs @@ -660,9 +660,8 @@ impl LanceFileSession { /// Delete a file from the object store. /// /// The path is interpreted relative to the session's `base_path`, matching - /// `contains`/`upload_file`/`download_file`. Deleting a missing object is a - /// no-op (idempotent) rather than an error, so this is safe to call on - /// best-effort cleanup paths. + /// `contains`/`upload_file`/`download_file`. Deleting a missing object + /// raises `OSError`, consistent with `download_file`. /// /// Parameters /// ---------- @@ -671,15 +670,12 @@ impl LanceFileSession { pub fn delete_file(&self, path: String) -> PyResult<()> { rt().block_on(None, async { let full_path = self.base_path.child_path(&Path::from(path)); - match self.object_store.inner.delete(&full_path).await { - Ok(()) => Ok(()), - // Deleting an object that does not exist is a no-op. - Err(object_store::Error::NotFound { .. }) => Ok(()), - Err(e) => Err(PyIOError::new_err(format!( - "Failed to delete remote file: {}", - e - ))), - } + self.object_store + .inner + .delete(&full_path) + .await + .map_err(|e| PyIOError::new_err(format!("Failed to delete remote file: {}", e)))?; + Ok(()) })? }