diff --git a/doc/whats-new.rst b/doc/whats-new.rst
index 9dad9dea10b..2809ea9002d 100644
--- a/doc/whats-new.rst
+++ b/doc/whats-new.rst
@@ -49,6 +49,10 @@ Bug Fixes
 - Fix :py:meth:`Dataset.sortby` and :py:meth:`DataArray.sortby` placing NaN values
   at the beginning instead of the end when using ``ascending=False`` (:issue:`7358`).
   By `Kristian Kollsgård <https://github.com/kkollsga>`_.
+- Raise :py:class:`FileNotFoundError` instead of a confusing ``ValueError`` when
+  :py:func:`open_dataset` is called with a non-existent local file path
+  (:issue:`10896`).
+  By `Kristian Kollsgård <https://github.com/kkollsga>`_.
 
 Documentation
 ~~~~~~~~~~~~~
diff --git a/xarray/backends/plugins.py b/xarray/backends/plugins.py
index 6e1784af5b6..db79dbae7ff 100644
--- a/xarray/backends/plugins.py
+++ b/xarray/backends/plugins.py
@@ -3,6 +3,7 @@
 import functools
 import inspect
 import itertools
+import os
 import warnings
 from collections.abc import Callable
 from importlib.metadata import entry_points
@@ -10,10 +11,9 @@
 
 from xarray.backends.common import BACKEND_ENTRYPOINTS, BackendEntrypoint
 from xarray.core.options import OPTIONS
-from xarray.core.utils import module_available
+from xarray.core.utils import is_remote_uri, module_available
 
 if TYPE_CHECKING:
-    import os
     from importlib.metadata import EntryPoint, EntryPoints
 
     from xarray.backends.common import AbstractDataStore
@@ -209,6 +209,11 @@ def guess_engine(
             "https://docs.xarray.dev/en/stable/getting-started-guide/installing.html"
         )
 
+    if isinstance(store_spec, str | os.PathLike):
+        store_spec_str = str(store_spec)
+        if not is_remote_uri(store_spec_str) and not os.path.exists(store_spec_str):
+            raise FileNotFoundError(f"No such file: '{store_spec_str}'")
+
     raise ValueError(error_msg)
 
 
diff --git a/xarray/tests/test_plugins.py b/xarray/tests/test_plugins.py
index e20f665c9ff..5bb17c06eb2 100644
--- a/xarray/tests/test_plugins.py
+++ b/xarray/tests/test_plugins.py
@@ -188,24 +188,66 @@ def test_build_engines_sorted() -> None:
     "xarray.backends.plugins.list_engines",
     mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}),
 )
-def test_no_matching_engine_found() -> None:
-    with pytest.raises(ValueError, match=r"did not find a match in any"):
+def test_no_matching_engine_found(tmp_path) -> None:
+    # Non-existent local file raises FileNotFoundError
+    with pytest.raises(FileNotFoundError, match=r"No such file"):
         plugins.guess_engine("not-valid")
 
+    # Existing file with unrecognized extension raises ValueError
+    existing_file = tmp_path / "test.unknown"
+    existing_file.write_bytes(b"")
+    with pytest.raises(ValueError, match=r"did not find a match in any"):
+        plugins.guess_engine(str(existing_file))
+
+    # Existing file with recognized magic number raises ValueError
+    nc_file = tmp_path / "foo.nc"
+    nc_file.write_bytes(b"CDF\x01\x00\x00\x00\x00")
     with pytest.raises(ValueError, match=r"found the following matches with the input"):
-        plugins.guess_engine("foo.nc")
+        plugins.guess_engine(str(nc_file))
 
 
 @mock.patch(
     "xarray.backends.plugins.list_engines",
     mock.MagicMock(return_value={}),
 )
-def test_engines_not_installed() -> None:
-    with pytest.raises(ValueError, match=r"xarray is unable to open"):
+def test_engines_not_installed(tmp_path) -> None:
+    # Non-existent local file raises FileNotFoundError
+    with pytest.raises(FileNotFoundError, match=r"No such file"):
         plugins.guess_engine("not-valid")
 
+    # Existing file with no matching engine raises ValueError
+    existing_file = tmp_path / "test.unknown"
+    existing_file.write_bytes(b"")
+    with pytest.raises(ValueError, match=r"xarray is unable to open"):
+        plugins.guess_engine(str(existing_file))
+
+    # Existing file with recognized magic number raises ValueError
+    nc_file = tmp_path / "foo.nc"
+    nc_file.write_bytes(b"CDF\x01\x00\x00\x00\x00")
     with pytest.raises(ValueError, match=r"found the following matches with the input"):
-        plugins.guess_engine("foo.nc")
+        plugins.guess_engine(str(nc_file))
+
+
+@mock.patch(
+    "xarray.backends.plugins.list_engines",
+    mock.MagicMock(return_value={"dummy": DummyBackendEntrypointArgs()}),
+)
+def test_guess_engine_file_not_found() -> None:
+    # Non-existent local file path (string)
+    with pytest.raises(
+        FileNotFoundError, match=r"No such file: '/nonexistent/path.h5'"
+    ):
+        plugins.guess_engine("/nonexistent/path.h5")
+
+    # Non-existent local file path (PathLike)
+    from pathlib import Path
+
+    with pytest.raises(FileNotFoundError, match=r"No such file"):
+        plugins.guess_engine(Path("/nonexistent/path.h5"))
+
+    # Remote URIs should not raise FileNotFoundError (raises ValueError instead)
+    with pytest.raises(ValueError):
+        plugins.guess_engine("https://example.com/missing.h5")
 
 
 @pytest.mark.parametrize("engine", common.BACKEND_ENTRYPOINTS.keys())