Skip to content

Commit

Permalink
chore: langchain fs upath
Browse files Browse the repository at this point in the history
  • Loading branch information
phil65 committed Feb 24, 2025
1 parent 745d521 commit 686ae37
Showing 1 changed file with 23 additions and 1 deletion.
24 changes: 23 additions & 1 deletion src/llmling_agent_resources/langchain_fs.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,30 @@

import fsspec
from fsspec.asyn import AsyncFileSystem, sync_wrapper
from upath import UPath, registry


if TYPE_CHECKING:
from langchain.document_loaders import BaseLoader
from langchain.schema import Document


class LangchainPath(UPath):
"""UPath implementation for browsing Langchain document loaders."""

__slots__ = ()

def iterdir(self):
if not self.is_dir():
raise NotADirectoryError(str(self))
yield from super().iterdir()

@property
def path(self) -> str:
path = super().path
return "/" if path == "." else path


class LangChainFileSystem(AsyncFileSystem):
"""Filesystem interface for LangChain document loaders."""

Expand Down Expand Up @@ -43,6 +60,10 @@ def __init__(
self._documents: dict[str, Document] = {}
self._loaded = False

def _make_path(self, path: str) -> UPath:
"""Create a path object from string."""
return LangchainPath(path)

async def _load_documents(self) -> None:
"""Load documents if not already loaded."""
if self._loaded:
Expand Down Expand Up @@ -144,7 +165,8 @@ def ukey(self, path: str) -> str:


# Register the filesystem implementation
fsspec.register_implementation("langchain", LangChainFileSystem)
fsspec.register_implementation("langchain", LangChainFileSystem, clobber=True)
registry.register_implementation("langchain", LangchainPath, clobber=True)


if __name__ == "__main__":
Expand Down

0 comments on commit 686ae37

Please sign in to comment.