From 9020d1a1a754bdf1a2fffcd2d08bc6afed38d8a2 Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Mon, 30 Dec 2024 20:22:44 -0300 Subject: [PATCH 1/6] feat: add fsspek client-side backend for remote files service --- .../remoteclient/api/jupyterhub/files.py | 263 ++++++++++++++++++ 1 file changed, 263 insertions(+) create mode 100644 spyder/plugins/remoteclient/api/jupyterhub/files.py diff --git a/spyder/plugins/remoteclient/api/jupyterhub/files.py b/spyder/plugins/remoteclient/api/jupyterhub/files.py new file mode 100644 index 00000000000..39dca8315ea --- /dev/null +++ b/spyder/plugins/remoteclient/api/jupyterhub/files.py @@ -0,0 +1,263 @@ +import json +import requests +import base64 +from websockets.sync.client import connect + +from fsspec.utils import stringify_path +from fsspec.spec import AbstractBufferedFile, AbstractFileSystem + + +class RemoteBufferedFile(AbstractBufferedFile): + """ + A buffered file-like object for reading/writing over the + WebSocket protocol. Inherit from AbstractBufferedFile, which + handles the high-level read/write logic, and only supply the + low-level chunk ops: + - _fetch_range() + - _initiate_upload() + - _upload_chunk() + - _finalize_upload() + """ + + def _fetch_range(self, start, end, **kwargs): + """ + Download and return [start:end] of the underlying file, as bytes. + Open a sync WebSocket, send a read_block request, then gather. + """ + length = end - start + if length < 0: + return b"" + + # Connect to the server’s WebSocket + with connect(self.fs.ws_url) as ws: + # Send JSON request: method=read_block + # Note: if offset or length is 0, the server should handle gracefully. + msg = { + "method": "read_block", + "args": [self.path, start, length], + "kwargs": {} # e.g. delimiter, if needed + } + ws.send_or_raise(self.fs._encode_json(msg)) + + # Now collect base64 chunks + data = b"" + while True: + resp_raw = ws.recv() + resp = self._decode_json(resp_raw) + + if "error" in resp: + raise RuntimeError(f"Server error: {resp['error']}") + msg_type = resp.get("type") + + if msg_type == "data": + chunk_b = base64.b64decode(resp["data"]) + data += chunk_b + elif msg_type == "eof": + break + else: + raise ValueError(f"Unexpected message type: {msg_type}") + return data + + def _initiate_upload(self): + """ + Called once when opening in write mode before writing the first data. + Open a new WebSocket, announce "write_file", and keep the socket open + for streaming chunks. Store the socket in self._ws. + """ + self._ws = connect(self.fs.ws_url) + # Announce write_file + msg = { + "method": "write_file", + "args": [self.path], + "kwargs": {} + } + self._ws.send_or_raise(self.fs._encode_json(msg)) + + def _upload_chunk(self, final=False): + """ + Called when the buffer is flushed to the backend. Encode self.buffer and + send it as {"type": "data", "data": }. If `final=True`, don't do + anything special here; the final call is in _finalize_upload(). + """ + if not self.buffer: + return + chunk_b64 = base64.b64encode(self.buffer).decode() + msg = {"type": "data", "data": chunk_b64} + self._ws.send_or_raise(self._encode_json(msg)) + + def _finalize_upload(self): + """ + Called once after all writing is done. Send 'eof', + then wait for "write_complete". + """ + # Send 'eof' + eof_msg = {"type": "eof"} + self._ws.send_or_raise(self._encode_json(eof_msg)) + + # Wait for server's final response + while True: + try: + resp_raw = self._ws.recv() + except: + break # socket closed? + resp = self._decode_json(resp_raw) + if "error" in resp: + raise RuntimeError(f"Server error: {resp['error']}") + if resp.get("type") == "write_complete": + break + + # Close the socket + self._ws.close() + + def close(self): + """ + Overridden close to ensure final flush (if writing). + """ + super().close() # calls self.flush(force=True) -> calls _finalize_upload() + # Additional logic if needed + + def _encode_json(data: dict) -> str: + """ + Encode a JSON-serializable dict as a string. + """ + return json.dumps(data) + + def _decode_json(data: str) -> dict: + """ + Decode a JSON string to a dict. + """ + return json.loads(data) + +class RemoteFileSystem(AbstractFileSystem): + """ + An example custom FileSystem that: + - uses REST endpoints for all metadata operations + - uses a WebSocket for chunked read/write + """ + cachable = False + + def __init__(self, rest_url, ws_url, *args, **kwargs): + super().__init__(*args, **kwargs) + self.rest_url = rest_url.rstrip("/") + self.ws_url = ws_url + + # Typically you’d keep a session for performance + self._session = requests.Session() + + # ---------------------------------------------------------------- + # Internally used helper for REST calls + # ---------------------------------------------------------------- + def _rest_get(self, endpoint, params=None): + url = f"{self.rest_url}/{endpoint}" + r = self._session.get(url, params=params) + r.raise_for_status() + return r.json() + + def _rest_post(self, endpoint, params=None): + url = f"{self.rest_url}/{endpoint}" + r = self._session.post(url, params=params) + r.raise_for_status() + return r.json() + + def _rest_delete(self, endpoint, params=None): + url = f"{self.rest_url}/{endpoint}" + r = self._session.delete(url, params=params) + r.raise_for_status() + return r.json() + + # ---------------------------------------------------------------- + # fsspec-required API: metadata + # ---------------------------------------------------------------- + def ls(self, path, detail=True, **kwargs): + """List files at a path.""" + params = {"path": path, "detail": str(detail).lower()} + out = self._rest_get("ls", params=params) + return out + + def info(self, path, **kwargs): + """Get info about a single file/directory.""" + params = {"path": path} + out = self._rest_get("info", params=params) + return out + + def isfile(self, path): + params = {"path": path} + out = self._rest_get("isfile", params=params) + return out["isfile"] + + def isdir(self, path): + params = {"path": path} + out = self._rest_get("isdir", params=params) + return out["isdir"] + + def exists(self, path): + params = {"path": path} + out = self._rest_get("exists", params=params) + return out["exists"] + + def mkdir(self, path, create_parents=True, **kwargs): + params = { + "path": path, + "create_parents": str(bool(create_parents)).lower(), + "exist_ok": str(bool(kwargs.get("exist_ok", False))).lower() + } + out = self._rest_post("mkdir", params=params) + return out + + def rmdir(self, path): + params = {"path": path} + out = self._rest_delete("rmdir", params=params) + return out + + def rm_file(self, path, missing_ok=False): + params = {"path": path, "missing_ok": str(bool(missing_ok)).lower()} + out = self._rest_delete("file", params=params) + return out + + def touch(self, path, truncate=True, **kwargs): + params = {"path": path, "truncate": str(bool(truncate)).lower()} + out = self._rest_post("touch", params=params) + return out + + # ---------------------------------------------------------------- + # fsspec open/read/write + # ---------------------------------------------------------------- + def _open(self, path, mode="rb", block_size=2**20, **kwargs): + """ + Return a file-like object that handles reading/writing with WebSocket. + + Parameters + ---------- + path : str + Path to the file. + mode : str + File mode, e.g., 'rb', 'wb', 'ab', 'r+b', etc. + block_size : int + Chunk size for reading/writing. Default is 1MB. + """ + return RemoteBufferedFile( + fs=self, + path=stringify_path(path), + mode=mode, + block_size=block_size, + **kwargs + ) + + def cat_file(self, path, start=None, end=None, **kwargs): + """ + Read entire file (or partial range) from WebSocket and return bytes. + """ + # A straightforward approach: open in read mode, read the data + with self._open(path, mode="rb") as f: + if start: + f.seek(start) + length = None + if end is not None and start is not None: + length = end - start + out = f.read(length) + return out + + def pipe_file(self, path, data, **kwargs): + """Write bytes to a file (full overwrite).""" + with self._open(path, mode="wb") as f: + f.write(data) From 60c68c1b51d8955c84e4c68155bb2dfd7df5e9b6 Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Mon, 30 Dec 2024 20:35:39 -0300 Subject: [PATCH 2/6] git subrepo pull (merge) external-deps/spyder-kernels subrepo: subdir: "external-deps/spyder-kernels" merged: "d7e4319b5" upstream: origin: "https://github.com/spyder-ide/spyder-kernels.git" branch: "master" commit: "d7e4319b5" git-subrepo: version: "0.4.9" origin: "https://github.com/ingydotnet/git-subrepo" commit: "cce3d93" --- external-deps/spyder-kernels/.gitrepo | 6 +++--- external-deps/spyder-kernels/RELEASE.md | 12 +++--------- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 4f04f00a36f..1d0f53f9d60 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/spyder-ide/spyder-kernels.git branch = master - commit = 07f24b6fde55585e64d1802036efba19514dbf0c - parent = 41c288acf1c34b7cd9f1175b4f0fa8233692a26c + commit = d7e4319b56d98e60d481e1c096a8b4718e5f68a7 + parent = 9020d1a1a754bdf1a2fffcd2d08bc6afed38d8a2 method = merge - cmdver = 0.4.3 + cmdver = 0.4.9 diff --git a/external-deps/spyder-kernels/RELEASE.md b/external-deps/spyder-kernels/RELEASE.md index c2aab89faa6..b6acedaed28 100644 --- a/external-deps/spyder-kernels/RELEASE.md +++ b/external-deps/spyder-kernels/RELEASE.md @@ -6,10 +6,10 @@ To release a new version of spyder-kernels on PyPI: * git fetch upstream && get merge upstream/3.x -* git clean -xfdi - * Update CHANGELOG.md with `loghub spyder-ide/spyder-kernels -m vX.X.X` +* git clean -xfdi + * Update `_version.py` (set release version, remove 'dev0') * git add . && git commit -m 'Release X.X.X' @@ -24,16 +24,10 @@ To release a new version of spyder-kernels on PyPI: * git tag -a vX.X.X -m 'Release X.X.X' -* Update `_version.py` (add 'dev0' and increment minor) +* Update `_version.py` (add 'dev0' and increment patch) * git add . && git commit -m 'Back to work' -* git checkout master - -* git merge 3.x - -* git push upstream master - * git push upstream 3.x * git push upstream --tags From 39a17e3c40518ad4d2dadf2a9f894dfeef5bb187 Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Mon, 30 Dec 2024 20:42:15 -0300 Subject: [PATCH 3/6] feat: add fsspec dependency --- binder/environment.yml | 1 + requirements/main.yml | 1 + setup.py | 1 + spyder/dependencies.py | 5 +++++ 4 files changed, 8 insertions(+) diff --git a/binder/environment.yml b/binder/environment.yml index bb5de9036cd..e67adbe2481 100644 --- a/binder/environment.yml +++ b/binder/environment.yml @@ -16,6 +16,7 @@ dependencies: - cookiecutter >=1.6.0 - diff-match-patch >=20181111 - fcitx-qt5 >=1.2.7 +- fsspec >= 2021.10.0 - fzf >=0.42.0 - importlib-metadata >=4.6.0 - intervaltree >=3.0.2 diff --git a/requirements/main.yml b/requirements/main.yml index 7371c0ecdbe..dfc8d2ca36c 100644 --- a/requirements/main.yml +++ b/requirements/main.yml @@ -13,6 +13,7 @@ dependencies: - cloudpickle >=0.5.0 - cookiecutter >=1.6.0 - diff-match-patch >=20181111 + - fsspec >= 2021.10.0 - fzf >=0.42.0 # Need at least some compatibility with python 3.10 features - importlib-metadata >=4.6.0 diff --git a/setup.py b/setup.py index 06039b9d6f2..ff4f4ca66d1 100644 --- a/setup.py +++ b/setup.py @@ -272,6 +272,7 @@ def run(self): 'cloudpickle>=0.5.0', 'cookiecutter>=1.6.0', 'diff-match-patch>=20181111', + 'fsspec>=2021.10.0', # While this is only required for python <3.10, it is safe enough to # install in all cases and helps the tests to pass. 'importlib-metadata>=4.6.0', diff --git a/spyder/dependencies.py b/spyder/dependencies.py index 67eea346d2b..2a886797891 100644 --- a/spyder/dependencies.py +++ b/spyder/dependencies.py @@ -42,6 +42,7 @@ CLOUDPICKLE_REQVER = '>=0.5.0' COOKIECUTTER_REQVER = '>=1.6.0' DIFF_MATCH_PATCH_REQVER = '>=20181111' +FSSPEC_REQVER = '>=2021.10.0' IMPORTLIB_METADATA_REQVER = '>=4.6.0' INTERVALTREE_REQVER = '>=3.0.2' IPYTHON_REQVER = ">=8.12.2,<8.13.0" if PY38 else ">=8.13.0,<9.0.0,!=8.17.1" @@ -130,6 +131,10 @@ 'package_name': "diff-match-patch", 'features': _("Compute text file diff changes during edition"), 'required_version': DIFF_MATCH_PATCH_REQVER}, + {'modname': "fsspec", + 'package_name': "fsspec", + 'features': _("File system abstraction layer for remote file systems"), + 'required_version': FSSPEC_REQVER}, {'modname': 'importlib_metadata', 'package_name': 'importlib-metadata', 'features': _('Access the metadata for a Python package'), From 737300c5fa4707720e9fe88ece216e43b510fd2c Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Thu, 9 Jan 2025 16:35:33 -0300 Subject: [PATCH 4/6] feat: refactor jupyter's rest api for plugin support --- spyder/plugins/remoteclient/api/client.py | 2 +- .../remoteclient/api/jupyterhub/files.py | 263 ------------------ .../plugins/remoteclient/api/rest/__init__.py | 0 .../api/{jupyterhub => rest}/auth.py | 0 .../{jupyterhub/__init__.py => rest/base.py} | 67 ++++- .../api/{jupyterhub => rest}/execute.py | 4 +- .../api/{jupyterhub => rest}/utils.py | 0 7 files changed, 68 insertions(+), 268 deletions(-) delete mode 100644 spyder/plugins/remoteclient/api/jupyterhub/files.py create mode 100644 spyder/plugins/remoteclient/api/rest/__init__.py rename spyder/plugins/remoteclient/api/{jupyterhub => rest}/auth.py (100%) rename spyder/plugins/remoteclient/api/{jupyterhub/__init__.py => rest/base.py} (89%) rename spyder/plugins/remoteclient/api/{jupyterhub => rest}/execute.py (97%) rename spyder/plugins/remoteclient/api/{jupyterhub => rest}/utils.py (100%) diff --git a/spyder/plugins/remoteclient/api/client.py b/spyder/plugins/remoteclient/api/client.py index f52ed3b907a..ec92db12589 100644 --- a/spyder/plugins/remoteclient/api/client.py +++ b/spyder/plugins/remoteclient/api/client.py @@ -19,7 +19,7 @@ SPYDER_REMOTE_MAX_VERSION, SPYDER_REMOTE_MIN_VERSION, ) -from spyder.plugins.remoteclient.api.jupyterhub import JupyterAPI +from spyder.plugins.remoteclient.api.rest.base import JupyterAPI from spyder.plugins.remoteclient.api.protocol import ( ConnectionInfo, ConnectionStatus, diff --git a/spyder/plugins/remoteclient/api/jupyterhub/files.py b/spyder/plugins/remoteclient/api/jupyterhub/files.py deleted file mode 100644 index 39dca8315ea..00000000000 --- a/spyder/plugins/remoteclient/api/jupyterhub/files.py +++ /dev/null @@ -1,263 +0,0 @@ -import json -import requests -import base64 -from websockets.sync.client import connect - -from fsspec.utils import stringify_path -from fsspec.spec import AbstractBufferedFile, AbstractFileSystem - - -class RemoteBufferedFile(AbstractBufferedFile): - """ - A buffered file-like object for reading/writing over the - WebSocket protocol. Inherit from AbstractBufferedFile, which - handles the high-level read/write logic, and only supply the - low-level chunk ops: - - _fetch_range() - - _initiate_upload() - - _upload_chunk() - - _finalize_upload() - """ - - def _fetch_range(self, start, end, **kwargs): - """ - Download and return [start:end] of the underlying file, as bytes. - Open a sync WebSocket, send a read_block request, then gather. - """ - length = end - start - if length < 0: - return b"" - - # Connect to the server’s WebSocket - with connect(self.fs.ws_url) as ws: - # Send JSON request: method=read_block - # Note: if offset or length is 0, the server should handle gracefully. - msg = { - "method": "read_block", - "args": [self.path, start, length], - "kwargs": {} # e.g. delimiter, if needed - } - ws.send_or_raise(self.fs._encode_json(msg)) - - # Now collect base64 chunks - data = b"" - while True: - resp_raw = ws.recv() - resp = self._decode_json(resp_raw) - - if "error" in resp: - raise RuntimeError(f"Server error: {resp['error']}") - msg_type = resp.get("type") - - if msg_type == "data": - chunk_b = base64.b64decode(resp["data"]) - data += chunk_b - elif msg_type == "eof": - break - else: - raise ValueError(f"Unexpected message type: {msg_type}") - return data - - def _initiate_upload(self): - """ - Called once when opening in write mode before writing the first data. - Open a new WebSocket, announce "write_file", and keep the socket open - for streaming chunks. Store the socket in self._ws. - """ - self._ws = connect(self.fs.ws_url) - # Announce write_file - msg = { - "method": "write_file", - "args": [self.path], - "kwargs": {} - } - self._ws.send_or_raise(self.fs._encode_json(msg)) - - def _upload_chunk(self, final=False): - """ - Called when the buffer is flushed to the backend. Encode self.buffer and - send it as {"type": "data", "data": }. If `final=True`, don't do - anything special here; the final call is in _finalize_upload(). - """ - if not self.buffer: - return - chunk_b64 = base64.b64encode(self.buffer).decode() - msg = {"type": "data", "data": chunk_b64} - self._ws.send_or_raise(self._encode_json(msg)) - - def _finalize_upload(self): - """ - Called once after all writing is done. Send 'eof', - then wait for "write_complete". - """ - # Send 'eof' - eof_msg = {"type": "eof"} - self._ws.send_or_raise(self._encode_json(eof_msg)) - - # Wait for server's final response - while True: - try: - resp_raw = self._ws.recv() - except: - break # socket closed? - resp = self._decode_json(resp_raw) - if "error" in resp: - raise RuntimeError(f"Server error: {resp['error']}") - if resp.get("type") == "write_complete": - break - - # Close the socket - self._ws.close() - - def close(self): - """ - Overridden close to ensure final flush (if writing). - """ - super().close() # calls self.flush(force=True) -> calls _finalize_upload() - # Additional logic if needed - - def _encode_json(data: dict) -> str: - """ - Encode a JSON-serializable dict as a string. - """ - return json.dumps(data) - - def _decode_json(data: str) -> dict: - """ - Decode a JSON string to a dict. - """ - return json.loads(data) - -class RemoteFileSystem(AbstractFileSystem): - """ - An example custom FileSystem that: - - uses REST endpoints for all metadata operations - - uses a WebSocket for chunked read/write - """ - cachable = False - - def __init__(self, rest_url, ws_url, *args, **kwargs): - super().__init__(*args, **kwargs) - self.rest_url = rest_url.rstrip("/") - self.ws_url = ws_url - - # Typically you’d keep a session for performance - self._session = requests.Session() - - # ---------------------------------------------------------------- - # Internally used helper for REST calls - # ---------------------------------------------------------------- - def _rest_get(self, endpoint, params=None): - url = f"{self.rest_url}/{endpoint}" - r = self._session.get(url, params=params) - r.raise_for_status() - return r.json() - - def _rest_post(self, endpoint, params=None): - url = f"{self.rest_url}/{endpoint}" - r = self._session.post(url, params=params) - r.raise_for_status() - return r.json() - - def _rest_delete(self, endpoint, params=None): - url = f"{self.rest_url}/{endpoint}" - r = self._session.delete(url, params=params) - r.raise_for_status() - return r.json() - - # ---------------------------------------------------------------- - # fsspec-required API: metadata - # ---------------------------------------------------------------- - def ls(self, path, detail=True, **kwargs): - """List files at a path.""" - params = {"path": path, "detail": str(detail).lower()} - out = self._rest_get("ls", params=params) - return out - - def info(self, path, **kwargs): - """Get info about a single file/directory.""" - params = {"path": path} - out = self._rest_get("info", params=params) - return out - - def isfile(self, path): - params = {"path": path} - out = self._rest_get("isfile", params=params) - return out["isfile"] - - def isdir(self, path): - params = {"path": path} - out = self._rest_get("isdir", params=params) - return out["isdir"] - - def exists(self, path): - params = {"path": path} - out = self._rest_get("exists", params=params) - return out["exists"] - - def mkdir(self, path, create_parents=True, **kwargs): - params = { - "path": path, - "create_parents": str(bool(create_parents)).lower(), - "exist_ok": str(bool(kwargs.get("exist_ok", False))).lower() - } - out = self._rest_post("mkdir", params=params) - return out - - def rmdir(self, path): - params = {"path": path} - out = self._rest_delete("rmdir", params=params) - return out - - def rm_file(self, path, missing_ok=False): - params = {"path": path, "missing_ok": str(bool(missing_ok)).lower()} - out = self._rest_delete("file", params=params) - return out - - def touch(self, path, truncate=True, **kwargs): - params = {"path": path, "truncate": str(bool(truncate)).lower()} - out = self._rest_post("touch", params=params) - return out - - # ---------------------------------------------------------------- - # fsspec open/read/write - # ---------------------------------------------------------------- - def _open(self, path, mode="rb", block_size=2**20, **kwargs): - """ - Return a file-like object that handles reading/writing with WebSocket. - - Parameters - ---------- - path : str - Path to the file. - mode : str - File mode, e.g., 'rb', 'wb', 'ab', 'r+b', etc. - block_size : int - Chunk size for reading/writing. Default is 1MB. - """ - return RemoteBufferedFile( - fs=self, - path=stringify_path(path), - mode=mode, - block_size=block_size, - **kwargs - ) - - def cat_file(self, path, start=None, end=None, **kwargs): - """ - Read entire file (or partial range) from WebSocket and return bytes. - """ - # A straightforward approach: open in read mode, read the data - with self._open(path, mode="rb") as f: - if start: - f.seek(start) - length = None - if end is not None and start is not None: - length = end - start - out = f.read(length) - return out - - def pipe_file(self, path, data, **kwargs): - """Write bytes to a file (full overwrite).""" - with self._open(path, mode="wb") as f: - f.write(data) diff --git a/spyder/plugins/remoteclient/api/rest/__init__.py b/spyder/plugins/remoteclient/api/rest/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/spyder/plugins/remoteclient/api/jupyterhub/auth.py b/spyder/plugins/remoteclient/api/rest/auth.py similarity index 100% rename from spyder/plugins/remoteclient/api/jupyterhub/auth.py rename to spyder/plugins/remoteclient/api/rest/auth.py diff --git a/spyder/plugins/remoteclient/api/jupyterhub/__init__.py b/spyder/plugins/remoteclient/api/rest/base.py similarity index 89% rename from spyder/plugins/remoteclient/api/jupyterhub/__init__.py rename to spyder/plugins/remoteclient/api/rest/base.py index 020779df554..08c64655763 100644 --- a/spyder/plugins/remoteclient/api/jupyterhub/__init__.py +++ b/spyder/plugins/remoteclient/api/rest/base.py @@ -3,7 +3,7 @@ # Copyright © Spyder Project Contributors # Licensed under the terms of the MIT License # (see spyder/__init__.py for details) - +from abc import abstractmethod import uuid import logging import time @@ -12,7 +12,8 @@ import yarl import aiohttp -from spyder.plugins.remoteclient.api.jupyterhub import auth +from spyder.api.utils import ABCMeta, abstract_attribute +from spyder.plugins.remoteclient.api.rest import auth logger = logging.getLogger(__name__) @@ -470,3 +471,65 @@ async def send_code(self, username, code, wait=True, timeout=None): # cell did not produce output elif msg["content"].get("execution_state") == "idle": return "" + + +class JupyterPluginBaseAPI(metaclass=ABCMeta): + """ + Base class for Jupyter API plugins. + + This class is must be subclassed to implement the API for a specific + Jupyter extension. Provides a context manager for the API session. + + Class Attributes + ---------- + base_url: str + The base URL for the Jupyter Extension's rest API. + + Attributes + ---------- + api_url: yarl.URL + The full URL for the rest. + + api_token: str + The API token for the Jupyter API. + + verify_ssl: bool + Whether to verify SSL certificates. + + session: aiohttp.ClientSession + The session for the Jupyter API requests. + """ + + @abstract_attribute + def base_url(self): + ... + + def __init__(self, hub_url, api_token, verify_ssl=True): + self.api_url = yarl.URL(hub_url) / self.base_url + self.api_token = api_token + self.verify_ssl = verify_ssl + + @classmethod + async def new(cls, hub_url, api_token, verify_ssl=True): + self = cls(hub_url, api_token, verify_ssl=verify_ssl) + return await self.__aenter__() + + async def close(self): + await self.__aexit__(None, None, None) + + async def __aenter__(self): + self.session = aiohttp.ClientSession( + headers={"Authorization": f"token {self.api_token}"}, + connector=aiohttp.TCPConnector( + ssl=None if self.verify_ssl else False + ), + raise_for_status = self._raise_for_status, + ) + return self + + async def __aexit__(self, exc_type, exc, tb): + await self.session.close() + + @abstractmethod + async def _raise_for_status(self, response: aiohttp.ClientResponse): + ... diff --git a/spyder/plugins/remoteclient/api/jupyterhub/execute.py b/spyder/plugins/remoteclient/api/rest/execute.py similarity index 97% rename from spyder/plugins/remoteclient/api/jupyterhub/execute.py rename to spyder/plugins/remoteclient/api/rest/execute.py index 360bb9fcd8a..5484740f6e6 100644 --- a/spyder/plugins/remoteclient/api/jupyterhub/execute.py +++ b/spyder/plugins/remoteclient/api/rest/execute.py @@ -9,8 +9,8 @@ import logging import textwrap -from spyder.plugins.remoteclient.api.jupyterhub import JupyterHubAPI -from spyder.plugins.remoteclient.api.jupyterhub.utils import ( +from spyder.plugins.remoteclient.api.rest.base import JupyterHubAPI +from spyder.plugins.remoteclient.api.rest.utils import ( parse_notebook_cells, ) diff --git a/spyder/plugins/remoteclient/api/jupyterhub/utils.py b/spyder/plugins/remoteclient/api/rest/utils.py similarity index 100% rename from spyder/plugins/remoteclient/api/jupyterhub/utils.py rename to spyder/plugins/remoteclient/api/rest/utils.py From 045c53efe1779ef2058d94cc0bece78eab5d6531 Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Thu, 9 Jan 2025 16:36:30 -0300 Subject: [PATCH 5/6] feat: add rest plugin for spyder remote files api --- .../plugins/remoteclient/api/rest/plugin.py | 262 ++++++++++++++++++ 1 file changed, 262 insertions(+) create mode 100644 spyder/plugins/remoteclient/api/rest/plugin.py diff --git a/spyder/plugins/remoteclient/api/rest/plugin.py b/spyder/plugins/remoteclient/api/rest/plugin.py new file mode 100644 index 00000000000..a5081863a8f --- /dev/null +++ b/spyder/plugins/remoteclient/api/rest/plugin.py @@ -0,0 +1,262 @@ +from __future__ import annotations +import base64 +from http import HTTPStatus +import json +from pathlib import Path + +import aiohttp + +from spyder.plugins.remoteclient.api.rest.base import JupyterPluginBaseAPI + + +SPYDER_PLUGIN_NAME = "spyder-services" # jupyter server's extension name for spyder-remote-services + + +class SpyderServicesError(Exception): + ... + +class RemoteFileServicesError(SpyderServicesError): + def __init__(self, + type, + message, + url, + tracebacks): + self.type = type + self.message = message + self.url = url + self.tracebacks = tracebacks + + def __str__(self): + return f"(type='{self.type}', message='{self.message}', url='{self.url}')" + +class RemoteOSError(OSError, RemoteFileServicesError): + def __init__(self, *args): + super(RemoteFileServicesError, self).__init__(*args) + + @classmethod + def from_json(cls, data, url): + err = cls(data["message"], url) + err.errno = data["errno"] + return err + + def __str__(self): + return f'{self.args[0]} ({self.args[1]})' + + +class SpyderRemoteFileAPI: + def __init__(self, path, mode="r", atomic=False, lock=False, encoding="utf-8"): + self.path = path + self.mode = mode + self.encoding = encoding + self.atomic = atomic + self.lock = lock + + self._websocket: aiohttp.ClientWebSocketResponse = None + + async def connect(self, session: aiohttp.ClientSession, api_url): + self._websocket = await session.ws_connect(api_url / "open" / f"file://{self.path}", + params={"mode": self.mode, + "atomic": str(self.atomic).lower(), + "lock": str(self.lock).lower(), + "encoding": self.encoding}) + await self._check_connection() + + async def _check_connection(self): + status = await self._websocket.receive() + + if status.type == aiohttp.WSMsgType.CLOSE: + await self._websocket.close() + if status.data == 1002: + data = json.loads(status.extra) + if data["status"] in (HTTPStatus.LOCKED, HTTPStatus.EXPECTATION_FAILED): + raise RemoteOSError.from_json(data, url=self._websocket._response.url) + + raise RemoteFileServicesError( + data.get("type", "UnknownError"), + data.get("message", "Unknown error"), + self._websocket._response.url, + data.get("tracebacks", []), + ) + else: + raise RemoteFileServicesError("UnknownError", "Failed to open file", self._websocket._response.url, []) + + async def close(self): + await self._websocket.close() + + def _decode_data(self, data: str | object) -> str | bytes | object: + """Decode data from a message.""" + if not isinstance(data, str): + return data + + if "b" in self.mode: + return base64.b64decode(data) + + return base64.b64decode(data).decode(self.encoding) + + def _encode_data(self, data: bytes | str | object) -> str: + """Encode data for a message.""" + if isinstance(data, bytes): + return base64.b64encode(data).decode("ascii") + if isinstance(data, str): + return base64.b64encode(data.encode(self.encoding)).decode("ascii") + return data + + async def _send_request(self, method: str, **args): + await self._websocket.send_json({"method": method, **args}) + + async def _get_response(self, timeout=None): + message = json.loads(await self._websocket.receive_bytes(timeout=timeout)) + + if message["status"] > 400: + if message["status"] == HTTPStatus.EXPECTATION_FAILED: + raise RemoteOSError.from_json(message, url=self._websocket._response.url) + + raise RemoteFileServicesError( + message.get("type", "UnknownError"), + message.get("message", "Unknown error"), + self._websocket._response.url, + message.get("tracebacks", []), + ) + data = message.get("data") + if data is None: + return None + + if isinstance(data, list): + return [self._decode_data(d) for d in data] + + return self._decode_data(data) + + async def write(self, data: bytes | str) -> int: + """Write data to the file.""" + await self._send_request("write", data=self._encode_data(data)) + return await self._get_response() + + async def flush(self): + """Flush the file.""" + await self._send_request("flush") + return await self._get_response() + + async def read(self, n: int = -1) -> bytes | str: + """Read data from the file.""" + await self._send_request("read", n=n) + return await self._get_response() + + async def seek(self, offset: int, whence: int = 0) -> int: + """Seek to a new position in the file.""" + await self._send_request("seek", offset=offset, whence=whence) + return await self._get_response() + + async def tell(self) -> int: + """Get the current file position.""" + await self._send_request("tell") + return await self._get_response() + + async def truncate(self, size: int | None = None) -> int: + """Truncate the file to a new size.""" + await self._send_request("truncate", size=size) + return await self._get_response() + + async def fileno(self): + """Flush the file to disk.""" + await self._send_request("fileno") + return await self._get_response() + + async def readline(self, size: int = -1) -> bytes | str: + """Read a line from the file.""" + await self._send_request("readline", size=size) + return await self._get_response() + + async def readlines(self, hint: int = -1) -> list[bytes | str]: + """Read lines from the file.""" + await self._send_request("readlines", hint=hint) + return await self._get_response() + + async def writelines(self, lines: list[bytes | str]): + """Write lines to the file.""" + await self._send_request("writelines", lines=[self._encode_data(l) for l in lines]) + return await self._get_response() + + async def isatty(self) -> bool: + """Check if the file is a TTY.""" + await self._send_request("isatty") + return await self._get_response() + + async def readable(self) -> bool: + """Check if the file is readable.""" + await self._send_request("readable") + return await self._get_response() + + async def writable(self) -> bool: + """Check if the file is writable.""" + await self._send_request("writable") + return await self._get_response() + + +class SpyderRemoteServicesFileAPI(JupyterPluginBaseAPI): + base_url = SPYDER_PLUGIN_NAME + "/fsspec" + + async def _raise_for_status(self, response): + if response.status == 500: + try: + data = await response.json() + except json.JSONDecodeError: + data = {} + + # If we're in a context we can rely on __aexit__() to release as the + # exception propagates. + if not response._in_context: + response.release() + + raise RemoteFileServicesError( + data.get("type", "UnknownError"), + data.get("message", "Unknown error"), + response.url, + data.get("tracebacks", []), + ) + elif not response.ok: + response.raise_for_status() + + async def ls(self, path: Path, detail: bool=True): + async with self.session.get(self.api_url / "ls" / f"file://{path}", params={"detail": str(detail).lower()}) as response: + return await response.json() + + async def info(self, path: Path): + async with self.session.get(self.api_url / "info" / f"file://{path}") as response: + return await response.json() + + async def exists(self, path: Path): + async with self.session.get(self.api_url / "exists" / f"file://{path}") as response: + return await response.json() + + async def isfile(self, path: Path): + async with self.session.get(self.api_url / "isfile" / f"file://{path}") as response: + return await response.json() + + async def isdir(self, path: Path): + async with self.session.get(self.api_url / "isdir" / f"file://{path}") as response: + return await response.json() + + async def mkdir(self, path: Path, create_parents: bool=True, exist_ok: bool=False): + async with self.session.post(self.api_url / "mkdir" / f"file://{path}", + params={"create_parents": str(create_parents).lower(), + "exist_ok": str(exist_ok).lower()}) as response: + return await response.json() + + async def rmdir(self, path: Path): + async with self.session.delete(self.api_url / "rmdir" / f"file://{path}") as response: + return await response.json() + + async def rm_file(self, path: Path, missing_ok: bool=False): + async with self.session.delete(self.api_url / "file" / f"file://{path}", + params={"missing_ok": str(missing_ok).lower()}) as response: + return await response.json() + + async def touch(self, path: Path, truncate: bool=True): + async with self.session.post(self.api_url / "touch" / f"file://{path}", + params={"truncate": str(truncate).lower()}) as response: + return await response.json() + + async def open(self, path: Path, mode: str="r", atomic: bool=False, lock: bool=False, encoding: str='utf-8'): + api = SpyderRemoteFileAPI(path, mode, atomic, lock, encoding) + await api.connect(self.session, self.api_url) + return api From 6db9945ff7840cd1633458b9831edad01d2bd31c Mon Sep 17 00:00:00 2001 From: Hendrik Dumith Louzada Date: Thu, 9 Jan 2025 16:37:12 -0300 Subject: [PATCH 6/6] feat: add custom metaclass for not implemented attributes --- spyder/api/utils.py | 63 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/spyder/api/utils.py b/spyder/api/utils.py index c62c562802b..2486a64a8c3 100644 --- a/spyder/api/utils.py +++ b/spyder/api/utils.py @@ -8,6 +8,7 @@ """ API utilities. """ +from abc import ABCMeta as BaseABCMeta def get_class_values(cls): @@ -64,3 +65,65 @@ class classproperty(property): def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() + + +class DummyAttribute: + """ + Dummy class to mark abstract attributes. + """ + pass + + +def abstract_attribute(obj=None): + """ + Decorator to mark abstract attributes. Must be used in conjunction with the + ABCMeta metaclass. + """ + if obj is None: + obj = DummyAttribute() + obj.__is_abstract_attribute__ = True + return obj + + +class ABCMeta(BaseABCMeta): + """ + Metaclass to mark abstract classes. + + Adds support for abstract attributes. If a class has abstract attributes + and is instantiated, a NotImplementedError is raised. + + Usage + ----- + class MyABC(metaclass=ABCMeta): + @abstract_attribute + def my_abstract_attribute(self): + pass + + class MyClassOK(MyABC): + def __init__(self): + self.my_abstract_attribute = 1 + + class MyClassNotOK(MyABC): + pass + + Raises + ------ + NotImplementedError: Can't instantiate abstract class with abstract attributes. + """ + + def __call__(cls, *args, **kwargs): + instance = BaseABCMeta.__call__(cls, *args, **kwargs) + abstract_attributes = { + name + for name in dir(instance) + if hasattr(getattr(instance, name), '__is_abstract_attribute__') + } + if abstract_attributes: + raise NotImplementedError( + "Can't instantiate abstract class {} with" + " abstract attributes: {}".format( + cls.__name__, + ', '.join(abstract_attributes) + ) + ) + return instance