diff --git a/ci/scripts/test_imports.sh b/ci/scripts/test_imports.sh index 5f310b69..feca005f 100644 --- a/ci/scripts/test_imports.sh +++ b/ci/scripts/test_imports.sh @@ -19,4 +19,5 @@ test_import "aws" "import dask_cloudprovider.aws" test_import "azure" "import dask_cloudprovider.azure" test_import "digitalocean" "import dask_cloudprovider.digitalocean" test_import "gcp" "import dask_cloudprovider.gcp" -test_import "openstack" "import dask_cloudprovider.openstack" \ No newline at end of file +test_import "ibm" "import dask_cloudprovider.ibm" +test_import "openstack" "import dask_cloudprovider.openstack" diff --git a/dask_cloudprovider/cloudprovider.yaml b/dask_cloudprovider/cloudprovider.yaml index 932bea1f..fcf201aa 100755 --- a/dask_cloudprovider/cloudprovider.yaml +++ b/dask_cloudprovider/cloudprovider.yaml @@ -119,6 +119,18 @@ cloudprovider: docker_image: "daskdev/dask:latest" # docker image to use bootstrap: true # It is assumed that the OS image does not have Docker and needs bootstrapping. Set this to false if using a custom image with Docker already installed. + ibm: + api_key: null + image: "ghcr.io/dask/dask:latest" + region: us-east + project_id: null + scheduler_cpu: "1.0" + scheduler_mem: 4G + scheduler_timeout: 600 # seconds + worker_cpu: "2.0" + worker_mem: 8G + worker_threads: 1 + openstack: region: "RegionOne" # The name of the region where resources will be allocated in OpenStack. List available regions using: `openstack region list`. size: null # Openstack flavors define the compute, memory, and storage capacity of computing instances. List available flavors using: `openstack flavor list` @@ -132,4 +144,4 @@ cloudprovider: security_group: null # The security group name that defines firewall rules for instances. List available security groups using: `openstack security group list` external_network_id: null # The ID of the external network used for assigning floating IPs. List available external networks using: `openstack network list --external` create_floating_ip: false # Specifies whether to assign a floating IP to each instance, enabling external access. Set to `True` if external connectivity is needed. - docker_image: "daskdev/dask:latest" # docker image to use \ No newline at end of file + docker_image: "daskdev/dask:latest" # docker image to use diff --git a/dask_cloudprovider/ibm/__init__.py b/dask_cloudprovider/ibm/__init__.py new file mode 100644 index 00000000..f91fab73 --- /dev/null +++ b/dask_cloudprovider/ibm/__init__.py @@ -0,0 +1 @@ +from .code_engine import IBMCodeEngineCluster diff --git a/dask_cloudprovider/ibm/code_engine.py b/dask_cloudprovider/ibm/code_engine.py new file mode 100644 index 00000000..67b4e38f --- /dev/null +++ b/dask_cloudprovider/ibm/code_engine.py @@ -0,0 +1,439 @@ +import json +import time +import urllib3 +import threading +import random + +import dask +from dask_cloudprovider.generic.vmcluster import ( + VMCluster, + VMInterface, + SchedulerMixin, + WorkerMixin, +) + +from distributed.core import Status +from distributed.security import Security + +try: + from ibm_code_engine_sdk.code_engine_v2 import CodeEngineV2 + from ibm_cloud_sdk_core.authenticators import IAMAuthenticator +except ImportError as e: + msg = ( + "Dask Cloud Provider IBM requirements are not installed.\n\n" + "Please either conda or pip install as follows:\n\n" + " conda install -c conda-forge dask-cloudprovider # either conda install\n" + ' pip install "dask-cloudprovider[ibm]" --upgrade # or python -m pip install' + ) + raise ImportError(msg) from e + + +urllib3.disable_warnings() + + +class IBMCodeEngine(VMInterface): + def __init__( + self, + cluster: str, + config, + image: str = None, + region: str = None, + project_id: str = None, + scheduler_cpu: str = None, + scheduler_mem: str = None, + scheduler_timeout: int = None, + worker_cpu: str = None, + worker_mem: str = None, + worker_threads: int = None, + api_key: str = None, + **kwargs, + ): + super().__init__(**kwargs) + self.cluster = cluster + self.config = config + self.image = image + self.region = region + self.project_id = project_id + self.scheduler_cpu = scheduler_cpu + self.scheduler_mem = scheduler_mem + self.scheduler_timeout = scheduler_timeout + self.worker_cpu = worker_cpu + self.worker_mem = worker_mem + self.worker_threads = worker_threads + self.api_key = api_key + + authenticator = IAMAuthenticator(self.api_key, url='https://iam.cloud.ibm.com') + authenticator.set_disable_ssl_verification(True) # Disable SSL verification for the authenticator + + self.code_engine_service = CodeEngineV2(authenticator=authenticator) + self.code_engine_service.set_service_url('https://api.' + self.region + '.codeengine.cloud.ibm.com/v2') + self.code_engine_service.set_disable_ssl_verification(True) # Disable SSL verification for the service instance + + async def create_vm(self): + # Deploy a scheduler on a Code Engine application + # It allows listening on a specific port and exposing it to the public + if "scheduler" in self.name: + self.code_engine_service.create_app( + project_id=self.project_id, + image_reference=self.image, + name=self.name, + run_commands=self.command, + image_port=8786, + scale_cpu_limit=self.cpu, + scale_min_instances=1, + scale_concurrency=1000, + scale_memory_limit=self.memory, + scale_request_timeout=self.cluster.scheduler_timeout, + run_env_variables=[ + { + "type": "literal", + "name": "DASK_INTERNAL_INHERIT_CONFIG", + "key": "DASK_INTERNAL_INHERIT_CONFIG", + "value": dask.config.serialize(dask.config.global_config), + } + ] + ) + + # Create a ConfigMap with the Dask configuration once time + self.code_engine_service.create_config_map( + project_id=self.project_id, + name=self.cluster.uuid, + data={ + "DASK_INTERNAL_INHERIT_CONFIG": dask.config.serialize(dask.config.global_config), + } + ) + + # This loop waits for the app to be ready, then returns the internal and public URLs + while True: + response = self.code_engine_service.get_app( + project_id=self.project_id, + name=self.name, + ) + app = response.get_result() + if app["status"] == "ready": + break + + time.sleep(0.5) + + internal_url = app["endpoint_internal"].split("//")[1] + public_url = app["endpoint"].split("//")[1] + + return internal_url, public_url + + # Deploy a worker on a Code Engine job run + else: + def create_job_run_thread(): + retry_delay = 1 + + # Add an exponential sleep to avoid overloading the Code Engine API + for attempt in range(5): + try: + self.code_engine_service.create_job_run( + project_id=self.project_id, + image_reference=self.image, + name=self.name, + run_commands=self.command, + scale_cpu_limit=self.cpu, + scale_memory_limit=self.memory, + run_env_variables=[ + { + "type": "config_map_key_reference", + "reference": self.cluster.uuid, + "name": "DASK_INTERNAL_INHERIT_CONFIG", + "key": "DASK_INTERNAL_INHERIT_CONFIG", + } + ] + ) + return + except Exception: + time.sleep(retry_delay) + retry_delay *= 2 + retry_delay += random.uniform(0, 1) + + raise Exception("Maximum retry attempts reached") + + # Create a thread to create multiples job runs in parallel + job_run_thread = threading.Thread(target=create_job_run_thread) + job_run_thread.start() + + async def destroy_vm(self): + self.cluster._log(f"Deleting Instance: {self.name}") + + if "scheduler" in self.name: + self.code_engine_service.delete_app( + project_id=self.project_id, + name=self.name, + ) + else: + self.code_engine_service.delete_job_run( + project_id=self.project_id, + name=self.name, + ) + try: + self.code_engine_service.delete_config_map( + project_id=self.project_id, + name=self.cluster.uuid, + ) + except Exception: + pass + + +class IBMCodeEngineScheduler(SchedulerMixin, IBMCodeEngine): + """Scheduler running in a GCP instance.""" + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.cpu = self.cluster.scheduler_cpu + self.memory = self.cluster.scheduler_mem + + self.command = [ + "python", + "-m", + "distributed.cli.dask_scheduler", + "--protocol", + "ws" + ] + + async def start(self): + self.cluster._log( + f"Launching cluster with the following configuration: " + f"\n Source Image: {self.image} " + f"\n Region: {self.region} " + f"\n Project id: {self.project_id} " + f"\n Scheduler CPU: {self.cpu} " + f"\n Scheduler Memory: {self.memory} " + f"\n Scheduler Timeout: {self.cluster.scheduler_timeout} " + f"\n Worker CPU: {self.cluster.worker_cpu} " + f"\n Worker Memory: {self.cluster.worker_mem} " + f"\n Worker Threads: {self.cluster.worker_threads} " + ) + self.cluster._log(f"Creating scheduler instance {self.name}") + + # It must use the external URL with the "wss" protocol and port 443 to establish a + # secure WebSocket connection between the client and the scheduler. + self.internal_ip, self.external_ip = await self.create_vm() + self.address = f"wss://{self.external_ip}:443" + + await self.wait_for_scheduler() + + self.cluster.scheduler_internal_ip = self.internal_ip + self.cluster.scheduler_external_ip = self.external_ip + self.cluster.scheduler_port = self.port + self.status = Status.running + + +class IBMCodeEngineWorker(WorkerMixin, IBMCodeEngine): + def __init__( + self, + *args, + worker_class: str = "distributed.cli.Nanny", + worker_options: dict = {}, + **kwargs + ): + super().__init__(*args, **kwargs) + self.worker_class = worker_class + self.worker_options = worker_options + self.cpu = self.cluster.worker_cpu + self.memory = self.cluster.worker_mem + + # On this case, the worker must connect to the scheduler internal URL with the "ws" protocol and port 80 + internal_scheduler = f"ws://{self.cluster.scheduler_internal_ip}:80" + + self.command = [ + "python", + "-m", + "distributed.cli.dask_spec", + internal_scheduler, + "--spec", + json.dumps( + { + "cls": self.worker_class, + "opts": { + **worker_options, + "name": self.name, + "nthreads": self.cluster.worker_threads, + }, + } + ), + ] + + async def start(self): + self.cluster._log(f"Creating worker instance {self.name}") + await self.create_vm() + self.status = Status.running + + +class IBMCodeEngineCluster(VMCluster): + """Cluster running on IBM Code Engine. + + This cluster manager builds a Dask cluster running on IBM Code Engine. + + When configuring your cluster, you may find it useful to refer to the IBM Cloud documentation for available options. + + https://cloud.ibm.com/docs/codeengine + + Parameters + ---------- + image: str + The Docker image to run on all instances. This image must have a valid Python environment and have ``dask`` + installed in order for the ``dask-scheduler`` and ``dask-worker`` commands to be available. + region: str + The IBM Cloud region to launch your cluster in. + + See: https://cloud.ibm.com/docs/codeengine?topic=codeengine-regions + project_id: str + Your IBM Cloud project ID. This must be set either here or in your Dask config. + scheduler_cpu: str + The amount of CPU to allocate to the scheduler. + + See: https://cloud.ibm.com/docs/codeengine?topic=codeengine-mem-cpu-combo + scheduler_mem: str + The amount of memory to allocate to the scheduler. + + See: https://cloud.ibm.com/docs/codeengine?topic=codeengine-mem-cpu-combo + scheduler_timeout: int + The timeout for the scheduler in seconds. + worker_cpu: str + The amount of CPU to allocate to each worker. + + See: https://cloud.ibm.com/docs/codeengine?topic=codeengine-mem-cpu-combo + worker_mem: str + The amount of memory to allocate to each worker. + + See: https://cloud.ibm.com/docs/codeengine?topic=codeengine-mem-cpu-combo + worker_threads: int + The number of threads to use on each worker. + debug: bool, optional + More information will be printed when constructing clusters to enable debugging. + + Notes + ----- + + **Credentials** + + In order to use the IBM Cloud API, you will need to set up an API key. You can create an API key in the IBM Cloud + console. + + The best practice way of doing this is to pass an API key to be used by workers. You can set this API key as an + environment variable. Here is a small example to help you do that. + + To expose your IBM API KEY, use this command: + export DASK_CLOUDPROVIDER__IBM__API_KEY=xxxxx + + **Certificates** + + This backend will need to use a Let's Encrypt certificate (ISRG Root X1) to connect the client to the scheduler + between websockets. More information can be found here: https://letsencrypt.org/certificates/ + + Examples + -------- + + Create the cluster. + + >>> from dask_cloudprovider.ibm import IBMCodeEngineCluster + >>> cluster = IBMCodeEngineCluster(n_workers=1) + Launching cluster with the following configuration: + Source Image: daskdev/dask:latest + Region: eu-de + Project id: f21626f6-54f7-4065-a038-75c8b9a0d2e0 + Scheduler CPU: 0.25 + Scheduler Memory: 1G + Scheduler Timeout: 600 + Worker CPU: 2 + Worker Memory: 4G + Creating scheduler dask-xxxxxxxx-scheduler + Waiting for scheduler to run at dask-xxxxxxxx-scheduler.xxxxxxxxxxxx.xx-xx.codeengine.appdomain.cloud:443 + Scheduler is running + Creating worker instance dask-xxxxxxxx-worker-xxxxxxxx + + >>> from dask.distributed import Client + >>> client = Client(cluster) + + Do some work. + + >>> import dask.array as da + >>> arr = da.random.random((1000, 1000), chunks=(100, 100)) + >>> arr.mean().compute() + 0.5001550986751964 + + Close the cluster + + >>> cluster.close() + Deleting Instance: dask-xxxxxxxx-worker-xxxxxxxx + Deleting Instance: dask-xxxxxxxx-scheduler + + You can also do this all in one go with context managers to ensure the cluster is created and cleaned up. + + >>> with IBMCodeEngineCluster(n_workers=1) as cluster: + ... with Client(cluster) as client: + ... print(da.random.random((1000, 1000), chunks=(100, 100)).mean().compute()) + Launching cluster with the following configuration: + Source Image: daskdev/dask:latest + Region: eu-de + Project id: f21626f6-54f7-4065-a038-75c8b9a0d2e0 + Scheduler CPU: 0.25 + Scheduler Memory: 1G + Scheduler Timeout: 600 + Worker CPU: 2 + Worker Memory: 4G + Creating scheduler dask-xxxxxxxx-scheduler + Waiting for scheduler to run at dask-xxxxxxxx-scheduler.xxxxxxxxxxxx.xx-xx.codeengine.appdomain.cloud:443 + Scheduler is running + Creating worker instance dask-xxxxxxxx-worker-xxxxxxxx + 0.5000812282861661 + Deleting Instance: dask-xxxxxxxx-worker-xxxxxxxx + Deleting Instance: dask-xxxxxxxx-scheduler + + """ + + def __init__( + self, + image: str = None, + region: str = None, + project_id: str = None, + scheduler_cpu: str = None, + scheduler_mem: str = None, + scheduler_timeout: int = None, + worker_cpu: str = None, + worker_mem: str = None, + worker_threads: int = 1, + debug: bool = False, + **kwargs, + ): + self.config = dask.config.get("cloudprovider.ibm", {}) + self.scheduler_class = IBMCodeEngineScheduler + self.worker_class = IBMCodeEngineWorker + + self.image = image or self.config.get("image") + self.region = region or self.config.get("region") + self.project_id = project_id or self.config.get("project_id") + api_key = self.config.get("api_key") + self.scheduler_cpu = scheduler_cpu or self.config.get("scheduler_cpu") + self.scheduler_mem = scheduler_mem or self.config.get("scheduler_mem") + self.scheduler_timeout = scheduler_timeout or self.config.get("scheduler_timeout") + self.worker_cpu = worker_cpu or self.config.get("worker_cpu") + self.worker_mem = worker_mem or self.config.get("worker_mem") + self.worker_threads = worker_threads or self.config.get("worker_threads") + + self.debug = debug + + self.options = { + "cluster": self, + "config": self.config, + "image": self.image, + "region": self.region, + "project_id": self.project_id, + "scheduler_cpu": self.scheduler_cpu, + "scheduler_mem": self.scheduler_mem, + "scheduler_timeout": self.scheduler_timeout, + "worker_cpu": self.worker_cpu, + "worker_mem": self.worker_mem, + "worker_threads": self.worker_threads, + "api_key": api_key, + } + self.scheduler_options = {**self.options} + self.worker_options = {**self.options} + + # https://letsencrypt.org/certificates/ --> ISRG Root X1 + sec = Security(require_encryption=False, tls_ca_file="-----BEGIN CERTIFICATE-----\nMIIFazCCA1OgAwIBAgIRAIIQz7DSQONZRGPgu2OCiwAwDQYJKoZIhvcNAQELBQAw\nTzELMAkGA1UEBhMCVVMxKTAnBgNVBAoTIEludGVybmV0IFNlY3VyaXR5IFJlc2Vh\ncmNoIEdyb3VwMRUwEwYDVQQDEwxJU1JHIFJvb3QgWDEwHhcNMTUwNjA0MTEwNDM4\nWhcNMzUwNjA0MTEwNDM4WjBPMQswCQYDVQQGEwJVUzEpMCcGA1UEChMgSW50ZXJu\nZXQgU2VjdXJpdHkgUmVzZWFyY2ggR3JvdXAxFTATBgNVBAMTDElTUkcgUm9vdCBY\nMTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAK3oJHP0FDfzm54rVygc\nh77ct984kIxuPOZXoHj3dcKi/vVqbvYATyjb3miGbESTtrFj/RQSa78f0uoxmyF+\n0TM8ukj13Xnfs7j/EvEhmkvBioZxaUpmZmyPfjxwv60pIgbz5MDmgK7iS4+3mX6U\nA5/TR5d8mUgjU+g4rk8Kb4Mu0UlXjIB0ttov0DiNewNwIRt18jA8+o+u3dpjq+sW\nT8KOEUt+zwvo/7V3LvSye0rgTBIlDHCNAymg4VMk7BPZ7hm/ELNKjD+Jo2FR3qyH\nB5T0Y3HsLuJvW5iB4YlcNHlsdu87kGJ55tukmi8mxdAQ4Q7e2RCOFvu396j3x+UC\nB5iPNgiV5+I3lg02dZ77DnKxHZu8A/lJBdiB3QW0KtZB6awBdpUKD9jf1b0SHzUv\nKBds0pjBqAlkd25HN7rOrFleaJ1/ctaJxQZBKT5ZPt0m9STJEadao0xAH0ahmbWn\nOlFuhjuefXKnEgV4We0+UXgVCwOPjdAvBbI+e0ocS3MFEvzG6uBQE3xDk3SzynTn\njh8BCNAw1FtxNrQHusEwMFxIt4I7mKZ9YIqioymCzLq9gwQbooMDQaHWBfEbwrbw\nqHyGO0aoSCqI3Haadr8faqU9GY/rOPNk3sgrDQoo//fb4hVC1CLQJ13hef4Y53CI\nrU7m2Ys6xt0nUW7/vGT1M0NPAgMBAAGjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNV\nHRMBAf8EBTADAQH/MB0GA1UdDgQWBBR5tFnme7bl5AFzgAiIyBpY9umbbjANBgkq\nhkiG9w0BAQsFAAOCAgEAVR9YqbyyqFDQDLHYGmkgJykIrGF1XIpu+ILlaS/V9lZL\nubhzEFnTIZd+50xx+7LSYK05qAvqFyFWhfFQDlnrzuBZ6brJFe+GnY+EgPbk6ZGQ\n3BebYhtF8GaV0nxvwuo77x/Py9auJ/GpsMiu/X1+mvoiBOv/2X/qkSsisRcOj/KK\nNFtY2PwByVS5uCbMiogziUwthDyC3+6WVwW6LLv3xLfHTjuCvjHIInNzktHCgKQ5\nORAzI4JMPJ+GslWYHb4phowim57iaztXOoJwTdwJx4nLCgdNbOhdjsnvzqvHu7Ur\nTkXWStAmzOVyyghqpZXjFaH3pO3JLF+l+/+sKAIuvtd7u+Nxe5AW0wdeRlN8NwdC\njNPElpzVmbUq4JUagEiuTDkHzsxHpFKVK7q4+63SM1N95R1NbdWhscdCb+ZAJzVc\noyi3B43njTOQ5yOf+1CceWxG1bQVs5ZufpsMljq4Ui0/1lvh+wjChP4kqKOJ2qxq\n4RgqsahDYVvTH9w7jXbyLeiNdd8XM2w9U/t7y0Ff/9yi0GE44Za4rF2LN9d11TPA\nmRGunUHBcnWEvgJBQl9nJEiU0Zsnvgc/ubhPgXRR4Xq37Z0j4r7g1SgEEzwxA57d\nemyPxgcYxn/eR44/KJ4EBs+lVDR3veyJm+kXQ99b21/+jh5Xos1AnX5iItreGCc=\n-----END CERTIFICATE-----") + super().__init__(security=sec, debug=debug, **kwargs) diff --git a/dask_cloudprovider/ibm/tests/test_code_engine.py b/dask_cloudprovider/ibm/tests/test_code_engine.py new file mode 100644 index 00000000..ec84a421 --- /dev/null +++ b/dask_cloudprovider/ibm/tests/test_code_engine.py @@ -0,0 +1,108 @@ +import pytest + +import dask + +codeengine = pytest.importorskip("ibm_code_engine_sdk.code_engine_v2") + +from dask_cloudprovider.ibm.code_engine import IBMCodeEngineCluster +from dask.distributed import Client +from distributed.core import Status + + +async def skip_without_credentials(): + if dask.config.get("cloudprovider.ibm.api_key") is None: + pytest.skip( + """ + You must configure a IBM API key to run this test. + + Either set this in your config + + # cloudprovider.yaml + cloudprovider: + ibm: + api_key: "your_api_key" + + Or by setting it as an environment variable + + export DASK_CLOUDPROVIDER__IBM__API_KEY="your_api_key" + + """ + ) + + if dask.config.get("cloudprovider.ibm.project_id") is None: + pytest.skip( + """ + You must configure a IBM project id to run this test. + + Either set this in your config + + # cloudprovider.yaml + cloudprovider: + ibm: + project_id: "your_project_id" + + Or by setting it as an environment variable + + export DASK_CLOUDPROVIDER__IBM__PROJECT_ID="your_project_id" + + """ + ) + + if dask.config.get("cloudprovider.ibm.region") is None: + pytest.skip( + """ + You must configure a IBM project id to run this test. + + Either set this in your config + + # cloudprovider.yaml + cloudprovider: + ibm: + region: "your_region" + + Or by setting it as an environment variable + + export DASK_CLOUDPROVIDER__IBM__REGION="your_region" + + """ + ) + + +@pytest.mark.asyncio +async def test_init(): + await skip_without_credentials() + cluster = IBMCodeEngineCluster(asynchronous=True) + assert cluster.status == Status.created + + +@pytest.mark.asyncio +@pytest.mark.timeout(1200) +@pytest.mark.external +async def test_create_cluster(): + async with IBMCodeEngineCluster(asynchronous=True) as cluster: + cluster.scale(2) + await cluster + assert len(cluster.workers) == 2 + + async with Client(cluster, asynchronous=True) as client: + + def inc(x): + return x + 1 + + assert await client.submit(inc, 10).result() == 11 + + +@pytest.mark.asyncio +@pytest.mark.timeout(1200) +@pytest.mark.external +async def test_create_cluster_sync(): + with IBMCodeEngineCluster() as cluster: + with Client(cluster) as client: + cluster.scale(1) + client.wait_for_workers(1) + assert len(cluster.workers) == 1 + + def inc(x): + return x + 1 + + assert client.submit(inc, 10).result() == 11 diff --git a/dask_cloudprovider/tests/test_imports.py b/dask_cloudprovider/tests/test_imports.py index 3dc42e66..08d36d6e 100644 --- a/dask_cloudprovider/tests/test_imports.py +++ b/dask_cloudprovider/tests/test_imports.py @@ -9,6 +9,7 @@ def test_imports(): from dask_cloudprovider.gcp import GCPCluster # noqa from dask_cloudprovider.digitalocean import DropletCluster # noqa from dask_cloudprovider.hetzner import HetznerCluster # noqa + from dask_cloudprovider.ibm import IBMCodeEngineCluster # noqa def test_import_exceptions(): @@ -24,5 +25,7 @@ def test_import_exceptions(): from dask_cloudprovider import GCPCluster # noqa with pytest.raises(ImportError): from dask_cloudprovider import DropletCluster # noqa + with pytest.raises(ImportError): + from dask_cloudprovider import IBMCodeEngineCluster # noqa with pytest.raises(ImportError): from dask_cloudprovider import OpenStackCluster # noqa diff --git a/doc/source/ibm.rst b/doc/source/ibm.rst new file mode 100644 index 00000000..c0ed620a --- /dev/null +++ b/doc/source/ibm.rst @@ -0,0 +1,58 @@ +IBM Cloud +============ + +.. currentmodule:: dask_cloudprovider.ibm + +.. autosummary:: + IBMCodeEngineCluster + +Overview +-------- + +Authentication +^^^^^^^^^^^^^^ + +To authenticate with IBM Cloud you must first generate an +`API key `_. + +Then you must put this in your Dask configuration at ``cloudprovider.ibm.api_key``. This can be done by +adding the API key to your YAML configuration or exporting an environment variable. + +.. code-block:: yaml + + # ~/.config/dask/cloudprovider.yaml + + cloudprovider: + ibm: + api_key: "your_api_key" + +.. code-block:: console + + $ export DASK_CLOUDPROVIDER__IBM__API_KEY="your_api_key" + +Project ID +^^^^^^^^^^ + +To use Dask Cloudprovider with IBM Cloud you must also configure your `Project ID `_. +This can be found at the top of the IBM Cloud dashboard. + +Your Project ID must be added to your Dask config file. + +.. code-block:: yaml + + # ~/.config/dask/cloudprovider.yaml + cloudprovider: + ibm: + project_id: "your_project_id" + +Or via an environment variable. + +.. code-block:: console + + $ export DASK_CLOUDPROVIDER__IBM__PROJECT_ID="your_project_id" + +Code Engine +------- + +.. autoclass:: IBMCodeEngineCluster + :members: \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index 74d2a661..bc7dda10 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -70,6 +70,7 @@ this code. gcp.rst azure.rst hetzner.rst + ibm.rst openstack.rst .. toctree:: diff --git a/doc/source/installation.rst b/doc/source/installation.rst index f4994435..c28a2be0 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -17,6 +17,7 @@ You can also restrict your install to just a specific cloud provider by giving t $ pip install dask-cloudprovider[azureml]  # or $ pip install dask-cloudprovider[digitalocean]  # or $ pip install dask-cloudprovider[gcp]  # or + $ pip install dask-cloudprovider[ibm]  # or $ pip install dask-cloudprovider[openstack] Conda diff --git a/setup.py b/setup.py index 5d3f3b78..0dd311c4 100644 --- a/setup.py +++ b/setup.py @@ -15,6 +15,7 @@ "digitalocean": ["python-digitalocean>=1.15.0"], "gcp": ["google-api-python-client>=1.12.5", "google-auth>=1.23.0"], "hetzner": ["hcloud>=1.10.0"], + "ibm": ["ibm_code_engine_sdk>=3.1.0"], "openstack": ["openstacksdk>=3.3.0"], } extras_require["all"] = set(pkg for pkgs in extras_require.values() for pkg in pkgs)