Merge pull request #9 from NASA-IMPACT/landsat-downloader

Landsat downloader

README.md

@@ -23,6 +23,9 @@ $ get_s2_granule_dir INPUTS2DIR
 ```bash
 $ parse_fmask FMASKOUTPUT
 ```
+```bash
+$ download_landsat BUCKET PATH OUTPUT_DIRECTORY  
+```
 
 
 ### Tests

download_landsat/download_landsat.py

@@ -0,0 +1,82 @@
+import boto3
+import click
+from pathlib import Path
+
+
+class KeyDoesNotExist(Exception):
+    pass
+
+
+def key_exists(client, bucket, path):
+    result = client.list_objects_v2(
+        Bucket=bucket,
+        Prefix=path,
+        RequestPayer="requester"
+    )
+    if result.get("KeyCount") > 0:
+        return True
+    else:
+        return False
+
+
+def download_files(client, bucket, path, output_directory):
+    result = client.list_objects_v2(
+        Bucket=bucket,
+        Prefix=path,
+        RequestPayer="requester"
+    )
+    contents = result.get("Contents")
+    for content in contents:
+        key = content.get("Key")
+        filename = Path(key).name
+        output_file = Path(output_directory).joinpath(filename)
+        client.download_file(bucket, key, str(output_file), ExtraArgs={
+            "RequestPayer": "requester"
+        })
+
+
+def get_updated_key(client, bucket, path):
+    path_root = Path(path).parent
+    result = client.list_objects_v2(
+        Bucket=bucket,
+        Prefix=str(path_root) + "/",
+        RequestPayer="requester",
+        Delimiter="/"
+    )
+    if result.get("KeyCount") == 0:
+        raise KeyDoesNotExist
+    else:
+        updated_key = [
+            prefix["Prefix"] for prefix in result.get("CommonPrefixes")
+            if prefix["Prefix"].split("_")[3] == path.split("_")[3]
+        ]
+        if len(updated_key) == 0:
+            raise KeyDoesNotExist
+        else:
+            return updated_key[0]
+
+
+def get_landsat(bucket, path, output_directory):
+    client = boto3.client("s3")
+    if(key_exists(client, bucket, path)):
+        download_files(client, bucket, path, output_directory)
+    else:
+        updated_path = get_updated_key(client, bucket, path)
+        download_files(client, bucket, updated_path, output_directory)
+
+
+@click.command()
+@click.argument(
+    "bucket",
+    type=click.STRING
+)
+@click.argument(
+    "path",
+    type=click.STRING,
+)
+@click.argument(
+    "output_directory",
+    type=click.Path(),
+)
+def main(bucket, path, output_directory):
+    get_landsat(bucket, path, output_directory)

setup.py

@@ -7,10 +7,14 @@
     install_requires=[
         "click~=7.1.0",
         "lxml",
+        "boto3~=1.17.91",
         "espa-python-library @ git+https://github.com/USGS-EROS/[email protected]#egg=espa-python-library"
     ],
     include_package_data=True,
-    extras_require={"dev": ["flake8", "black"], "test": ["flake8", "pytest"]},
+    extras_require={
+        "dev": ["flake8", "black"],
+        "test": ["flake8", "pytest", "Jinja2==2.10.1", "moto[s3]~=2.0.8"]
+    },
     entry_points={"console_scripts": [
         "parse_fmask=parse_fmask.parse_fmask:main",
         "check_solar_zenith_sentinel=check_solar_zenith_sentinel.check_solar_zenith_sentinel:main",
@@ -20,5 +24,6 @@
         "create_sr_hdf_xml=create_sr_hdf_xml.create_sr_hdf_xml:main",
         "create_landsat_sr_hdf_xml=create_landsat_sr_hdf_xml.create_landsat_sr_hdf_xml:main",
         "check_sentinel_clouds=check_sentinel_clouds.check_sentinel_clouds:main",
+        "download_landsat=download_landsat.download_landsat:main",
     ]},
 )

tests/test_download_landsat.py

@@ -0,0 +1,74 @@
+import os
+import io
+import pytest
+import boto3
+from click.testing import CliRunner
+from moto import mock_s3
+from download_landsat.download_landsat import get_landsat, main
+from download_landsat.download_landsat import KeyDoesNotExist
+
+BUCKET = "usgs-landsat1"
+BASE_KEY = "collection02/level-1/standard/oli-tirs/2021/155/018"
+RT_KEY = f"{BASE_KEY}/LC08_L1TP_155018_20210603_20210603_02_RT"
+T1_KEY = f"{BASE_KEY}/LC08_L1TP_155018_20210603_20210608_02_T1"
+
+
+@pytest.fixture(scope="module")
+def clear_test_file():
+    yield None
+    os.system("rm ./test.json")
+
+
+@pytest.fixture(scope="function")
+def aws_credentials():
+    """Mocked AWS Credentials for moto."""
+    os.environ["AWS_ACCESS_KEY_ID"] = "testing"
+    os.environ["AWS_SECRET_ACCESS_KEY"] = "testing"
+    os.environ["AWS_SECURITY_TOKEN"] = "testing"
+    os.environ["AWS_SESSION_TOKEN"] = "testing"
+    os.environ["AWS_DEFAULT_REGION"] = "us-east-1"
+
+
+@pytest.fixture(scope="function")
+def s3(aws_credentials):
+    with mock_s3():
+        client = boto3.client("s3", region_name="us-east-1")
+        client.create_bucket(Bucket=BUCKET)
+        client.put_bucket_request_payment(
+            Bucket=BUCKET,
+            RequestPaymentConfiguration={
+                "Payer": "Requester"
+            },
+        )
+        yield client
+
+
+def test_download(s3, tmp_path):
+    fo = io.BytesIO(b"file object in RAM")
+    filename = "rt.json"
+    s3.upload_fileobj(fo, BUCKET, f"{RT_KEY}/{filename}")
+    get_landsat(BUCKET, RT_KEY, str(tmp_path))
+    assert os.path.isfile(tmp_path.joinpath(filename))
+
+
+def test_download_no_keys(s3, tmp_path):
+    with pytest.raises(KeyDoesNotExist):
+        get_landsat(BUCKET, RT_KEY, str(tmp_path))
+
+
+def test_download_updated_tier(s3, tmp_path):
+    fo = io.BytesIO(b"file object in RAM")
+    filename = "t1.json"
+    s3.upload_fileobj(fo, BUCKET, f"{T1_KEY}/{filename}")
+    get_landsat(BUCKET, RT_KEY, str(tmp_path))
+    assert os.path.isfile(tmp_path.joinpath(filename))
+
+
+def test_download_landsat_cli(s3, tmp_path):
+    fo = io.BytesIO(b"file object in RAM")
+    filename = "rt.json"
+    s3.upload_fileobj(fo, BUCKET, f"{RT_KEY}/{filename}")
+
+    runner = CliRunner(echo_stdin=True)
+    result = runner.invoke(main, [BUCKET, RT_KEY, str(tmp_path)])
+    assert result.exit_code == 0