Merge pull request #53 from Indicio-tech/feat/put-file-by-hash-valida…

…tion

feat: add endpoints for putting and getting tails files by hash

docker/manage

@@ -46,7 +46,7 @@ exportEnvironment() {
   export GENESIS_URL=${GENESIS_URL:-http://$DOCKERHOST:9000/genesis}
   export STORAGE_PATH=${STORAGE_PATH:-/tmp/tails-files}
   export LOG_LEVEL=${LOG_LEVEL:-INFO}
-  export TAILS_SERVER_URL=${TAILS_SERVER_URL:-http://$DOCKERHOST:6543}
+  export TAILS_SERVER_URL=${TAILS_SERVER_URL:-http://host.docker.internal:6543}
 }
 
 function logs() {

tails_server/web.py

@@ -56,6 +56,30 @@ async def get_file(request):
 
     await response.write_eof()
 
+@routes.get("/hash/{tails_hash}")
+async def get_file_by_hash(request):
+    tails_hash = request.match_info["tails_hash"]
+    storage_path = request.app["settings"]["storage_path"]
+
+    response = web.StreamResponse()
+    response.enable_compression()
+    response.enable_chunked_encoding()
+
+    # Stream the response since the file could be big.
+    try:
+        with open(os.path.join(storage_path, tails_hash), "rb") as tails_file:
+            await response.prepare(request)
+            while True:
+                chunk = tails_file.read(CHUNK_SIZE)
+                if not chunk:
+                    break
+                await response.write(chunk)
+
+    except FileNotFoundError:
+        raise web.HTTPNotFound()
+
+    await response.write_eof()
+
 
 @routes.put("/{revocation_reg_id}")
 async def put_file(request):
@@ -146,6 +170,75 @@ async def put_file(request):
     return web.Response(text=tails_hash)
 
 
+
+@routes.put("/hash/{tails_hash}")
+async def put_file_by_hash(request):
+    storage_path = request.app["settings"]["storage_path"]
+
+    # Check content-type for multipart
+    content_type_header = request.headers.get("Content-Type")
+    if "multipart" not in content_type_header:
+        LOGGER.debug(f"Bad Content-Type header: {content_type_header}")
+        raise web.HTTPBadRequest(text="Expected mutlipart content type")
+
+    reader = await request.multipart()
+
+    tails_hash = request.match_info["tails_hash"]
+
+    # Get first field
+    field = await reader.next()
+
+    # Process the file in chunks so we don't explode on large files.
+    # Construct hash and write file in chunks.
+    sha256 = hashlib.sha256()
+    try:
+        # This should be atomic across networked filesystems:
+        # https://linux.die.net/man/3/open
+        # http://nfs.sourceforge.net/ (D10)
+        # 'x' mode == O_EXCL | O_CREAT
+        with NamedTemporaryFile("w+b") as tmp_file:
+            while True:
+                chunk = await field.read_chunk(CHUNK_SIZE)
+                if not chunk:
+                    break
+                sha256.update(chunk)
+                tmp_file.write(chunk)
+
+            # Check file integrity against tails_hash
+            digest = sha256.digest()
+            b58_digest = base58.b58encode(digest).decode("utf-8")
+            if tails_hash != b58_digest:
+                raise web.HTTPBadRequest(text="tailsHash does not match hash of file.")
+
+            # Basic validation of tails file:
+            # Tails file must start with "00 02"
+            tmp_file.seek(0)
+            if tmp_file.read(2) != b'\x00\x02':
+                raise web.HTTPBadRequest(text='Tails file must start with "00 02".')
+
+            # Since each tail is 128 bytes, tails file size must be a multiple of 128
+            # plus the 2-byte version tag
+            tmp_file.seek(0, 2)
+            if (tmp_file.tell() - 2) % 128 != 0:
+                raise web.HTTPBadRequest(text="Tails file is not the correct size.")
+
+            # File integrity is good so write file to permanent location.
+            tmp_file.seek(0)
+            with open(
+                os.path.join(storage_path, tails_hash), "xb"
+            ) as tails_file:
+                while True:
+                    chunk = tmp_file.read(CHUNK_SIZE)
+                    if not chunk:
+                        break
+                    tails_file.write(chunk)
+
+    except FileExistsError:
+        raise web.HTTPConflict(text="This tails file already exists.")
+
+    return web.Response(text=tails_hash)
+
+
 def start(settings):
     app = web.Application()
     app["settings"] = settings

test/integration.py

@@ -8,6 +8,7 @@
 import base58
 from random import randrange
 import aiofiles
+import io
 
 from tempfile import NamedTemporaryFile
 
@@ -191,6 +192,9 @@ async def run_tests(genesis_url, tails_server_url):
     pool.close()
 
     await test_upload_bad_tails_file(genesis_file.name, tails_server_url, revo_reg_def)
+    await test_put_file_by_hash(tails_server_url)
+    await test_put_file_by_hash_x_version_tag(tails_server_url)
+    await test_put_file_by_hash_x_file_size(tails_server_url)
 
     pool = await connect_to_ledger(genesis_file.name)
     log_event("Publishing revocation registry to ledger...")
@@ -417,6 +421,78 @@ async def download():
     log_event("Passed")
 
 
+async def test_put_file_by_hash(tails_server_url):
+    file = open("test_tails.bin", "wb+")
+    file = io.BytesIO(b'\x00\x02')
+
+    sha256 = hashlib.sha256()
+    sha256.update(file.read())
+    digest = sha256.digest()
+    tails_hash = base58.b58encode(digest).decode("utf-8")
+
+    with aiohttp.MultipartWriter('mixed') as mpwriter:
+        file.seek(0)
+        mpwriter.append(file.read())
+        session = aiohttp.ClientSession()
+        async with session.put(
+            f"{tails_server_url}/hash/{tails_hash}",
+            data=mpwriter,
+        ) as resp:
+            assert resp.status == 200
+
+    file.close()
+    os.remove("test_tails.bin")
+
+
+async def test_put_file_by_hash_x_version_tag(tails_server_url):
+    file = open("test_tails_x_version_tag.bin", "wb+")
+    file = io.BytesIO(b'\x00\x03')
+
+    sha256 = hashlib.sha256()
+    sha256.update(file.read())
+    digest = sha256.digest()
+    tails_hash = base58.b58encode(digest).decode("utf-8")
+
+    with aiohttp.MultipartWriter('mixed') as mpwriter:
+        file.seek(0)
+        mpwriter.append(file.read())
+        session = aiohttp.ClientSession()
+        async with session.put(
+            f"{tails_server_url}/hash/{tails_hash}",
+            data=mpwriter,
+        ) as resp:
+            assert resp.status == 400
+            assert await resp.text() == 'Tails file must start with "00 02".'
+
+    file.close()
+    os.remove("test_tails_x_version_tag.bin")
+
+
+async def test_put_file_by_hash_x_file_size(tails_server_url):
+    file = open("test_tails_x_file_size.bin", "wb+")
+    file = io.BytesIO(b'\x00\x02\x01')
+
+    sha256 = hashlib.sha256()
+    sha256.update(file.read())
+    digest = sha256.digest()
+    tails_hash = base58.b58encode(digest).decode("utf-8")
+
+    with aiohttp.MultipartWriter('mixed') as mpwriter:
+        file.seek(0)
+        mpwriter.append(file.read())
+        session = aiohttp.ClientSession()
+        async with session.put(
+            f"{tails_server_url}/hash/{tails_hash}",
+            data=mpwriter,
+        ) as resp:
+            assert resp.status == 400
+            assert await resp.text() == "Tails file is not the correct size."
+
+    file.close()
+    os.remove("test_tails_x_file_size.bin")
+
+
+
 PARSER = argparse.ArgumentParser(description="Runs integration tests.")
 PARSER.add_argument(
     "--genesis-url",