diff --git a/moto/s3/exceptions.py b/moto/s3/exceptions.py
index 3cf2ecb5..ee65a88e 100644
--- a/moto/s3/exceptions.py
+++ b/moto/s3/exceptions.py
@@ -15,6 +15,10 @@ ERROR_WITH_ARGUMENT = """{% extends 'single_error' %}
 <ArgumentValue>{{ value }}</ArgumentValue>{% endblock %}
 """
 
+ERROR_WITH_UPLOADID = """{% extends 'single_error' %}
+{% block extra %}<UploadId>{{ upload_id }}</UploadId>{% endblock %}
+"""
+
 ERROR_WITH_CONDITION_NAME = """{% extends 'single_error' %}
 {% block extra %}<Condition>{{ condition }}</Condition>{% endblock %}
 """
@@ -418,9 +422,15 @@ class NoSystemTags(S3ClientError):
 class NoSuchUpload(S3ClientError):
     code = 404
 
-    def __init__(self):
+    def __init__(self, upload_id, *args, **kwargs):
+        kwargs.setdefault("template", "error_uploadid")
+        kwargs["upload_id"] = upload_id
+        self.templates["error_uploadid"] = ERROR_WITH_UPLOADID
         super(NoSuchUpload, self).__init__(
-            "NoSuchUpload", "The specified multipart upload does not exist."
+            "NoSuchUpload",
+            "The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.",
+            *args,
+            **kwargs
         )
 
 
diff --git a/moto/s3/models.py b/moto/s3/models.py
index d6a98282..d50d1519 100644
--- a/moto/s3/models.py
+++ b/moto/s3/models.py
@@ -1604,11 +1604,13 @@ class S3Backend(BaseBackend):
         bucket = self.get_bucket(bucket_name)
         multipart_data = bucket.multiparts.get(multipart_id, None)
         if not multipart_data:
-            raise NoSuchUpload()
+            raise NoSuchUpload(upload_id=multipart_id)
         del bucket.multiparts[multipart_id]
 
     def list_multipart(self, bucket_name, multipart_id):
         bucket = self.get_bucket(bucket_name)
+        if multipart_id not in bucket.multiparts:
+            raise NoSuchUpload(upload_id=multipart_id)
         return list(bucket.multiparts[multipart_id].list_parts())
 
     def get_all_multiparts(self, bucket_name):
diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py
index 9df7476a..7a9f2d99 100644
--- a/tests/test_s3/test_s3.py
+++ b/tests/test_s3/test_s3.py
@@ -2310,7 +2310,12 @@ def test_s3_abort_multipart_data_with_invalid_upload_and_key():
         client.abort_multipart_upload(
             Bucket="blah", Key="foobar", UploadId="dummy_upload_id"
         )
-    err.value.response["Error"]["Code"].should.equal("NoSuchUpload")
+    err = err.value.response["Error"]
+    err["Code"].should.equal("NoSuchUpload")
+    err["Message"].should.equal(
+        "The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed."
+    )
+    err["UploadId"].should.equal("dummy_upload_id")
 
 
 @mock_s3
diff --git a/tests/test_s3/test_s3_multipart.py b/tests/test_s3/test_s3_multipart.py
new file mode 100644
index 00000000..03c71d25
--- /dev/null
+++ b/tests/test_s3/test_s3_multipart.py
@@ -0,0 +1,24 @@
+from botocore.exceptions import ClientError
+from moto import mock_s3
+import boto3
+import pytest
+import sure  # noqa
+
+
+@mock_s3
+def test_multipart_should_throw_nosuchupload_if_there_are_no_parts():
+    bucket = boto3.resource("s3").Bucket("randombucketname")
+    bucket.create()
+    s3_object = bucket.Object("my/test2")
+
+    multipart_upload = s3_object.initiate_multipart_upload()
+    multipart_upload.abort()
+
+    with pytest.raises(ClientError) as ex:
+        list(multipart_upload.parts.all())
+    err = ex.value.response["Error"]
+    err["Code"].should.equal("NoSuchUpload")
+    err["Message"].should.equal(
+        "The specified upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed."
+    )
+    err["UploadId"].should.equal(multipart_upload.id)