diff --git a/moto/cloudformation/exceptions.py b/moto/cloudformation/exceptions.py
index a0a4eb63..c8bb5b39 100644
--- a/moto/cloudformation/exceptions.py
+++ b/moto/cloudformation/exceptions.py
@@ -1,6 +1,26 @@
 from __future__ import unicode_literals
+from boto.exception import BotoServerError
+from jinja2 import Template
+
 
 
 class UnformattedGetAttTemplateException(Exception):
     description = 'Template error: resource {0} does not support attribute type {1} in Fn::GetAtt'
     status_code = 400
+
+
+class ValidationError(BotoServerError):
+    def __init__(self, name_or_id):
+        template = Template(STACK_DOES_NOT_EXIST_RESPONSE)
+        super(ValidationError, self).__init__(status=400, reason='Bad Request',
+                                              body=template.render(name_or_id=name_or_id))
+
+STACK_DOES_NOT_EXIST_RESPONSE = """<ErrorResponse xmlns="http://cloudformation.amazonaws.com/doc/2010-05-15/">
+  <Error>
+    <Type>Sender</Type>
+    <Code>ValidationError</Code>
+    <Message>Stack:{{ name_or_id }} does not exist</Message>
+  </Error>
+  <RequestId>cf4c737e-5ae2-11e4-a7c9-ad44eEXAMPLE</RequestId>
+</ErrorResponse>
+"""
diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py
index f5418217..3df740e6 100644
--- a/moto/cloudformation/models.py
+++ b/moto/cloudformation/models.py
@@ -5,6 +5,7 @@ from moto.core import BaseBackend
 
 from .parsing import ResourceMap, OutputMap
 from .utils import generate_stack_id
+from .exceptions import ValidationError
 
 
 class FakeStack(object):
@@ -50,10 +51,12 @@ class CloudFormationBackend(BaseBackend):
             for stack in stacks:
                 if stack.name == name_or_stack_id or stack.stack_id == name_or_stack_id:
                     return [stack]
-            deleted_stacks = self.deleted_stacks.values()
-            for stack in deleted_stacks:
-                if stack.stack_id == name_or_stack_id:
-                    return [stack]
+            if self.deleted_stacks:
+                deleted_stacks = self.deleted_stacks.values()
+                for stack in deleted_stacks:
+                    if stack.stack_id == name_or_stack_id:
+                        return [stack]
+            raise ValidationError(name_or_stack_id)
         else:
             return stacks
 
diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py
index 4267a6e4..1e4b7376 100644
--- a/moto/cloudformation/responses.py
+++ b/moto/cloudformation/responses.py
@@ -20,7 +20,7 @@ class CloudFormationResponse(BaseResponse):
         stack_body = {
             'CreateStackResponse': {
                 'CreateStackResult': {
-                    'StackId': stack.name,
+                    'StackId': stack.stack_id,
                 }
             }
         }
diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud.py b/tests/test_cloudformation/test_cloudformation_stack_crud.py
index f818e42e..477272a3 100644
--- a/tests/test_cloudformation/test_cloudformation_stack_crud.py
+++ b/tests/test_cloudformation/test_cloudformation_stack_crud.py
@@ -1,10 +1,13 @@
 from __future__ import unicode_literals
 import json
-
 import boto
 import sure  # noqa
+# Ensure 'assert_raises' context manager support for Python 2.6
+import tests.backport_assert_raises
+from nose.tools import assert_raises
 
 from moto import mock_cloudformation
+from moto.cloudformation.exceptions import ValidationError
 
 dummy_template = {
     "AWSTemplateFormatVersion": "2010-09-09",
@@ -131,6 +134,17 @@ def test_delete_stack_by_id():
     conn.list_stacks().should.have.length_of(1)
     conn.delete_stack(stack_id)
     conn.list_stacks().should.have.length_of(0)
+    with assert_raises(ValidationError):
+        conn.describe_stacks("test_stack")
+
+    conn.describe_stacks(stack_id).should.have.length_of(1)
+
+
+@mock_cloudformation
+def test_bad_describe_stack():
+    conn = boto.connect_cloudformation()
+    with assert_raises(ValidationError):
+        conn.describe_stacks("bad_stack")
 
 
 # @mock_cloudformation