[MRG+1] TST remove temp files and folders
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 6333efceb15..f9980f7b2e6 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -1,6 +1,7 @@
 import os
 import six
 import contextlib
+import shutil
 try:
     from unittest import mock
 except ImportError:
@@ -84,11 +85,13 @@ class FileTestCase(unittest.TestCase):
 
     def setUp(self):
         self.tmpname = self.mktemp()
-        fd = open(self.tmpname + '^', 'w')
-        fd.write('0123456789')
-        fd.close()
+        with open(self.tmpname + '^', 'w') as f:
+            f.write('0123456789')
         self.download_request = FileDownloadHandler(Settings()).download_request
 
+    def tearDown(self):
+        os.unlink(self.tmpname + '^')
+
     def test_download(self):
         def _test(response):
             self.assertEquals(response.url, request.url)
@@ -134,10 +137,10 @@ class HttpTestCase(unittest.TestCase):
     certfile = 'keys/cert.pem'
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"hang-after-headers", ForeverTakingResource(write=True))
@@ -165,6 +168,7 @@ def tearDown(self):
         yield self.port.stopListening()
         if hasattr(self.download_handler, 'close'):
             yield self.download_handler.close()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "%s://%s:%d/%s" % (self.scheme, self.host, self.portno, path)
@@ -709,6 +713,9 @@ def setUp(self):
         self.download_handler = FTPDownloadHandler(Settings())
         self.addCleanup(self.port.stopListening)
 
+    def tearDown(self):
+        shutil.rmtree(self.directory)
+
     def _add_test_callbacks(self, deferred, callback=None, errback=None):
         def _clean(data):
             self.download_handler.client.transport.loseConnection()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index e93d2bafbec..2d137edf48a 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -57,8 +57,11 @@ def _assert_stores(self, storage, path):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+        finally:
+            os.unlink(path)
 
 
 class FTPFeedStorageTest(unittest.TestCase):
@@ -79,12 +82,15 @@ def _assert_stores(self, storage, path):
         file.write(b"content")
         yield storage.store(file)
         self.assertTrue(os.path.exists(path))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"content")
-        # again, to check s3 objects are overwritten
-        yield storage.store(BytesIO(b"new content"))
-        with open(path, 'rb') as fp:
-            self.assertEqual(fp.read(), b"new content")
+        try:
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"content")
+            # again, to check s3 objects are overwritten
+            yield storage.store(BytesIO(b"new content"))
+            with open(path, 'rb') as fp:
+                self.assertEqual(fp.read(), b"new content")
+        finally:
+            os.unlink(path)
 
 
 class BlockingFeedStorageTest(unittest.TestCase):
diff --git a/tests/test_spiderstate.py b/tests/test_spiderstate.py
index d1d6debec20..383fadfeb61 100644
--- a/tests/test_spiderstate.py
+++ b/tests/test_spiderstate.py
@@ -1,5 +1,6 @@
 import os
 from datetime import datetime
+import shutil
 from twisted.trial import unittest
 
 from scrapy.extensions.spiderstate import SpiderState
@@ -13,20 +14,23 @@ class SpiderStateTest(unittest.TestCase):
     def test_store_load(self):
         jobdir = self.mktemp()
         os.mkdir(jobdir)
-        spider = Spider(name='default')
-        dt = datetime.now()
-
-        ss = SpiderState(jobdir)
-        ss.spider_opened(spider)
-        spider.state['one'] = 1
-        spider.state['dt'] = dt
-        ss.spider_closed(spider)
-
-        spider2 = Spider(name='default')
-        ss2 = SpiderState(jobdir)
-        ss2.spider_opened(spider2)
-        self.assertEqual(spider.state, {'one': 1, 'dt': dt})
-        ss2.spider_closed(spider2)
+        try:
+            spider = Spider(name='default')
+            dt = datetime.now()
+
+            ss = SpiderState(jobdir)
+            ss.spider_opened(spider)
+            spider.state['one'] = 1
+            spider.state['dt'] = dt
+            ss.spider_closed(spider)
+
+            spider2 = Spider(name='default')
+            ss2 = SpiderState(jobdir)
+            ss2.spider_opened(spider2)
+            self.assertEqual(spider.state, {'one': 1, 'dt': dt})
+            ss2.spider_closed(spider2)
+        finally:
+            shutil.rmtree(jobdir)
 
     def test_state_attribute(self):
         # state attribute must be present if jobdir is not set, to provide a
diff --git a/tests/test_webclient.py b/tests/test_webclient.py
index 9b5beda4cac..3ad1aa70e09 100644
--- a/tests/test_webclient.py
+++ b/tests/test_webclient.py
@@ -4,7 +4,7 @@
 """
 import os
 import six
-from six.moves.urllib.parse import urlparse
+import shutil
 
 from twisted.trial import unittest
 from twisted.web import server, static, util, resource
@@ -12,6 +12,7 @@
 from twisted.test.proto_helpers import StringTransport
 from twisted.python.filepath import FilePath
 from twisted.protocols.policies import WrappingFactory
+from twisted.internet.defer import inlineCallbacks
 
 from scrapy.core.downloader import webclient as client
 from scrapy.http import Request, Headers
@@ -229,10 +230,10 @@ def _listen(self, site):
         return reactor.listenTCP(0, site, interface="127.0.0.1")
 
     def setUp(self):
-        name = self.mktemp()
-        os.mkdir(name)
-        FilePath(name).child("file").setContent(b"0123456789")
-        r = static.File(name)
+        self.tmpname = self.mktemp()
+        os.mkdir(self.tmpname)
+        FilePath(self.tmpname).child("file").setContent(b"0123456789")
+        r = static.File(self.tmpname)
         r.putChild(b"redirect", util.Redirect(b"/file"))
         r.putChild(b"wait", ForeverTakingResource())
         r.putChild(b"error", ErrorResource())
@@ -246,8 +247,10 @@ def setUp(self):
         self.port = self._listen(self.wrapper)
         self.portno = self.port.getHost().port
 
+    @inlineCallbacks
     def tearDown(self):
-        return self.port.stopListening()
+        yield self.port.stopListening()
+        shutil.rmtree(self.tmpname)
 
     def getURL(self, path):
         return "http://127.0.0.1:%d/%s" % (self.portno, path)
@@ -266,7 +269,6 @@ def testHostHeader(self):
             getPage(self.getURL("host"), headers={"Host": "www.example.com"}).addCallback(
                 self.assertEquals, to_bytes("www.example.com"))])
 
-
     def test_getPage(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -276,7 +278,6 @@ def test_getPage(self):
         d.addCallback(self.assertEquals, b"0123456789")
         return d
 
-
     def test_getPageHead(self):
         """
         L{client.getPage} returns a L{Deferred} which is called back with
@@ -289,7 +290,6 @@ def _getPage(method):
             _getPage("head").addCallback(self.assertEqual, b""),
             _getPage("HEAD").addCallback(self.assertEqual, b"")])
 
-
     def test_timeoutNotTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and the page is
@@ -301,7 +301,6 @@ def test_timeoutNotTriggering(self):
             self.assertEquals, to_bytes("127.0.0.1:%d" % self.portno))
         return d
 
-
     def test_timeoutTriggering(self):
         """
         When a non-zero timeout is passed to L{getPage} and that many
@@ -351,7 +350,7 @@ def _cbRedirect(self, pageData):
                 b'    </head>\n    <body bgcolor="#FFFFFF" text="#000000">\n    '
                 b'<a href="/file">click here</a>\n    </body>\n</html>\n')
 
-    def test_Encoding(self):
+    def test_encoding(self):
         """ Test that non-standart body encoding matches
         Content-Encoding header """
         body = b'\xd0\x81\xd1\x8e\xd0\xaf'

This PR makes sure temp files are deleted in tests. It also fixes a **weird** pytest+OS X issue. Previously tests failed for me with exception like this:

```
______________________________________________ ERROR at setup of WebClientTestCase.test_Encoding ______________________________________________

cls = <class 'py._path.local.LocalPath'>, prefix = 'test_Encoding'
rootdir = local('/private/var/folders/_5/cbsg50991szfp1r9nwxpx8580000gn/T/pytest-of-kmike/pytest-0'), keep = 0, lock_timeout = None

    def make_numbered_dir(cls, prefix='session-', rootdir=None, keep=3,
                          lock_timeout = 172800):   # two days
        """ return unique directory with a number greater than the current
                maximum one.  The number is assumed to start directly after prefix.
                if keep is true directories with a number less than (maxnum-keep)
                will be removed.
            """
        if rootdir is None:
            rootdir = cls.get_temproot()

        def parse_num(path):
            """ parse the number out of a path (if it matches the prefix) """
            bn = path.basename
            if bn.startswith(prefix):
                try:
                    return int(bn[len(prefix):])
                except ValueError:
                    pass

        # compute the maximum number currently in use with the
        # prefix
        lastmax = None
        while True:
            maxnum = -1
            for path in rootdir.listdir():
                num = parse_num(path)
                if num is not None:
                    maxnum = max(maxnum, num)

            # make the new directory
            try:
>               udir = rootdir.mkdir(prefix + str(maxnum+1))

/Users/kmike/svn/scrapy/.tox/py36/lib/python3.6/site-packages/py/_path/local.py:825:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _
/Users/kmike/svn/scrapy/.tox/py36/lib/python3.6/site-packages/py/_path/local.py:459: in mkdir
    py.error.checked_call(os.mkdir, fspath(p))
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _

self = <py._error.ErrorMaker object at 0x110844518>, func = <built-in function mkdir>
args = ('/private/var/folders/_5/cbsg50991szfp1r9nwxpx8580000gn/T/pytest-of-kmike/pytest-0/test_Encoding0',), kwargs = {}
__tracebackhide__ = False, cls = <class 'py.error.EEXIST'>, value = FileExistsError(17, 'File exists'), tb = <traceback object at 0x117b3f248>
errno = 17

    def checked_call(self, func, *args, **kwargs):
        """ call a function and raise an errno-exception if applicable. """
        __tracebackhide__ = True
        try:
            return func(*args, **kwargs)
        except self.Error:
            raise
        except (OSError, EnvironmentError):
            cls, value, tb = sys.exc_info()
            if not hasattr(value, 'errno'):
                raise
            __tracebackhide__ = False
            errno = value.errno
            try:
                if not isinstance(value, WindowsError):
                    raise NameError
            except NameError:
                # we are not on Windows, or we got a proper OSError
                cls = self._geterrnoclass(errno)
            else:
                try:
                    cls = self._geterrnoclass(_winerrnomap[errno])
                except KeyError:
                    raise value
>           raise cls("%s%r" % (func.__name__, args))
E           py.error.EEXIST: [File exists]: mkdir('/private/var/folders/_5/cbsg50991szfp1r9nwxpx8580000gn/T/pytest-of-kmike/pytest-0/test_Encoding0',)
```