Fix race condition on cache directory creation

os.makedirs() will raise OSError in case the directory passed as argument
already exists. We do check that this is not the case right before the
call, but there is still a tiny time window in which the directory might
be concurrently created between the isdir() check and the makedirs() call.

In some unlucky cases under heavy I/O the following stack trace is produced:

The minion function caused an exception: Traceback (most recent call last):
...
  File "/usr/lib/python2.7/site-packages/salt/fileclient.py", line 165, in cache_file
    return self.get_url(path, '', True, saltenv, cachedir=cachedir)
...
  File "/usr/lib/python2.7/site-packages/salt/fileclient.py", line 126, in _cache_loc
    os.makedirs(destdir)
  File "/usr/lib64/python2.7/os.py", line 157, in makedirs
    mkdir(name, mode)
OSError: [Errno 17] File exists: <PATH>

salt/fileclient.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import
 
 # Import python libs
 import contextlib
+import errno
 import logging
 import os
 import string
@@ -142,12 +143,19 @@ class Client(object):
                                     path)
         destdir = os.path.dirname(dest)
         cumask = os.umask(63)
-        if not os.path.isdir(destdir):
-            # remove destdir if it is a regular file to avoid an OSError when
-            # running os.makedirs below
-            if os.path.isfile(destdir):
-                os.remove(destdir)
+
+        # remove destdir if it is a regular file to avoid an OSError when
+        # running os.makedirs below
+        if os.path.isfile(destdir):
+            os.remove(destdir)
+
+        # ensure destdir exists
+        try:
             os.makedirs(destdir)
+        except OSError as exc:
+            if exc.errno != errno.EEXIST: # ignore if it was there already
+                raise
+
         yield dest
         os.umask(cumask)