1  # 2005/12/06 
  2  # Version 0.2.4 
  3  # pathutils.py 
  4  # Functions useful for working with files and paths. 
  5  # http://www.voidspace.org.uk/python/recipebook.shtml#utils 
  6   
  7  # Copyright Michael Foord 2004 
  8  # Released subject to the BSD License 
  9  # Please see http://www.voidspace.org.uk/python/license.shtml 
 10   
 11  # For information about bugfixes, updates and support, please join the Pythonutils mailing list. 
 12  # http://groups.google.com/group/pythonutils/ 
 13  # Comments, suggestions and bug reports welcome. 
 14  # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml 
 15  # E-mail fuzzyman@voidspace.org.uk 
 16   
 17  """ 
 18  This module contains convenience functions for working with files and paths. 
 19  """ 
 20   
 21  __version__ = '0.2.4' 
 22   
 23  from __future__ import generators 
 24  import os 
 25  import sys 
 26  import time 
 27   
 28  __all__ = ( 
 29      'readlines', 
 30      'writelines', 
 31      'readbinary', 
 32      'writebinary', 
 33      'readfile', 
 34      'writefile', 
 35      'tslash', 
 36      'relpath', 
 37      'splitall', 
 38      'walkfiles', 
 39      'walkdirs', 
 40      'walkemptydirs', 
 41      'formatbytes', 
 42      'fullcopy', 
 43      'import_path', 
 44      'onerror', 
 45      'get_main_dir', 
 46      'main_is_frozen', 
 47      'Lock', 
 48      'LockError', 
 49      'LockFile', 
 50      '__version__', 
 51      ) 
 52   
 53  ###################################### 
 54  # Functions to read and write files in text and binary mode. 
 55   
 56 -def readlines(filename): 
 57      """Passed a filename, it reads it, and returns a list of lines. (Read in text mode)""" 
 58      filehandle = open(filename, 'r') 
 59      outfile = filehandle.readlines() 
 60      filehandle.close() 
 61      return outfile 
 62   
 63 -def writelines(filename, infile, newline=False): 
 64      """ 
 65      Given a filename and a list of lines it writes the file. (In text mode) 
 66       
 67      If ``newline`` is ``True`` (default is ``False``) it adds a newline to each 
 68      line. 
 69      """ 
 70      filehandle = open(filename, 'w') 
 71      if newline: 
 72          infile = [line + '\n' for line in infile] 
 73      filehandle.writelines(infile) 
 74      filehandle.close() 
 75   
 76 -def readbinary(filename): 
 77      """Given a filename, read a file in binary mode. It returns a single string.""" 
 78      filehandle = open(filename, 'rb') 
 79      thisfile = filehandle.read() 
 80      filehandle.close() 
 81      return thisfile 
 82   
 83 -def writebinary(filename, infile): 
 84      """Given a filename and a string, write the file in binary mode. """ 
 85      filehandle = open(filename, 'wb') 
 86      filehandle.write(infile) 
 87      filehandle.close() 
 88   
 89 -def readfile(filename): 
 90      """Given a filename, read a file in text mode. It returns a single string.""" 
 91      filehandle = open(filename, 'r') 
 92      outfile = filehandle.read() 
 93      filehandle.close() 
 94      return outfile 
 95   
 96 -def writefile(filename, infile): 
 97      """Given a filename and a string, write the file in text mode.""" 
 98      filehandle = open(filename, 'w') 
 99      filehandle.write(infile) 
100      filehandle.close() 
101       
102  #################################################################### 
103  # Some functions for dealing with paths 
104   
105 -def tslash(apath): 
106      """ 
107      Add a trailing slash (``/``) to a path if it lacks one. 
108       
109      It doesn't use ``os.sep`` because you end up in trouble on windoze, when you 
110      want separators for URLs. 
111      """ 
112      if apath and apath != '.' and not apath.endswith('/') and not apath.endswith('\\'): 
113          return apath + '/' 
114      else: 
115          return apath 
116   
117 -def relpath(origin, dest): 
118      """ 
119      Return the relative path between origin and dest. 
120       
121      If it's not possible return dest. 
122       
123       
124      If they are identical return ``os.curdir`` 
125       
126      Adapted from `path.py <http://www.jorendorff.com/articles/python/path/>`_ by Jason Orendorff.  
127      """ 
128      origin = os.path.abspath(origin).replace('\\', '/') 
129      dest = os.path.abspath(dest).replace('\\', '/') 
130      # 
131      orig_list = splitall(os.path.normcase(origin)) 
132      # Don't normcase dest!  We want to preserve the case. 
133      dest_list = splitall(dest) 
134      # 
135      if orig_list[0] != os.path.normcase(dest_list[0]): 
136          # Can't get here from there. 
137          return dest 
138      # 
139      # Find the location where the two paths start to differ. 
140      i = 0 
141      for start_seg, dest_seg in zip(orig_list, dest_list): 
142          if start_seg != os.path.normcase(dest_seg): 
143              break 
144          i += 1 
145      # 
146      # Now i is the point where the two paths diverge. 
147      # Need a certain number of "os.pardir"s to work up 
148      # from the origin to the point of divergence. 
149      segments = [os.pardir] * (len(orig_list) - i) 
150      # Need to add the diverging part of dest_list. 
151      segments += dest_list[i:] 
152      if len(segments) == 0: 
153          # If they happen to be identical, use os.curdir. 
154          return os.curdir 
155      else: 
156          return os.path.join(*segments).replace('\\', '/') 
157   
158 -def splitall(loc): 
159      """ 
160      Return a list of the path components in loc. (Used by relpath_). 
161       
162      The first item in the list will be  either ``os.curdir``, ``os.pardir``, empty, 
163      or the root directory of loc (for example, ``/`` or ``C:\\). 
164       
165      The other items in the list will be strings. 
166           
167      Adapted from *path.py* by Jason Orendorff. 
168      """ 
169      parts = [] 
170      while loc != os.curdir and loc != os.pardir: 
171          prev = loc 
172          loc, child = os.path.split(prev) 
173          if loc == prev: 
174              break 
175          parts.append(child) 
176      parts.append(loc) 
177      parts.reverse() 
178      return parts 
179   
180  ####################################################################### 
181  # a pre 2.3 walkfiles function - adapted from the path module by Jason Orendorff 
182   
183  join = os.path.join 
184  isdir = os.path.isdir 
185  isfile = os.path.isfile 
186   
187 -def walkfiles(thisdir): 
188      """ 
189      walkfiles(D) -> iterator over files in D, recursively. Yields full file paths. 
190       
191      Adapted from path.py by Jason Orendorff. 
192      """ 
193      for child in os.listdir(thisdir): 
194          thischild = join(thisdir, child) 
195          if isfile(thischild): 
196              yield thischild 
197          elif isdir(thischild): 
198              for f in walkfiles(thischild): 
199                  yield f 
200                   
201 -def walkdirs(thisdir): 
202      """ 
203      Walk through all the subdirectories in a tree. Recursively yields directory 
204      names (full paths). 
205      """ 
206      for child in os.listdir(thisdir): 
207          thischild = join(thisdir, child) 
208          if isfile(thischild): 
209              continue 
210          elif isdir(thischild): 
211              for f in walkdirs(thischild): 
212                  yield f 
213              yield thischild 
214   
215 -def walkemptydirs(thisdir): 
216      """ 
217      Recursively yield names of *empty* directories. 
218       
219      These are the only paths omitted when using ``walkfiles``. 
220      """ 
221      if not os.listdir(thisdir): 
222          # if the directory is empty.. then yield it 
223          yield thisdir    
224      for child in os.listdir(thisdir): 
225          thischild = join(thisdir, child) 
226          if isdir(thischild): 
227              for emptydir in walkemptydirs(thischild): 
228                  yield emptydir 
229   
230  ############################################################### 
231  # formatbytes takes a filesize (as returned by os.getsize() ) 
232  # and formats it for display in one of two ways !! 
233   
234 -def formatbytes(sizeint, configdict=None, **configs): 
235      """ 
236      Given a file size as an integer, return a nicely formatted string that 
237      represents the size. Has various options to control it's output. 
238       
239      You can pass in a dictionary of arguments or keyword arguments. Keyword 
240      arguments override the dictionary and there are sensible defaults for options 
241      you don't set. 
242       
243      Options and defaults are as follows : 
244       
245      *    ``forcekb = False`` -         If set this forces the output to be in terms 
246      of kilobytes and bytes only. 
247       
248      *    ``largestonly = True`` -    If set, instead of outputting  
249          ``1 Mbytes, 307 Kbytes, 478 bytes`` it outputs using only the largest  
250          denominator - e.g. ``1.3 Mbytes`` or ``17.2 Kbytes`` 
251       
252      *    ``kiloname = 'Kbytes'`` -    The string to use for kilobytes 
253       
254      *    ``meganame = 'Mbytes'`` - The string to use for Megabytes 
255       
256      *    ``bytename = 'bytes'`` -     The string to use for bytes 
257       
258      *    ``nospace = True`` -        If set it outputs ``1Mbytes, 307Kbytes``,  
259          notice there is no space. 
260       
261      Example outputs : :: 
262       
263          19Mbytes, 75Kbytes, 255bytes 
264          2Kbytes, 0bytes 
265          23.8Mbytes 
266       
267      .. note:: 
268       
269          It currently uses the plural form even for singular. 
270      """ 
271      defaultconfigs = {  'forcekb' : False, 
272                          'largestonly' : True, 
273                          'kiloname' : 'Kbytes', 
274                          'meganame' : 'Mbytes', 
275                          'bytename' : 'bytes', 
276                          'nospace' : True} 
277      if configdict is None: 
278          configdict = {} 
279      for entry in configs: 
280          # keyword parameters override the dictionary passed in 
281          configdict[entry] = configs[entry] 
282      # 
283      for keyword in defaultconfigs: 
284          if not configdict.has_key(keyword): 
285              configdict[keyword] = defaultconfigs[keyword] 
286      # 
287      if configdict['nospace']: 
288          space = '' 
289      else: 
290          space = ' ' 
291      # 
292      mb, kb, rb = bytedivider(sizeint) 
293      if configdict['largestonly']: 
294          if mb and not configdict['forcekb']: 
295              return stringround(mb, kb)+ space + configdict['meganame'] 
296          elif kb or configdict['forcekb']: 
297              if mb and configdict['forcekb']: 
298                  kb += 1024*mb 
299              return stringround(kb, rb) + space+ configdict['kiloname'] 
300          else: 
301              return str(rb) + space + configdict['bytename'] 
302      else: 
303          outstr = '' 
304          if mb and not configdict['forcekb']: 
305              outstr = str(mb) + space + configdict['meganame'] +', ' 
306          if kb or configdict['forcekb'] or mb: 
307              if configdict['forcekb']: 
308                  kb += 1024*mb 
309              outstr += str(kb) + space + configdict['kiloname'] +', ' 
310          return outstr + str(rb) + space + configdict['bytename'] 
311   
312 -def stringround(main, rest): 
313      """ 
314      Given a file size in either (mb, kb) or (kb, bytes) - round it 
315      appropriately. 
316      """ 
317      # divide an int by a float... get a float 
318      value = main + rest/1024.0 
319      return str(round(value, 1)) 
320   
321 -def bytedivider(nbytes): 
322      """ 
323      Given an integer (probably a long integer returned by os.getsize() ) 
324      it returns a tuple of (megabytes, kilobytes, bytes). 
325       
326      This can be more easily converted into a formatted string to display the 
327      size of the file. 
328      """ 
329      mb, remainder = divmod(nbytes, 1048576) 
330      kb, rb = divmod(remainder, 1024) 
331      return (mb, kb, rb) 
332   
333  ######################################## 
334   
335 -def fullcopy(src, dst): 
336      """ 
337      Copy file from src to dst. 
338       
339      If the dst directory doesn't exist, we will attempt to create it using makedirs. 
340      """ 
341      import shutil 
342      if not os.path.isdir(os.path.dirname(dst)): 
343          os.makedirs(os.path.dirname(dst)) 
344      shutil.copy(src, dst)     
345   
346  ####################################### 
347   
348 -def import_path(fullpath, strict=True): 
349      """ 
350      Import a file from the full path. Allows you to import from anywhere, 
351      something ``__import__`` does not do. 
352       
353      If strict is ``True`` (the default), raise an ``ImportError`` if the module 
354      is found in the "wrong" directory. 
355       
356      Taken from firedrop2_ by `Hans Nowak`_ 
357       
358      .. _firedrop2: http://www.voidspace.org.uk/python/firedrop2/ 
359      .. _Hans Nowak: http://zephyrfalcon.org 
360      """ 
361      path, filename = os.path.split(fullpath) 
362      filename, ext = os.path.splitext(filename) 
363      sys.path.insert(0, path) 
364      try: 
365          module = __import__(filename) 
366      except ImportError: 
367          del sys.path[0] 
368          raise 
369      del sys.path[0] 
370      # 
371      if strict: 
372          path = os.path.split(module.__file__)[0] 
373          # FIXME: doesn't *startswith* allow room for errors ? 
374          if not fullpath.startswith(path): 
375              raise ImportError, "Module '%s' found, but not in '%s'" % ( 
376                    filename, fullpath) 
377      # 
378      return module 
379   
380  ############################################################################## 
381  # These functions get us our directory name 
382  # Even if py2exe or another freeze tool has been used 
383   
384 -def main_is_frozen(): 
385      """Return ``True`` if we're running from a frozen program.""" 
386      import imp 
387      return ( 
388          # new py2exe 
389          hasattr(sys, "frozen") or 
390          # tools/freeze 
391          imp.is_frozen("__main__")) 
392   
393 -def get_main_dir(): 
394      """Return the script directory - whether we're frozen or not.""" 
395      if main_is_frozen(): 
396          return os.path.abspath(os.path.dirname(sys.executable)) 
397      return os.path.abspath(os.path.dirname(sys.argv[0])) 
398   
399  ############################## 
400   
401 -def onerror(func, path, exc_info): 
402      """ 
403      Error handler for ``shutil.rmtree``. 
404   
405      If the error is due to an access error (read only file) 
406      it attempts to add write permission and then retries. 
407   
408      If the error is for another reason it re-raises the error. 
409       
410      Usage : ``shutil.rmtree(path, onerror=onerror)`` 
411      """ 
412      import stat 
413      if not os.access(path, os.W_OK): 
414          # Is the error an access error ? 
415          os.chmod(path, stat.S_IWUSR) 
416          func(path) 
417      else: 
418          raise 
419   
420  ########################################################## 
421  # A set of object for providing simple, cross-platform file locking 
422   
423 -class LockError(IOError): 
424      """The generic error for locking - it is a subclass of ``IOError``.""" 
425   
426 -class Lock(object): 
427      """A simple file lock, compatible with windows and Unixes.""" 
428   
429 -    def __init__(self, filename, timeout=5, step=0.1): 
430          """ 
431          Create a ``Lock`` object on file ``filename`` 
432           
433          ``timeout`` is the time in seconds to wait before timing out, when 
434          attempting to acquire the lock. 
435           
436          ``step`` is the number of seconds to wait in between each attempt to 
437          acquire the lock. 
438           
439          """ 
440          self.timeout = timeout 
441          self.step = step 
442          self.filename = filename 
443          self.locked = False 
444   
445 -    def lock(self, force=True): 
446          """ 
447          Lock the file for access by creating a directory of the same name (plus 
448          a trailing underscore). 
449           
450          The file is only locked if you use this class to acquire the lock 
451          before accessing. 
452           
453          If ``force`` is ``True`` (the default), then on timeout we forcibly 
454          acquire the lock. 
455           
456          If ``force`` is ``False``, then on timeout a ``LockError`` is raised. 
457          """ 
458          if self.locked: 
459              raise LockError('%s is already locked' % self.filename) 
460          t = 0 
461          name = self._mungedname() 
462          while t < self.timeout: 
463              t += self.step 
464              try: 
465                  if os.apth.isdir(name): 
466                      raise os.error 
467                  else: 
468                      os.mkdir() 
469              except os.error, err: 
470                  time.sleep(self.step) 
471              else: 
472                  self.locked = True 
473                  return 
474          if force: 
475              self.locked = True 
476          else: 
477              raise LockError('Failed to acquire lock on %s' % self.filename) 
478   
479 -    def unlock(self, ignore=True): 
480          """ 
481          Release the lock. 
482           
483          If ``ignore`` is ``True`` and removing the lock directory fails, then 
484          the error is surpressed. (This may happen if the lock was acquired 
485          via a timeout.) 
486          """ 
487          if not self.locked: 
488              raise LockError('%s is not locked' % self.filename) 
489          self.locked = False 
490          try: 
491              os.rmdir(self._mungedname()) 
492          except os.error, err: 
493              if not ignore: 
494                  raise LockError('unlocking appeared to fail - %s' % 
495                      self.filename) 
496   
497 -    def _mungedname(self): 
498          """ 
499          Override this in a subclass if you want to change the way ``Lock``  
500          creates the directory name. 
501          """ 
502          return self.filename + '_' 
503   
504 -    def __del__(self): 
505          """Auto unlock when object is deleted.""" 
506          if self.locked: 
507              self.unlock() 
508   
509 -class LockFile(Lock): 
510      """ 
511      A file like object with an exclusive lock, whilst it is open. 
512       
513      The lock is provided by the ``Lock`` class, which creates a directory 
514      with the same name as the file (plus a trailing underscore), to indicate 
515      that the file is locked. 
516       
517      This is simple and cross platform, with some limitations : 
518       
519          * Unusual process termination could result in the directory 
520            being left. 
521          * The process acquiring the lock must have permission to create a 
522            directory in the same location as the file. 
523          * It only locks the file against other processes that attempt to 
524            acquire a lock using ``LockFile`` or ``Lock``. 
525      """ 
526   
527 -    def __init__(self, filename, mode='r', bufsize=-1, timeout=5, step=0.1, 
528          force=True): 
529          """ 
530          Create a file like object that is locked (using the ``Lock`` class) 
531          until it is closed. 
532           
533          The file is only locked against another process that attempts to 
534          acquire a lock using ``Lock`` (or ``LockFile``). 
535           
536          The lock is released automatically when the file is closed. 
537           
538          The filename, mode and bufsize arguments have the same meaning as for 
539          the built in function ``open``. 
540           
541          The timeout and step arguments have the same meaning as for a ``Lock`` 
542          object. 
543           
544          The force argument has the same meaning as for the ``Lock.lock`` method. 
545           
546          A ``LockFile`` object has all the normal ``file`` methods and 
547          attributes. 
548          """ 
549          Lock.__init__(self, filename, timeout, step) 
550          # may raise an error if lock is ``False`` 
551          self.lock(force) 
552          # may also raise an error 
553          self._file = open(filename, mode, bufsize) 
554   
555 -    def close(self, ignore=True): 
556          """ 
557          close the file and release the lock. 
558           
559          ignore has the same meaning as for ``Lock.unlock`` 
560          """ 
561          self._file.close() 
562          self.unlock(ignore) 
563   
564 -    def __getattr__(self, name): 
565          """delegate appropriate method/attribute calls to the file.""" 
566          if name not in self.__dict__: 
567              return getattr(self._file, name) 
568          else: 
569              return self.__dict__[self, name] 
570   
571 -    def __setattr__(self, name, value): 
572          """Only allow attribute setting that don't clash with the file.""" 
573          if not '_file' in self.__dict__: 
574              Lock.__setattr__(self, name, value) 
575          elif hasattr(self._file, name): 
576              return setattr(self._file, name, value) 
577          else: 
578              Lock.__setattr__(self, name, value) 
579   
580 -    def __del__(self): 
581          """Auto unlock (and close file) when object is deleted.""" 
582          if self.locked: 
583              self.unlock() 
584              self._file.close() 
585   
586  """ 
587   
588  Changelog 
589  ========= 
590   
591  2005/02/03 
592  ---------- 
593   
594  Added a workaround in ``Lock`` for operating systems that don't raise 
595  ``os.error`` when attempting to create a directory that already exists. 
596   
597  2005/12/06      Version 0.2.4 
598  ----------------------------- 
599   
600  Fixed bug in ``onerror``. (Missing stat import) 
601   
602   
603  2005/11/26      Version 0.2.3 
604  ----------------------------- 
605   
606  Added ``Lock``, ``LockError``, and ``LockFile`` 
607   
608  Added ``__version__`` 
609   
610   
611  2005/11/13      Version 0.2.2 
612  ----------------------------- 
613   
614  Added the py2exe support functions. 
615   
616  Added ``onerror``. 
617   
618   
619  2005/08/28      Version 0.2.1 
620  ----------------------------- 
621   
622  * Added ``import_path`` 
623  * Added ``__all__`` 
624  * Code cleanup 
625   
626   
627  2005/06/01      Version 0.2.0 
628  ----------------------------- 
629   
630  Added ``walkdirs`` generator. 
631   
632   
633  2005/03/11      Version 0.1.1 
634  ----------------------------- 
635   
636  Added rounding to ``formatbytes`` and improved ``bytedivider`` with ``divmod``. 
637   
638  Now explicit keyword parameters override the ``configdict`` in ``formatbytes``. 
639   
640   
641  2005/02/18      Version 0.1.0 
642  ----------------------------- 
643   
644  The first numbered version. 
645  """ 
646