Package pythonutils :: Module pathutils
[hide private]
[frames] | no frames]

Source Code for Module pythonutils.pathutils

  1  # 2005/12/06 
  2  # Version 0.2.4 
  3  # pathutils.py 
  4  # Functions useful for working with files and paths. 
  5  # http://www.voidspace.org.uk/python/recipebook.shtml#utils 
  6   
  7  # Copyright Michael Foord 2004 
  8  # Released subject to the BSD License 
  9  # Please see http://www.voidspace.org.uk/python/license.shtml 
 10   
 11  # For information about bugfixes, updates and support, please join the Pythonutils mailing list. 
 12  # http://groups.google.com/group/pythonutils/ 
 13  # Comments, suggestions and bug reports welcome. 
 14  # Scripts maintained at http://www.voidspace.org.uk/python/index.shtml 
 15  # E-mail fuzzyman@voidspace.org.uk 
 16   
 17  """ 
 18  This module contains convenience functions for working with files and paths. 
 19  """ 
 20   
 21  __version__ = '0.2.4' 
 22   
 23  from __future__ import generators 
 24  import os 
 25  import sys 
 26  import time 
 27   
 28  __all__ = ( 
 29      'readlines', 
 30      'writelines', 
 31      'readbinary', 
 32      'writebinary', 
 33      'readfile', 
 34      'writefile', 
 35      'tslash', 
 36      'relpath', 
 37      'splitall', 
 38      'walkfiles', 
 39      'walkdirs', 
 40      'walkemptydirs', 
 41      'formatbytes', 
 42      'fullcopy', 
 43      'import_path', 
 44      'onerror', 
 45      'get_main_dir', 
 46      'main_is_frozen', 
 47      'Lock', 
 48      'LockError', 
 49      'LockFile', 
 50      '__version__', 
 51      ) 
 52   
 53  ###################################### 
 54  # Functions to read and write files in text and binary mode. 
 55   
56 -def readlines(filename):
57 """Passed a filename, it reads it, and returns a list of lines. (Read in text mode)""" 58 filehandle = open(filename, 'r') 59 outfile = filehandle.readlines() 60 filehandle.close() 61 return outfile
62
63 -def writelines(filename, infile, newline=False):
64 """ 65 Given a filename and a list of lines it writes the file. (In text mode) 66 67 If ``newline`` is ``True`` (default is ``False``) it adds a newline to each 68 line. 69 """ 70 filehandle = open(filename, 'w') 71 if newline: 72 infile = [line + '\n' for line in infile] 73 filehandle.writelines(infile) 74 filehandle.close()
75
76 -def readbinary(filename):
77 """Given a filename, read a file in binary mode. It returns a single string.""" 78 filehandle = open(filename, 'rb') 79 thisfile = filehandle.read() 80 filehandle.close() 81 return thisfile
82
83 -def writebinary(filename, infile):
84 """Given a filename and a string, write the file in binary mode. """ 85 filehandle = open(filename, 'wb') 86 filehandle.write(infile) 87 filehandle.close()
88
89 -def readfile(filename):
90 """Given a filename, read a file in text mode. It returns a single string.""" 91 filehandle = open(filename, 'r') 92 outfile = filehandle.read() 93 filehandle.close() 94 return outfile
95
96 -def writefile(filename, infile):
97 """Given a filename and a string, write the file in text mode.""" 98 filehandle = open(filename, 'w') 99 filehandle.write(infile) 100 filehandle.close()
101 102 #################################################################### 103 # Some functions for dealing with paths 104
105 -def tslash(apath):
106 """ 107 Add a trailing slash (``/``) to a path if it lacks one. 108 109 It doesn't use ``os.sep`` because you end up in trouble on windoze, when you 110 want separators for URLs. 111 """ 112 if apath and apath != '.' and not apath.endswith('/') and not apath.endswith('\\'): 113 return apath + '/' 114 else: 115 return apath
116
117 -def relpath(origin, dest):
118 """ 119 Return the relative path between origin and dest. 120 121 If it's not possible return dest. 122 123 124 If they are identical return ``os.curdir`` 125 126 Adapted from `path.py <http://www.jorendorff.com/articles/python/path/>`_ by Jason Orendorff. 127 """ 128 origin = os.path.abspath(origin).replace('\\', '/') 129 dest = os.path.abspath(dest).replace('\\', '/') 130 # 131 orig_list = splitall(os.path.normcase(origin)) 132 # Don't normcase dest! We want to preserve the case. 133 dest_list = splitall(dest) 134 # 135 if orig_list[0] != os.path.normcase(dest_list[0]): 136 # Can't get here from there. 137 return dest 138 # 139 # Find the location where the two paths start to differ. 140 i = 0 141 for start_seg, dest_seg in zip(orig_list, dest_list): 142 if start_seg != os.path.normcase(dest_seg): 143 break 144 i += 1 145 # 146 # Now i is the point where the two paths diverge. 147 # Need a certain number of "os.pardir"s to work up 148 # from the origin to the point of divergence. 149 segments = [os.pardir] * (len(orig_list) - i) 150 # Need to add the diverging part of dest_list. 151 segments += dest_list[i:] 152 if len(segments) == 0: 153 # If they happen to be identical, use os.curdir. 154 return os.curdir 155 else: 156 return os.path.join(*segments).replace('\\', '/')
157
158 -def splitall(loc):
159 """ 160 Return a list of the path components in loc. (Used by relpath_). 161 162 The first item in the list will be either ``os.curdir``, ``os.pardir``, empty, 163 or the root directory of loc (for example, ``/`` or ``C:\\). 164 165 The other items in the list will be strings. 166 167 Adapted from *path.py* by Jason Orendorff. 168 """ 169 parts = [] 170 while loc != os.curdir and loc != os.pardir: 171 prev = loc 172 loc, child = os.path.split(prev) 173 if loc == prev: 174 break 175 parts.append(child) 176 parts.append(loc) 177 parts.reverse() 178 return parts
179 180 ####################################################################### 181 # a pre 2.3 walkfiles function - adapted from the path module by Jason Orendorff 182 183 join = os.path.join 184 isdir = os.path.isdir 185 isfile = os.path.isfile 186
187 -def walkfiles(thisdir):
188 """ 189 walkfiles(D) -> iterator over files in D, recursively. Yields full file paths. 190 191 Adapted from path.py by Jason Orendorff. 192 """ 193 for child in os.listdir(thisdir): 194 thischild = join(thisdir, child) 195 if isfile(thischild): 196 yield thischild 197 elif isdir(thischild): 198 for f in walkfiles(thischild): 199 yield f
200
201 -def walkdirs(thisdir):
202 """ 203 Walk through all the subdirectories in a tree. Recursively yields directory 204 names (full paths). 205 """ 206 for child in os.listdir(thisdir): 207 thischild = join(thisdir, child) 208 if isfile(thischild): 209 continue 210 elif isdir(thischild): 211 for f in walkdirs(thischild): 212 yield f 213 yield thischild
214
215 -def walkemptydirs(thisdir):
216 """ 217 Recursively yield names of *empty* directories. 218 219 These are the only paths omitted when using ``walkfiles``. 220 """ 221 if not os.listdir(thisdir): 222 # if the directory is empty.. then yield it 223 yield thisdir 224 for child in os.listdir(thisdir): 225 thischild = join(thisdir, child) 226 if isdir(thischild): 227 for emptydir in walkemptydirs(thischild): 228 yield emptydir
229 230 ############################################################### 231 # formatbytes takes a filesize (as returned by os.getsize() ) 232 # and formats it for display in one of two ways !! 233
234 -def formatbytes(sizeint, configdict=None, **configs):
235 """ 236 Given a file size as an integer, return a nicely formatted string that 237 represents the size. Has various options to control it's output. 238 239 You can pass in a dictionary of arguments or keyword arguments. Keyword 240 arguments override the dictionary and there are sensible defaults for options 241 you don't set. 242 243 Options and defaults are as follows : 244 245 * ``forcekb = False`` - If set this forces the output to be in terms 246 of kilobytes and bytes only. 247 248 * ``largestonly = True`` - If set, instead of outputting 249 ``1 Mbytes, 307 Kbytes, 478 bytes`` it outputs using only the largest 250 denominator - e.g. ``1.3 Mbytes`` or ``17.2 Kbytes`` 251 252 * ``kiloname = 'Kbytes'`` - The string to use for kilobytes 253 254 * ``meganame = 'Mbytes'`` - The string to use for Megabytes 255 256 * ``bytename = 'bytes'`` - The string to use for bytes 257 258 * ``nospace = True`` - If set it outputs ``1Mbytes, 307Kbytes``, 259 notice there is no space. 260 261 Example outputs : :: 262 263 19Mbytes, 75Kbytes, 255bytes 264 2Kbytes, 0bytes 265 23.8Mbytes 266 267 .. note:: 268 269 It currently uses the plural form even for singular. 270 """ 271 defaultconfigs = { 'forcekb' : False, 272 'largestonly' : True, 273 'kiloname' : 'Kbytes', 274 'meganame' : 'Mbytes', 275 'bytename' : 'bytes', 276 'nospace' : True} 277 if configdict is None: 278 configdict = {} 279 for entry in configs: 280 # keyword parameters override the dictionary passed in 281 configdict[entry] = configs[entry] 282 # 283 for keyword in defaultconfigs: 284 if not configdict.has_key(keyword): 285 configdict[keyword] = defaultconfigs[keyword] 286 # 287 if configdict['nospace']: 288 space = '' 289 else: 290 space = ' ' 291 # 292 mb, kb, rb = bytedivider(sizeint) 293 if configdict['largestonly']: 294 if mb and not configdict['forcekb']: 295 return stringround(mb, kb)+ space + configdict['meganame'] 296 elif kb or configdict['forcekb']: 297 if mb and configdict['forcekb']: 298 kb += 1024*mb 299 return stringround(kb, rb) + space+ configdict['kiloname'] 300 else: 301 return str(rb) + space + configdict['bytename'] 302 else: 303 outstr = '' 304 if mb and not configdict['forcekb']: 305 outstr = str(mb) + space + configdict['meganame'] +', ' 306 if kb or configdict['forcekb'] or mb: 307 if configdict['forcekb']: 308 kb += 1024*mb 309 outstr += str(kb) + space + configdict['kiloname'] +', ' 310 return outstr + str(rb) + space + configdict['bytename']
311
312 -def stringround(main, rest):
313 """ 314 Given a file size in either (mb, kb) or (kb, bytes) - round it 315 appropriately. 316 """ 317 # divide an int by a float... get a float 318 value = main + rest/1024.0 319 return str(round(value, 1))
320
321 -def bytedivider(nbytes):
322 """ 323 Given an integer (probably a long integer returned by os.getsize() ) 324 it returns a tuple of (megabytes, kilobytes, bytes). 325 326 This can be more easily converted into a formatted string to display the 327 size of the file. 328 """ 329 mb, remainder = divmod(nbytes, 1048576) 330 kb, rb = divmod(remainder, 1024) 331 return (mb, kb, rb)
332 333 ######################################## 334
335 -def fullcopy(src, dst):
336 """ 337 Copy file from src to dst. 338 339 If the dst directory doesn't exist, we will attempt to create it using makedirs. 340 """ 341 import shutil 342 if not os.path.isdir(os.path.dirname(dst)): 343 os.makedirs(os.path.dirname(dst)) 344 shutil.copy(src, dst)
345 346 ####################################### 347
348 -def import_path(fullpath, strict=True):
349 """ 350 Import a file from the full path. Allows you to import from anywhere, 351 something ``__import__`` does not do. 352 353 If strict is ``True`` (the default), raise an ``ImportError`` if the module 354 is found in the "wrong" directory. 355 356 Taken from firedrop2_ by `Hans Nowak`_ 357 358 .. _firedrop2: http://www.voidspace.org.uk/python/firedrop2/ 359 .. _Hans Nowak: http://zephyrfalcon.org 360 """ 361 path, filename = os.path.split(fullpath) 362 filename, ext = os.path.splitext(filename) 363 sys.path.insert(0, path) 364 try: 365 module = __import__(filename) 366 except ImportError: 367 del sys.path[0] 368 raise 369 del sys.path[0] 370 # 371 if strict: 372 path = os.path.split(module.__file__)[0] 373 # FIXME: doesn't *startswith* allow room for errors ? 374 if not fullpath.startswith(path): 375 raise ImportError, "Module '%s' found, but not in '%s'" % ( 376 filename, fullpath) 377 # 378 return module
379 380 ############################################################################## 381 # These functions get us our directory name 382 # Even if py2exe or another freeze tool has been used 383
384 -def main_is_frozen():
385 """Return ``True`` if we're running from a frozen program.""" 386 import imp 387 return ( 388 # new py2exe 389 hasattr(sys, "frozen") or 390 # tools/freeze 391 imp.is_frozen("__main__"))
392
393 -def get_main_dir():
394 """Return the script directory - whether we're frozen or not.""" 395 if main_is_frozen(): 396 return os.path.abspath(os.path.dirname(sys.executable)) 397 return os.path.abspath(os.path.dirname(sys.argv[0]))
398 399 ############################## 400
401 -def onerror(func, path, exc_info):
402 """ 403 Error handler for ``shutil.rmtree``. 404 405 If the error is due to an access error (read only file) 406 it attempts to add write permission and then retries. 407 408 If the error is for another reason it re-raises the error. 409 410 Usage : ``shutil.rmtree(path, onerror=onerror)`` 411 """ 412 import stat 413 if not os.access(path, os.W_OK): 414 # Is the error an access error ? 415 os.chmod(path, stat.S_IWUSR) 416 func(path) 417 else: 418 raise
419 420 ########################################################## 421 # A set of object for providing simple, cross-platform file locking 422
423 -class LockError(IOError):
424 """The generic error for locking - it is a subclass of ``IOError``."""
425
426 -class Lock(object):
427 """A simple file lock, compatible with windows and Unixes.""" 428
429 - def __init__(self, filename, timeout=5, step=0.1):
430 """ 431 Create a ``Lock`` object on file ``filename`` 432 433 ``timeout`` is the time in seconds to wait before timing out, when 434 attempting to acquire the lock. 435 436 ``step`` is the number of seconds to wait in between each attempt to 437 acquire the lock. 438 439 """ 440 self.timeout = timeout 441 self.step = step 442 self.filename = filename 443 self.locked = False
444
445 - def lock(self, force=True):
446 """ 447 Lock the file for access by creating a directory of the same name (plus 448 a trailing underscore). 449 450 The file is only locked if you use this class to acquire the lock 451 before accessing. 452 453 If ``force`` is ``True`` (the default), then on timeout we forcibly 454 acquire the lock. 455 456 If ``force`` is ``False``, then on timeout a ``LockError`` is raised. 457 """ 458 if self.locked: 459 raise LockError('%s is already locked' % self.filename) 460 t = 0 461 name = self._mungedname() 462 while t < self.timeout: 463 t += self.step 464 try: 465 if os.apth.isdir(name): 466 raise os.error 467 else: 468 os.mkdir() 469 except os.error, err: 470 time.sleep(self.step) 471 else: 472 self.locked = True 473 return 474 if force: 475 self.locked = True 476 else: 477 raise LockError('Failed to acquire lock on %s' % self.filename)
478
479 - def unlock(self, ignore=True):
480 """ 481 Release the lock. 482 483 If ``ignore`` is ``True`` and removing the lock directory fails, then 484 the error is surpressed. (This may happen if the lock was acquired 485 via a timeout.) 486 """ 487 if not self.locked: 488 raise LockError('%s is not locked' % self.filename) 489 self.locked = False 490 try: 491 os.rmdir(self._mungedname()) 492 except os.error, err: 493 if not ignore: 494 raise LockError('unlocking appeared to fail - %s' % 495 self.filename)
496
497 - def _mungedname(self):
498 """ 499 Override this in a subclass if you want to change the way ``Lock`` 500 creates the directory name. 501 """ 502 return self.filename + '_'
503
504 - def __del__(self):
505 """Auto unlock when object is deleted.""" 506 if self.locked: 507 self.unlock()
508
509 -class LockFile(Lock):
510 """ 511 A file like object with an exclusive lock, whilst it is open. 512 513 The lock is provided by the ``Lock`` class, which creates a directory 514 with the same name as the file (plus a trailing underscore), to indicate 515 that the file is locked. 516 517 This is simple and cross platform, with some limitations : 518 519 * Unusual process termination could result in the directory 520 being left. 521 * The process acquiring the lock must have permission to create a 522 directory in the same location as the file. 523 * It only locks the file against other processes that attempt to 524 acquire a lock using ``LockFile`` or ``Lock``. 525 """ 526
527 - def __init__(self, filename, mode='r', bufsize=-1, timeout=5, step=0.1, 528 force=True):
529 """ 530 Create a file like object that is locked (using the ``Lock`` class) 531 until it is closed. 532 533 The file is only locked against another process that attempts to 534 acquire a lock using ``Lock`` (or ``LockFile``). 535 536 The lock is released automatically when the file is closed. 537 538 The filename, mode and bufsize arguments have the same meaning as for 539 the built in function ``open``. 540 541 The timeout and step arguments have the same meaning as for a ``Lock`` 542 object. 543 544 The force argument has the same meaning as for the ``Lock.lock`` method. 545 546 A ``LockFile`` object has all the normal ``file`` methods and 547 attributes. 548 """ 549 Lock.__init__(self, filename, timeout, step) 550 # may raise an error if lock is ``False`` 551 self.lock(force) 552 # may also raise an error 553 self._file = open(filename, mode, bufsize)
554
555 - def close(self, ignore=True):
556 """ 557 close the file and release the lock. 558 559 ignore has the same meaning as for ``Lock.unlock`` 560 """ 561 self._file.close() 562 self.unlock(ignore)
563
564 - def __getattr__(self, name):
565 """delegate appropriate method/attribute calls to the file.""" 566 if name not in self.__dict__: 567 return getattr(self._file, name) 568 else: 569 return self.__dict__[self, name]
570
571 - def __setattr__(self, name, value):
572 """Only allow attribute setting that don't clash with the file.""" 573 if not '_file' in self.__dict__: 574 Lock.__setattr__(self, name, value) 575 elif hasattr(self._file, name): 576 return setattr(self._file, name, value) 577 else: 578 Lock.__setattr__(self, name, value)
579
580 - def __del__(self):
581 """Auto unlock (and close file) when object is deleted.""" 582 if self.locked: 583 self.unlock() 584 self._file.close()
585 586 """ 587 588 Changelog 589 ========= 590 591 2005/02/03 592 ---------- 593 594 Added a workaround in ``Lock`` for operating systems that don't raise 595 ``os.error`` when attempting to create a directory that already exists. 596 597 2005/12/06 Version 0.2.4 598 ----------------------------- 599 600 Fixed bug in ``onerror``. (Missing stat import) 601 602 603 2005/11/26 Version 0.2.3 604 ----------------------------- 605 606 Added ``Lock``, ``LockError``, and ``LockFile`` 607 608 Added ``__version__`` 609 610 611 2005/11/13 Version 0.2.2 612 ----------------------------- 613 614 Added the py2exe support functions. 615 616 Added ``onerror``. 617 618 619 2005/08/28 Version 0.2.1 620 ----------------------------- 621 622 * Added ``import_path`` 623 * Added ``__all__`` 624 * Code cleanup 625 626 627 2005/06/01 Version 0.2.0 628 ----------------------------- 629 630 Added ``walkdirs`` generator. 631 632 633 2005/03/11 Version 0.1.1 634 ----------------------------- 635 636 Added rounding to ``formatbytes`` and improved ``bytedivider`` with ``divmod``. 637 638 Now explicit keyword parameters override the ``configdict`` in ``formatbytes``. 639 640 641 2005/02/18 Version 0.1.0 642 ----------------------------- 643 644 The first numbered version. 645 """ 646