1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides general-purpose utilities.
43
44 @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList,
45 RegexList, _Vertex, DirectedGraph, PathResolverSingleton,
46 sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine,
47 resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice,
48 deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,
49 ISO_SECTOR_SIZE, BYTES_PER_SECTOR,
50 BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,
51 SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,
52 UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS
53
54 @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes.
55 @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector.
56 @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB).
57 @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB).
58 @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB).
59 @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB).
60 @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB).
61 @var SECONDS_PER_MINUTE: Number of seconds per minute.
62 @var MINUTES_PER_HOUR: Number of minutes per hour.
63 @var HOURS_PER_DAY: Number of hours per day.
64 @var SECONDS_PER_DAY: Number of seconds per day.
65 @var UNIT_BYTES: Constant representing the byte (B) unit for conversion.
66 @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion.
67 @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion.
68 @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion.
69 @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion.
70
71 @author: Kenneth J. Pronovici <pronovic@ieee.org>
72 """
73
74
75
76
77
78
79 import sys
80 import math
81 import os
82 import re
83 import time
84 import logging
85 from subprocess import Popen, STDOUT, PIPE
86 from functools import total_ordering
87 from numbers import Real
88 from decimal import Decimal
89 import collections
90
91 try:
92 import pwd
93 import grp
94 _UID_GID_AVAILABLE = True
95 except ImportError:
96 _UID_GID_AVAILABLE = False
97
98 from CedarBackup3.release import VERSION, DATE
99
100
101
102
103
104
105 logger = logging.getLogger("CedarBackup3.log.util")
106 outputLogger = logging.getLogger("CedarBackup3.output")
107
108 ISO_SECTOR_SIZE = 2048.0
109 BYTES_PER_SECTOR = ISO_SECTOR_SIZE
110
111 BYTES_PER_KBYTE = 1024.0
112 KBYTES_PER_MBYTE = 1024.0
113 MBYTES_PER_GBYTE = 1024.0
114 BYTES_PER_MBYTE = BYTES_PER_KBYTE * KBYTES_PER_MBYTE
115 BYTES_PER_GBYTE = BYTES_PER_MBYTE * MBYTES_PER_GBYTE
116
117 SECONDS_PER_MINUTE = 60.0
118 MINUTES_PER_HOUR = 60.0
119 HOURS_PER_DAY = 24.0
120 SECONDS_PER_DAY = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY
121
122 UNIT_BYTES = 0
123 UNIT_KBYTES = 1
124 UNIT_MBYTES = 2
125 UNIT_GBYTES = 4
126 UNIT_SECTORS = 3
127
128 MTAB_FILE = "/etc/mtab"
129
130 MOUNT_COMMAND = [ "mount", ]
131 UMOUNT_COMMAND = [ "umount", ]
132
133 DEFAULT_LANGUAGE = "C"
134 LANG_VAR = "LANG"
135 LOCALE_VARS = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE",
136 "LC_CTYPE", "LC_IDENTIFICATION",
137 "LC_MEASUREMENT", "LC_MESSAGES",
138 "LC_MONETARY", "LC_NAME", "LC_NUMERIC",
139 "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ]
147
148 """
149 Class representing an "unordered list".
150
151 An "unordered list" is a list in which only the contents matter, not the
152 order in which the contents appear in the list.
153
154 For instance, we might be keeping track of set of paths in a list, because
155 it's convenient to have them in that form. However, for comparison
156 purposes, we would only care that the lists contain exactly the same
157 contents, regardless of order.
158
159 I have come up with two reasonable ways of doing this, plus a couple more
160 that would work but would be a pain to implement. My first method is to
161 copy and sort each list, comparing the sorted versions. This will only work
162 if two lists with exactly the same members are guaranteed to sort in exactly
163 the same order. The second way would be to create two Sets and then compare
164 the sets. However, this would lose information about any duplicates in
165 either list. I've decided to go with option #1 for now. I'll modify this
166 code if I run into problems in the future.
167
168 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__},
169 C{__le__} and C{__lt__} list methods to change the definition of the various
170 comparison operators. In all cases, the comparison is changed to return the
171 result of the original operation I{but instead comparing sorted lists}.
172 This is going to be quite a bit slower than a normal list, so you probably
173 only want to use it on small lists.
174 """
175
177 """
178 Definition of C{==} operator for this class.
179 @param other: Other object to compare to.
180 @return: True/false depending on whether C{self == other}.
181 """
182 if other is None:
183 return False
184 selfSorted = UnorderedList.mixedsort(self[:])
185 otherSorted = UnorderedList.mixedsort(other[:])
186 return selfSorted.__eq__(otherSorted)
187
189 """
190 Definition of C{!=} operator for this class.
191 @param other: Other object to compare to.
192 @return: True/false depending on whether C{self != other}.
193 """
194 if other is None:
195 return True
196 selfSorted = UnorderedList.mixedsort(self[:])
197 otherSorted = UnorderedList.mixedsort(other[:])
198 return selfSorted.__ne__(otherSorted)
199
201 """
202 Definition of S{>=} operator for this class.
203 @param other: Other object to compare to.
204 @return: True/false depending on whether C{self >= other}.
205 """
206 if other is None:
207 return True
208 selfSorted = UnorderedList.mixedsort(self[:])
209 otherSorted = UnorderedList.mixedsort(other[:])
210 return selfSorted.__ge__(otherSorted)
211
213 """
214 Definition of C{>} operator for this class.
215 @param other: Other object to compare to.
216 @return: True/false depending on whether C{self > other}.
217 """
218 if other is None:
219 return True
220 selfSorted = UnorderedList.mixedsort(self[:])
221 otherSorted = UnorderedList.mixedsort(other[:])
222 return selfSorted.__gt__(otherSorted)
223
225 """
226 Definition of S{<=} operator for this class.
227 @param other: Other object to compare to.
228 @return: True/false depending on whether C{self <= other}.
229 """
230 if other is None:
231 return False
232 selfSorted = UnorderedList.mixedsort(self[:])
233 otherSorted = UnorderedList.mixedsort(other[:])
234 return selfSorted.__le__(otherSorted)
235
237 """
238 Definition of C{<} operator for this class.
239 @param other: Other object to compare to.
240 @return: True/false depending on whether C{self < other}.
241 """
242 if other is None:
243 return False
244 selfSorted = UnorderedList.mixedsort(self[:])
245 otherSorted = UnorderedList.mixedsort(other[:])
246 return selfSorted.__lt__(otherSorted)
247
248 @staticmethod
250 """
251 Sort a list, making sure we don't blow up if the list happens to include mixed values.
252 @see: http://stackoverflow.com/questions/26575183/how-can-i-get-2-x-like-sorting-behaviour-in-python-3-x
253 """
254 return sorted(value, key=UnorderedList.mixedkey)
255
256 @staticmethod
258 """Provide a key for use by mixedsort()"""
259 numeric = Real, Decimal
260 if isinstance(value, numeric):
261 typeinfo = numeric
262 else:
263 typeinfo = type(value)
264 try:
265 x = value < value
266 except TypeError:
267 value = repr(value)
268 return repr(typeinfo), value
269
276
277 """
278 Class representing a list of absolute paths.
279
280 This is an unordered list.
281
282 We override the C{append}, C{insert} and C{extend} methods to ensure that
283 any item added to the list is an absolute path.
284
285 Each item added to the list is encoded using L{encodePath}. If we don't do
286 this, we have problems trying certain operations between strings and unicode
287 objects, particularly for "odd" filenames that can't be encoded in standard
288 ASCII.
289 """
290
292 """
293 Overrides the standard C{append} method.
294 @raise ValueError: If item is not an absolute path.
295 """
296 if not os.path.isabs(item):
297 raise ValueError("Not an absolute path: [%s]" % item)
298 list.append(self, encodePath(item))
299
300 - def insert(self, index, item):
301 """
302 Overrides the standard C{insert} method.
303 @raise ValueError: If item is not an absolute path.
304 """
305 if not os.path.isabs(item):
306 raise ValueError("Not an absolute path: [%s]" % item)
307 list.insert(self, index, encodePath(item))
308
310 """
311 Overrides the standard C{insert} method.
312 @raise ValueError: If any item is not an absolute path.
313 """
314 for item in seq:
315 if not os.path.isabs(item):
316 raise ValueError("Not an absolute path: [%s]" % item)
317 for item in seq:
318 list.append(self, encodePath(item))
319
326
327 """
328 Class representing a list containing only objects with a certain type.
329
330 This is an unordered list.
331
332 We override the C{append}, C{insert} and C{extend} methods to ensure that
333 any item added to the list matches the type that is requested. The
334 comparison uses the built-in C{isinstance}, which should allow subclasses of
335 of the requested type to be added to the list as well.
336
337 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a
338 CollectDir object."} if C{objectName} is C{"CollectDir"}.
339 """
340
341 - def __init__(self, objectType, objectName):
342 """
343 Initializes a typed list for a particular type.
344 @param objectType: Type that the list elements must match.
345 @param objectName: Short string containing the "name" of the type.
346 """
347 super(ObjectTypeList, self).__init__()
348 self.objectType = objectType
349 self.objectName = objectName
350
352 """
353 Overrides the standard C{append} method.
354 @raise ValueError: If item does not match requested type.
355 """
356 if not isinstance(item, self.objectType):
357 raise ValueError("Item must be a %s object." % self.objectName)
358 list.append(self, item)
359
360 - def insert(self, index, item):
361 """
362 Overrides the standard C{insert} method.
363 @raise ValueError: If item does not match requested type.
364 """
365 if not isinstance(item, self.objectType):
366 raise ValueError("Item must be a %s object." % self.objectName)
367 list.insert(self, index, item)
368
370 """
371 Overrides the standard C{insert} method.
372 @raise ValueError: If item does not match requested type.
373 """
374 for item in seq:
375 if not isinstance(item, self.objectType):
376 raise ValueError("All items must be %s objects." % self.objectName)
377 list.extend(self, seq)
378
379
380
381
382
383
384 -class RestrictedContentList(UnorderedList):
385
386 """
387 Class representing a list containing only object with certain values.
388
389 This is an unordered list.
390
391 We override the C{append}, C{insert} and C{extend} methods to ensure that
392 any item added to the list is among the valid values. We use a standard
393 comparison, so pretty much anything can be in the list of valid values.
394
395 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be
396 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}.
397
398 @note: This class doesn't make any attempt to trap for nonsensical
399 arguments. All of the values in the values list should be of the same type
400 (i.e. strings). Then, all list operations also need to be of that type
401 (i.e. you should always insert or append just strings). If you mix types --
402 for instance lists and strings -- you will likely see AttributeError
403 exceptions or other problems.
404 """
405
406 - def __init__(self, valuesList, valuesDescr, prefix=None):
407 """
408 Initializes a list restricted to containing certain values.
409 @param valuesList: List of valid values.
410 @param valuesDescr: Short string describing list of values.
411 @param prefix: Prefix to use in error messages (None results in prefix "Item")
412 """
413 super(RestrictedContentList, self).__init__()
414 self.prefix = "Item"
415 if prefix is not None: self.prefix = prefix
416 self.valuesList = valuesList
417 self.valuesDescr = valuesDescr
418
419 - def append(self, item):
420 """
421 Overrides the standard C{append} method.
422 @raise ValueError: If item is not in the values list.
423 """
424 if item not in self.valuesList:
425 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
426 list.append(self, item)
427
428 - def insert(self, index, item):
429 """
430 Overrides the standard C{insert} method.
431 @raise ValueError: If item is not in the values list.
432 """
433 if item not in self.valuesList:
434 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
435 list.insert(self, index, item)
436
437 - def extend(self, seq):
438 """
439 Overrides the standard C{insert} method.
440 @raise ValueError: If item is not in the values list.
441 """
442 for item in seq:
443 if item not in self.valuesList:
444 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
445 list.extend(self, seq)
446
453
454 """
455 Class representing a list containing only strings that match a regular expression.
456
457 If C{emptyAllowed} is passed in as C{False}, then empty strings are
458 explicitly disallowed, even if they happen to match the regular expression.
459 (C{None} values are always disallowed, since string operations are not
460 permitted on C{None}.)
461
462 This is an unordered list.
463
464 We override the C{append}, C{insert} and C{extend} methods to ensure that
465 any item added to the list matches the indicated regular expression.
466
467 @note: If you try to put values that are not strings into the list, you will
468 likely get either TypeError or AttributeError exceptions as a result.
469 """
470
471 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
472 """
473 Initializes a list restricted to containing certain values.
474 @param valuesRegex: Regular expression that must be matched, as a string
475 @param emptyAllowed: Indicates whether empty or None values are allowed.
476 @param prefix: Prefix to use in error messages (None results in prefix "Item")
477 """
478 super(RegexMatchList, self).__init__()
479 self.prefix = "Item"
480 if prefix is not None: self.prefix = prefix
481 self.valuesRegex = valuesRegex
482 self.emptyAllowed = emptyAllowed
483 self.pattern = re.compile(self.valuesRegex)
484
486 """
487 Overrides the standard C{append} method.
488 @raise ValueError: If item is None
489 @raise ValueError: If item is empty and empty values are not allowed
490 @raise ValueError: If item does not match the configured regular expression
491 """
492 if item is None or (not self.emptyAllowed and item == ""):
493 raise ValueError("%s cannot be empty." % self.prefix)
494 if not self.pattern.search(item):
495 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
496 list.append(self, item)
497
498 - def insert(self, index, item):
499 """
500 Overrides the standard C{insert} method.
501 @raise ValueError: If item is None
502 @raise ValueError: If item is empty and empty values are not allowed
503 @raise ValueError: If item does not match the configured regular expression
504 """
505 if item is None or (not self.emptyAllowed and item == ""):
506 raise ValueError("%s cannot be empty." % self.prefix)
507 if not self.pattern.search(item):
508 raise ValueError("%s is not valid [%s]" % (self.prefix, item))
509 list.insert(self, index, item)
510
512 """
513 Overrides the standard C{insert} method.
514 @raise ValueError: If any item is None
515 @raise ValueError: If any item is empty and empty values are not allowed
516 @raise ValueError: If any item does not match the configured regular expression
517 """
518 for item in seq:
519 if item is None or (not self.emptyAllowed and item == ""):
520 raise ValueError("%s cannot be empty." % self.prefix)
521 if not self.pattern.search(item):
522 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
523 list.extend(self, seq)
524
525
526
527
528
529
530 -class RegexList(UnorderedList):
531
532 """
533 Class representing a list of valid regular expression strings.
534
535 This is an unordered list.
536
537 We override the C{append}, C{insert} and C{extend} methods to ensure that
538 any item added to the list is a valid regular expression.
539 """
540
542 """
543 Overrides the standard C{append} method.
544 @raise ValueError: If item is not an absolute path.
545 """
546 try:
547 re.compile(item)
548 except re.error:
549 raise ValueError("Not a valid regular expression: [%s]" % item)
550 list.append(self, item)
551
552 - def insert(self, index, item):
553 """
554 Overrides the standard C{insert} method.
555 @raise ValueError: If item is not an absolute path.
556 """
557 try:
558 re.compile(item)
559 except re.error:
560 raise ValueError("Not a valid regular expression: [%s]" % item)
561 list.insert(self, index, item)
562
564 """
565 Overrides the standard C{insert} method.
566 @raise ValueError: If any item is not an absolute path.
567 """
568 for item in seq:
569 try:
570 re.compile(item)
571 except re.error:
572 raise ValueError("Not a valid regular expression: [%s]" % item)
573 for item in seq:
574 list.append(self, item)
575
576
577
578
579
580
581 -class _Vertex(object):
582
583 """
584 Represents a vertex (or node) in a directed graph.
585 """
586
588 """
589 Constructor.
590 @param name: Name of this graph vertex.
591 @type name: String value.
592 """
593 self.name = name
594 self.endpoints = []
595 self.state = None
596
599
600 """
601 Represents a directed graph.
602
603 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set
604 B{E} of vertex pairs or edges. In a directed graph, each edge also has an
605 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph}
606 object provides a way to construct a directed graph and execute a depth-
607 first search.
608
609 This data structure was designed based on the graphing chapter in
610 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>},
611 by Steven S. Skiena.
612
613 This class is intended to be used by Cedar Backup for dependency ordering.
614 Because of this, it's not quite general-purpose. Unlike a "general" graph,
615 every vertex in this graph has at least one edge pointing to it, from a
616 special "start" vertex. This is so no vertices get "lost" either because
617 they have no dependencies or because nothing depends on them.
618 """
619
620 _UNDISCOVERED = 0
621 _DISCOVERED = 1
622 _EXPLORED = 2
623
625 """
626 Directed graph constructor.
627
628 @param name: Name of this graph.
629 @type name: String value.
630 """
631 if name is None or name == "":
632 raise ValueError("Graph name must be non-empty.")
633 self._name = name
634 self._vertices = {}
635 self._startVertex = _Vertex(None)
636
638 """
639 Official string representation for class instance.
640 """
641 return "DirectedGraph(%s)" % self.name
642
644 """
645 Informal string representation for class instance.
646 """
647 return self.__repr__()
648
650 """Equals operator, implemented in terms of original Python 2 compare operator."""
651 return self.__cmp__(other) == 0
652
654 """Less-than operator, implemented in terms of original Python 2 compare operator."""
655 return self.__cmp__(other) < 0
656
658 """Greater-than operator, implemented in terms of original Python 2 compare operator."""
659 return self.__cmp__(other) > 0
660
662 """
663 Original Python 2 comparison operator.
664 @param other: Other object to compare to.
665 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
666 """
667
668 if other is None:
669 return 1
670 if self.name != other.name:
671 if str(self.name or "") < str(other.name or ""):
672 return -1
673 else:
674 return 1
675 if self._vertices != other._vertices:
676 if self._vertices < other._vertices:
677 return -1
678 else:
679 return 1
680 return 0
681
683 """
684 Property target used to get the graph name.
685 """
686 return self._name
687
688 name = property(_getName, None, None, "Name of the graph.")
689
691 """
692 Creates a named vertex.
693 @param name: vertex name
694 @raise ValueError: If the vertex name is C{None} or empty.
695 """
696 if name is None or name == "":
697 raise ValueError("Vertex name must be non-empty.")
698 vertex = _Vertex(name)
699 self._startVertex.endpoints.append(vertex)
700 self._vertices[name] = vertex
701
703 """
704 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex.
705 @param start: Name of start vertex.
706 @param finish: Name of finish vertex.
707 @raise ValueError: If one of the named vertices is unknown.
708 """
709 try:
710 startVertex = self._vertices[start]
711 finishVertex = self._vertices[finish]
712 startVertex.endpoints.append(finishVertex)
713 except KeyError as e:
714 raise ValueError("Vertex [%s] could not be found." % e)
715
717 """
718 Implements a topological sort of the graph.
719
720 This method also enforces that the graph is a directed acyclic graph,
721 which is a requirement of a topological sort.
722
723 A directed acyclic graph (or "DAG") is a directed graph with no directed
724 cycles. A topological sort of a DAG is an ordering on the vertices such
725 that all edges go from left to right. Only an acyclic graph can have a
726 topological sort, but any DAG has at least one topological sort.
727
728 Since a topological sort only makes sense for an acyclic graph, this
729 method throws an exception if a cycle is found.
730
731 A depth-first search only makes sense if the graph is acyclic. If the
732 graph contains any cycles, it is not possible to determine a consistent
733 ordering for the vertices.
734
735 @note: If a particular vertex has no edges, then its position in the
736 final list depends on the order in which the vertices were created in the
737 graph. If you're using this method to determine a dependency order, this
738 makes sense: a vertex with no dependencies can go anywhere (and will).
739
740 @return: Ordering on the vertices so that all edges go from left to right.
741
742 @raise ValueError: If a cycle is found in the graph.
743 """
744 ordering = []
745 for key in self._vertices:
746 vertex = self._vertices[key]
747 vertex.state = self._UNDISCOVERED
748 for key in self._vertices:
749 vertex = self._vertices[key]
750 if vertex.state == self._UNDISCOVERED:
751 self._topologicalSort(self._startVertex, ordering)
752 return ordering
753
755 """
756 Recursive depth first search function implementing topological sort.
757 @param vertex: Vertex to search
758 @param ordering: List of vertices in proper order
759 """
760 vertex.state = self._DISCOVERED
761 for endpoint in vertex.endpoints:
762 if endpoint.state == self._UNDISCOVERED:
763 self._topologicalSort(endpoint, ordering)
764 elif endpoint.state != self._EXPLORED:
765 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name))
766 if vertex.name is not None:
767 ordering.insert(0, vertex.name)
768 vertex.state = self._EXPLORED
769
776
777 """
778 Singleton used for resolving executable paths.
779
780 Various functions throughout Cedar Backup (including extensions) need a way
781 to resolve the path of executables that they use. For instance, the image
782 functionality needs to find the C{mkisofs} executable, and the Subversion
783 extension needs to find the C{svnlook} executable. Cedar Backup's original
784 behavior was to assume that the simple name (C{"svnlook"} or whatever) was
785 available on the caller's C{$PATH}, and to fail otherwise. However, this
786 turns out to be less than ideal, since for instance the root user might not
787 always have executables like C{svnlook} in its path.
788
789 One solution is to specify a path (either via an absolute path or some sort
790 of path insertion or path appending mechanism) that would apply to the
791 C{executeCommand()} function. This is not difficult to implement, but it
792 seem like kind of a "big hammer" solution. Besides that, it might also
793 represent a security flaw (for instance, I prefer not to mess with root's
794 C{$PATH} on the application level if I don't have to).
795
796 The alternative is to set up some sort of configuration for the path to
797 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or
798 whatever. This PathResolverSingleton aims to provide a good solution to the
799 mapping problem. Callers of all sorts (extensions or not) can get an
800 instance of the singleton. Then, they call the C{lookup} method to try and
801 resolve the executable they are looking for. Through the C{lookup} method,
802 the caller can also specify a default to use if a mapping is not found.
803 This way, with no real effort on the part of the caller, behavior can neatly
804 degrade to something equivalent to the current behavior if there is no
805 special mapping or if the singleton was never initialized in the first
806 place.
807
808 Even better, extensions automagically get access to the same resolver
809 functionality, and they don't even need to understand how the mapping
810 happens. All extension authors need to do is document what executables
811 their code requires, and the standard resolver configuration section will
812 meet their needs.
813
814 The class should be initialized once through the constructor somewhere in
815 the main routine. Then, the main routine should call the L{fill} method to
816 fill in the resolver's internal structures. Everyone else who needs to
817 resolve a path will get an instance of the class using L{getInstance} and
818 will then just call the L{lookup} method.
819
820 @cvar _instance: Holds a reference to the singleton
821 @ivar _mapping: Internal mapping from resource name to path.
822 """
823
824 _instance = None
825
827 """Helper class to provide a singleton factory method."""
836
837 getInstance = _Helper()
838
843
844 - def lookup(self, name, default=None):
845 """
846 Looks up name and returns the resolved path associated with the name.
847 @param name: Name of the path resource to resolve.
848 @param default: Default to return if resource cannot be resolved.
849 @return: Resolved path associated with name, or default if name can't be resolved.
850 """
851 value = default
852 if name in list(self._mapping.keys()):
853 value = self._mapping[name]
854 logger.debug("Resolved command [%s] to [%s].", name, value)
855 return value
856
857 - def fill(self, mapping):
858 """
859 Fills in the singleton's internal mapping from name to resource.
860 @param mapping: Mapping from resource name to path.
861 @type mapping: Dictionary mapping name to path, both as strings.
862 """
863 self._mapping = { }
864 for key in list(mapping.keys()):
865 self._mapping[key] = mapping[key]
866
867
868
869
870
871
872 -class Pipe(Popen):
873 """
874 Specialized pipe class for use by C{executeCommand}.
875
876 The L{executeCommand} function needs a specialized way of interacting
877 with a pipe. First, C{executeCommand} only reads from the pipe, and
878 never writes to it. Second, C{executeCommand} needs a way to discard all
879 output written to C{stderr}, as a means of simulating the shell
880 C{2>/dev/null} construct.
881 """
882 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
883 stderr = STDOUT
884 if ignoreStderr:
885 devnull = nullDevice()
886 stderr = os.open(devnull, os.O_RDWR)
887 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
888
895
896 """
897 Class holding runtime diagnostic information.
898
899 Diagnostic information is information that is useful to get from users for
900 debugging purposes. I'm consolidating it all here into one object.
901
902 @sort: __init__, __repr__, __str__
903 """
904
905
907 """
908 Constructor for the C{Diagnostics} class.
909 """
910
912 """
913 Official string representation for class instance.
914 """
915 return "Diagnostics()"
916
918 """
919 Informal string representation for class instance.
920 """
921 return self.__repr__()
922
924 """
925 Get a map containing all of the diagnostic values.
926 @return: Map from diagnostic name to diagnostic value.
927 """
928 values = {}
929 values['version'] = self.version
930 values['interpreter'] = self.interpreter
931 values['platform'] = self.platform
932 values['encoding'] = self.encoding
933 values['locale'] = self.locale
934 values['timestamp'] = self.timestamp
935 return values
936
938 """
939 Pretty-print diagnostic information to a file descriptor.
940 @param fd: File descriptor used to print information.
941 @param prefix: Prefix string (if any) to place onto printed lines
942 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
943 """
944 lines = self._buildDiagnosticLines(prefix)
945 for line in lines:
946 fd.write("%s\n" % line)
947
949 """
950 Pretty-print diagnostic information using a logger method.
951 @param method: Logger method to use for logging (i.e. logger.info)
952 @param prefix: Prefix string (if any) to place onto printed lines
953 """
954 lines = self._buildDiagnosticLines(prefix)
955 for line in lines:
956 method("%s" % line)
957
959 """
960 Build a set of pretty-printed diagnostic lines.
961 @param prefix: Prefix string (if any) to place onto printed lines
962 @return: List of strings, not terminated by newlines.
963 """
964 values = self.getValues()
965 keys = list(values.keys())
966 keys.sort()
967 tmax = Diagnostics._getMaxLength(keys) + 3
968 lines = []
969 for key in keys:
970 title = key.title()
971 title += (tmax - len(title)) * '.'
972 value = values[key]
973 line = "%s%s: %s" % (prefix, title, value)
974 lines.append(line)
975 return lines
976
977 @staticmethod
979 """
980 Get the maximum length from among a list of strings.
981 """
982 tmax = 0
983 for value in values:
984 if len(value) > tmax:
985 tmax = len(value)
986 return tmax
987
989 """
990 Property target to get the Cedar Backup version.
991 """
992 return "Cedar Backup %s (%s)" % (VERSION, DATE)
993
995 """
996 Property target to get the Python interpreter version.
997 """
998 version = sys.version_info
999 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
1000
1002 """
1003 Property target to get the filesystem encoding.
1004 """
1005 return sys.getfilesystemencoding() or sys.getdefaultencoding()
1006
1019
1021 """
1022 Property target to get the default locale that is in effect.
1023 """
1024 try:
1025 import locale
1026 return locale.getdefaultlocale()[0]
1027 except:
1028 return "(unknown)"
1029
1031 """
1032 Property target to get a current date/time stamp.
1033 """
1034 try:
1035 import datetime
1036 return datetime.datetime.utcnow().ctime() + " UTC"
1037 except:
1038 return "(unknown)"
1039
1040 version = property(_getVersion, None, None, "Cedar Backup version.")
1041 interpreter = property(_getInterpreter, None, None, "Python interpreter version.")
1042 platform = property(_getPlatform, None, None, "Platform identifying information.")
1043 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.")
1044 locale = property(_getLocale, None, None, "Locale that is in effect.")
1045 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056 -def sortDict(d):
1057 """
1058 Returns the keys of the dictionary sorted by value.
1059 @param d: Dictionary to operate on
1060 @return: List of dictionary keys sorted in order by dictionary value.
1061 """
1062 items = list(d.items())
1063 items.sort(key=lambda x: (x[1], x[0]))
1064 return [key for key, value in items]
1065
1066
1067
1068
1069
1070
1071 -def removeKeys(d, keys):
1072 """
1073 Removes all of the keys from the dictionary.
1074 The dictionary is altered in-place.
1075 Each key must exist in the dictionary.
1076 @param d: Dictionary to operate on
1077 @param keys: List of keys to remove
1078 @raise KeyError: If one of the keys does not exist
1079 """
1080 for key in keys:
1081 del d[key]
1082
1083
1084
1085
1086
1087
1088 -def convertSize(size, fromUnit, toUnit):
1089 """
1090 Converts a size in one unit to a size in another unit.
1091
1092 This is just a convenience function so that the functionality can be
1093 implemented in just one place. Internally, we convert values to bytes and
1094 then to the final unit.
1095
1096 The available units are:
1097
1098 - C{UNIT_BYTES} - Bytes
1099 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B
1100 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB
1101 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB
1102 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B
1103
1104 @param size: Size to convert
1105 @type size: Integer or float value in units of C{fromUnit}
1106
1107 @param fromUnit: Unit to convert from
1108 @type fromUnit: One of the units listed above
1109
1110 @param toUnit: Unit to convert to
1111 @type toUnit: One of the units listed above
1112
1113 @return: Number converted to new unit, as a float.
1114 @raise ValueError: If one of the units is invalid.
1115 """
1116 if size is None:
1117 raise ValueError("Cannot convert size of None.")
1118 if fromUnit == UNIT_BYTES:
1119 byteSize = float(size)
1120 elif fromUnit == UNIT_KBYTES:
1121 byteSize = float(size) * BYTES_PER_KBYTE
1122 elif fromUnit == UNIT_MBYTES:
1123 byteSize = float(size) * BYTES_PER_MBYTE
1124 elif fromUnit == UNIT_GBYTES:
1125 byteSize = float(size) * BYTES_PER_GBYTE
1126 elif fromUnit == UNIT_SECTORS:
1127 byteSize = float(size) * BYTES_PER_SECTOR
1128 else:
1129 raise ValueError("Unknown 'from' unit %s." % fromUnit)
1130 if toUnit == UNIT_BYTES:
1131 return byteSize
1132 elif toUnit == UNIT_KBYTES:
1133 return byteSize / BYTES_PER_KBYTE
1134 elif toUnit == UNIT_MBYTES:
1135 return byteSize / BYTES_PER_MBYTE
1136 elif toUnit == UNIT_GBYTES:
1137 return byteSize / BYTES_PER_GBYTE
1138 elif toUnit == UNIT_SECTORS:
1139 return byteSize / BYTES_PER_SECTOR
1140 else:
1141 raise ValueError("Unknown 'to' unit %s." % toUnit)
1142
1143
1144
1145
1146
1147
1148 -def displayBytes(bytes, digits=2):
1149 """
1150 Format a byte quantity so it can be sensibly displayed.
1151
1152 It's rather difficult to look at a number like "72372224 bytes" and get any
1153 meaningful information out of it. It would be more useful to see something
1154 like "69.02 MB". That's what this function does. Any time you want to display
1155 a byte value, i.e.::
1156
1157 print "Size: %s bytes" % bytes
1158
1159 Call this function instead::
1160
1161 print "Size: %s" % displayBytes(bytes)
1162
1163 What comes out will be sensibly formatted. The indicated number of digits
1164 will be listed after the decimal point, rounded based on whatever rules are
1165 used by Python's standard C{%f} string format specifier. (Values less than 1
1166 kB will be listed in bytes and will not have a decimal point, since the
1167 concept of a fractional byte is nonsensical.)
1168
1169 @param bytes: Byte quantity.
1170 @type bytes: Integer number of bytes.
1171
1172 @param digits: Number of digits to display after the decimal point.
1173 @type digits: Integer value, typically 2-5.
1174
1175 @return: String, formatted for sensible display.
1176 """
1177 if bytes is None:
1178 raise ValueError("Cannot display byte value of None.")
1179 bytes = float(bytes)
1180 if math.fabs(bytes) < BYTES_PER_KBYTE:
1181 fmt = "%.0f bytes"
1182 value = bytes
1183 elif math.fabs(bytes) < BYTES_PER_MBYTE:
1184 fmt = "%." + "%d" % digits + "f kB"
1185 value = bytes / BYTES_PER_KBYTE
1186 elif math.fabs(bytes) < BYTES_PER_GBYTE:
1187 fmt = "%." + "%d" % digits + "f MB"
1188 value = bytes / BYTES_PER_MBYTE
1189 else:
1190 fmt = "%." + "%d" % digits + "f GB"
1191 value = bytes / BYTES_PER_GBYTE
1192 return fmt % value
1193
1200 """
1201 Gets a reference to a named function.
1202
1203 This does some hokey-pokey to get back a reference to a dynamically named
1204 function. For instance, say you wanted to get a reference to the
1205 C{os.path.isdir} function. You could use::
1206
1207 myfunc = getFunctionReference("os.path", "isdir")
1208
1209 Although we won't bomb out directly, behavior is pretty much undefined if
1210 you pass in C{None} or C{""} for either C{module} or C{function}.
1211
1212 The only validation we enforce is that whatever we get back must be
1213 callable.
1214
1215 I derived this code based on the internals of the Python unittest
1216 implementation. I don't claim to completely understand how it works.
1217
1218 @param module: Name of module associated with function.
1219 @type module: Something like "os.path" or "CedarBackup3.util"
1220
1221 @param function: Name of function
1222 @type function: Something like "isdir" or "getUidGid"
1223
1224 @return: Reference to function associated with name.
1225
1226 @raise ImportError: If the function cannot be found.
1227 @raise ValueError: If the resulting reference is not callable.
1228
1229 @copyright: Some of this code, prior to customization, was originally part
1230 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
1231 Software Foundation; All Rights Reserved.
1232 """
1233 parts = []
1234 if module is not None and module != "":
1235 parts = module.split(".")
1236 if function is not None and function != "":
1237 parts.append(function)
1238 copy = parts[:]
1239 while copy:
1240 try:
1241 module = __import__(".".join(copy))
1242 break
1243 except ImportError:
1244 del copy[-1]
1245 if not copy: raise
1246 parts = parts[1:]
1247 obj = module
1248 for part in parts:
1249 obj = getattr(obj, part)
1250 if not isinstance(obj, collections.Callable):
1251 raise ValueError("Reference to %s.%s is not callable." % (module, function))
1252 return obj
1253
1254
1255
1256
1257
1258
1259 -def getUidGid(user, group):
1260 """
1261 Get the uid/gid associated with a user/group pair
1262
1263 This is a no-op if user/group functionality is not available on the platform.
1264
1265 @param user: User name
1266 @type user: User name as a string
1267
1268 @param group: Group name
1269 @type group: Group name as a string
1270
1271 @return: Tuple C{(uid, gid)} matching passed-in user and group.
1272 @raise ValueError: If the ownership user/group values are invalid
1273 """
1274 if _UID_GID_AVAILABLE:
1275 try:
1276 uid = pwd.getpwnam(user)[2]
1277 gid = grp.getgrnam(group)[2]
1278 return (uid, gid)
1279 except Exception as e:
1280 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e)
1281 raise ValueError("Unable to lookup up uid and gid for passed in user/group.")
1282 else:
1283 return (0, 0)
1284
1291 """
1292 Changes ownership of path to match the user and group.
1293
1294 This is a no-op if user/group functionality is not available on the
1295 platform, or if the either passed-in user or group is C{None}. Further, we
1296 won't even try to do it unless running as root, since it's unlikely to work.
1297
1298 @param path: Path whose ownership to change.
1299 @param user: User which owns file.
1300 @param group: Group which owns file.
1301 """
1302 if _UID_GID_AVAILABLE:
1303 if user is None or group is None:
1304 logger.debug("User or group is None, so not attempting to change owner on [%s].", path)
1305 elif not isRunningAsRoot():
1306 logger.debug("Not root, so not attempting to change owner on [%s].", path)
1307 else:
1308 try:
1309 (uid, gid) = getUidGid(user, group)
1310 os.chown(path, uid, gid)
1311 except Exception as e:
1312 logger.error("Error changing ownership of [%s]: %s", path, e)
1313
1320 """
1321 Indicates whether the program is running as the root user.
1322 """
1323 return os.getuid() == 0
1324
1331 """
1332 Splits a command line string into a list of arguments.
1333
1334 Unfortunately, there is no "standard" way to parse a command line string,
1335 and it's actually not an easy problem to solve portably (essentially, we
1336 have to emulate the shell argument-processing logic). This code only
1337 respects double quotes (C{"}) for grouping arguments, not single quotes
1338 (C{'}). Make sure you take this into account when building your command
1339 line.
1340
1341 Incidentally, I found this particular parsing method while digging around in
1342 Google Groups, and I tweaked it for my own use.
1343
1344 @param commandLine: Command line string
1345 @type commandLine: String, i.e. "cback3 --verbose stage store"
1346
1347 @return: List of arguments, suitable for passing to C{popen2}.
1348
1349 @raise ValueError: If the command line is None.
1350 """
1351 if commandLine is None:
1352 raise ValueError("Cannot split command line of None.")
1353 fields = re.findall('[^ "]+|"[^"]+"', commandLine)
1354 fields = [field.replace('"', '') for field in fields]
1355 return fields
1356
1363 """
1364 Resolves the real path to a command through the path resolver mechanism.
1365
1366 Both extensions and standard Cedar Backup functionality need a way to
1367 resolve the "real" location of various executables. Normally, they assume
1368 that these executables are on the system path, but some callers need to
1369 specify an alternate location.
1370
1371 Ideally, we want to handle this configuration in a central location. The
1372 Cedar Backup path resolver mechanism (a singleton called
1373 L{PathResolverSingleton}) provides the central location to store the
1374 mappings. This function wraps access to the singleton, and is what all
1375 functions (extensions or standard functionality) should call if they need to
1376 find a command.
1377
1378 The passed-in command must actually be a list, in the standard form used by
1379 all existing Cedar Backup code (something like C{["svnlook", ]}). The
1380 lookup will actually be done on the first element in the list, and the
1381 returned command will always be in list form as well.
1382
1383 If the passed-in command can't be resolved or no mapping exists, then the
1384 command itself will be returned unchanged. This way, we neatly fall back on
1385 default behavior if we have no sensible alternative.
1386
1387 @param command: Command to resolve.
1388 @type command: List form of command, i.e. C{["svnlook", ]}.
1389
1390 @return: Path to command or just command itself if no mapping exists.
1391 """
1392 singleton = PathResolverSingleton.getInstance()
1393 name = command[0]
1394 result = command[:]
1395 result[0] = singleton.lookup(name, name)
1396 return result
1397
1398
1399
1400
1401
1402
1403 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1404 """
1405 Executes a shell command, hopefully in a safe way.
1406
1407 This function exists to replace direct calls to C{os.popen} in the Cedar
1408 Backup code. It's not safe to call a function such as C{os.popen()} with
1409 untrusted arguments, since that can cause problems if the string contains
1410 non-safe variables or other constructs (imagine that the argument is
1411 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/;
1412 echo"} in the current environment).
1413
1414 Instead, it's safer to pass a list of arguments in the style supported bt
1415 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe}
1416 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}.
1417
1418 Under the normal case, this function will return a tuple of C{(status,
1419 None)} where the status is the wait-encoded return status of the call per
1420 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as
1421 C{True}, the function will return a tuple of C{(status, output)} where
1422 C{output} is a list of strings, one entry per line in the output from the
1423 command. Output is always logged to the C{outputLogger.info()} target,
1424 regardless of whether it's returned.
1425
1426 By default, C{stdout} and C{stderr} will be intermingled in the output.
1427 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be
1428 included in the output.
1429
1430 The C{doNotLog} parameter exists so that callers can force the function to
1431 not log command output to the debug log. Normally, you would want to log.
1432 However, if you're using this function to write huge output files (i.e.
1433 database backups written to C{stdout}) then you might want to avoid putting
1434 all that information into the debug log.
1435
1436 The C{outputFile} parameter exists to make it easier for a caller to push
1437 output into a file, i.e. as a substitute for redirection to a file. If this
1438 value is passed in, each time a line of output is generated, it will be
1439 written to the file using C{outputFile.write()}. At the end, the file
1440 descriptor will be flushed using C{outputFile.flush()}. The caller
1441 maintains responsibility for closing the file object appropriately.
1442
1443 @note: I know that it's a bit confusing that the command and the arguments
1444 are both lists. I could have just required the caller to pass in one big
1445 list. However, I think it makes some sense to keep the command (the
1446 constant part of what we're executing, i.e. C{"scp -B"}) separate from its
1447 arguments, even if they both end up looking kind of similar.
1448
1449 @note: You cannot redirect output via shell constructs (i.e. C{>file},
1450 C{2>/dev/null}, etc.) using this function. The redirection string would be
1451 passed to the command just like any other argument. However, you can
1452 implement the equivalent to redirection using C{ignoreStderr} and
1453 C{outputFile}, as discussed above.
1454
1455 @note: The operating system environment is partially sanitized before
1456 the command is invoked. See L{sanitizeEnvironment} for details.
1457
1458 @param command: Shell command to execute
1459 @type command: List of individual arguments that make up the command
1460
1461 @param args: List of arguments to the command
1462 @type args: List of additional arguments to the command
1463
1464 @param returnOutput: Indicates whether to return the output of the command
1465 @type returnOutput: Boolean C{True} or C{False}
1466
1467 @param ignoreStderr: Whether stderr should be discarded
1468 @type ignoreStderr: Boolean True or False
1469
1470 @param doNotLog: Indicates that output should not be logged.
1471 @type doNotLog: Boolean C{True} or C{False}
1472
1473 @param outputFile: File object that all output should be written to.
1474 @type outputFile: File object as returned from C{open()} or C{file()}, configured for binary write
1475
1476 @return: Tuple of C{(result, output)} as described above.
1477 """
1478 logger.debug("Executing command %s with args %s.", command, args)
1479 outputLogger.info("Executing command %s with args %s.", command, args)
1480 if doNotLog:
1481 logger.debug("Note: output will not be logged, per the doNotLog flag.")
1482 outputLogger.info("Note: output will not be logged, per the doNotLog flag.")
1483 output = []
1484 fields = command[:]
1485 fields.extend(args)
1486 try:
1487 sanitizeEnvironment()
1488 try:
1489 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1490 except OSError:
1491
1492
1493
1494 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1495 while True:
1496 line = pipe.stdout.readline()
1497 if not line: break
1498 if returnOutput: output.append(line.decode("utf-8"))
1499 if outputFile is not None: outputFile.write(line)
1500 if not doNotLog: outputLogger.info(line.decode("utf-8")[:-1])
1501 if outputFile is not None:
1502 try:
1503 outputFile.flush()
1504 except: pass
1505 if returnOutput:
1506 return (pipe.wait(), output)
1507 else:
1508 return (pipe.wait(), None)
1509 except OSError as e:
1510 try:
1511 if returnOutput:
1512 if output != []:
1513 return (pipe.wait(), output)
1514 else:
1515 return (pipe.wait(), [ e, ])
1516 else:
1517 return (pipe.wait(), None)
1518 except UnboundLocalError:
1519 if returnOutput:
1520 return (256, [])
1521 else:
1522 return (256, None)
1523
1530 """
1531 Calculates the age (in days) of a file.
1532
1533 The "age" of a file is the amount of time since the file was last used, per
1534 the most recent of the file's C{st_atime} and C{st_mtime} values.
1535
1536 Technically, we only intend this function to work with files, but it will
1537 probably work with anything on the filesystem.
1538
1539 @param path: Path to a file on disk.
1540
1541 @return: Age of the file in days (possibly fractional).
1542 @raise OSError: If the file doesn't exist.
1543 """
1544 currentTime = int(time.time())
1545 fileStats = os.stat(path)
1546 lastUse = max(fileStats.st_atime, fileStats.st_mtime)
1547 ageInSeconds = currentTime - lastUse
1548 ageInDays = ageInSeconds / SECONDS_PER_DAY
1549 return ageInDays
1550
1551
1552
1553
1554
1555
1556 -def mount(devicePath, mountPoint, fsType):
1557 """
1558 Mounts the indicated device at the indicated mount point.
1559
1560 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount
1561 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any
1562 filesystem type that is supported by C{mount} on your platform. If the type
1563 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may
1564 not work on all systems.
1565
1566 @note: This only works on platforms that have a concept of "mounting" a
1567 filesystem through a command-line C{"mount"} command, like UNIXes. It
1568 won't work on Windows.
1569
1570 @param devicePath: Path of device to be mounted.
1571 @param mountPoint: Path that device should be mounted at.
1572 @param fsType: Type of the filesystem assumed to be available via the device.
1573
1574 @raise IOError: If the device cannot be mounted.
1575 """
1576 if fsType is None:
1577 args = [ devicePath, mountPoint ]
1578 else:
1579 args = [ "-t", fsType, devicePath, mountPoint ]
1580 command = resolveCommand(MOUNT_COMMAND)
1581 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0]
1582 if result != 0:
1583 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1584
1585
1586
1587
1588
1589
1590 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1591 """
1592 Unmounts whatever device is mounted at the indicated mount point.
1593
1594 Sometimes, it might not be possible to unmount the mount point immediately,
1595 if there are still files open there. Use the C{attempts} and C{waitSeconds}
1596 arguments to indicate how many unmount attempts to make and how many seconds
1597 to wait between attempts. If you pass in zero attempts, no attempts will be
1598 made (duh).
1599
1600 If the indicated mount point is not really a mount point per
1601 C{os.path.ismount()}, then it will be ignored. This seems to be a safer
1602 check then looking through C{/etc/mtab}, since C{ismount()} is already in
1603 the Python standard library and is documented as working on all POSIX
1604 systems.
1605
1606 If C{removeAfter} is C{True}, then the mount point will be removed using
1607 C{os.rmdir()} after the unmount action succeeds. If for some reason the
1608 mount point is not a directory, then it will not be removed.
1609
1610 @note: This only works on platforms that have a concept of "mounting" a
1611 filesystem through a command-line C{"mount"} command, like UNIXes. It
1612 won't work on Windows.
1613
1614 @param mountPoint: Mount point to be unmounted.
1615 @param removeAfter: Remove the mount point after unmounting it.
1616 @param attempts: Number of times to attempt the unmount.
1617 @param waitSeconds: Number of seconds to wait between repeated attempts.
1618
1619 @raise IOError: If the mount point is still mounted after attempts are exhausted.
1620 """
1621 if os.path.ismount(mountPoint):
1622 for attempt in range(0, attempts):
1623 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint)
1624 command = resolveCommand(UMOUNT_COMMAND)
1625 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0]
1626 if result != 0:
1627 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt)
1628 elif os.path.ismount(mountPoint):
1629 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint)
1630 else:
1631 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt)
1632 break
1633 if attempt+1 < attempts:
1634 if waitSeconds > 0:
1635 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds)
1636 time.sleep(waitSeconds)
1637 else:
1638 if os.path.ismount(mountPoint):
1639 raise IOError("Unable to unmount [%s] after %d attempts.", mountPoint, attempts)
1640 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint)
1641 if os.path.isdir(mountPoint) and removeAfter:
1642 logger.debug("Removing mount point [%s].", mountPoint)
1643 os.rmdir(mountPoint)
1644
1651 """
1652 Indicates whether a specific filesystem device is currently mounted.
1653
1654 We determine whether the device is mounted by looking through the system's
1655 C{mtab} file. This file shows every currently-mounted filesystem, ordered
1656 by device. We only do the check if the C{mtab} file exists and is readable.
1657 Otherwise, we assume that the device is not mounted.
1658
1659 @note: This only works on platforms that have a concept of an mtab file
1660 to show mounted volumes, like UNIXes. It won't work on Windows.
1661
1662 @param devicePath: Path of device to be checked
1663
1664 @return: True if device is mounted, false otherwise.
1665 """
1666 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK):
1667 realPath = os.path.realpath(devicePath)
1668 with open(MTAB_FILE) as f:
1669 lines = f.readlines()
1670 for line in lines:
1671 (mountDevice, mountPoint, remainder) = line.split(None, 2)
1672 if mountDevice in [ devicePath, realPath, ]:
1673 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint)
1674 return True
1675 return False
1676
1683 """
1684 Safely encodes a filesystem path as a Unicode string, converting bytes to fileystem encoding if necessary.
1685 @param path: Path to encode
1686 @return: Path, as a string, encoded appropriately
1687 @raise ValueError: If the path cannot be encoded properly.
1688 @see: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/
1689 """
1690 if path is None:
1691 return path
1692 try:
1693 if isinstance(path, bytes):
1694 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
1695 path = path.decode(encoding, "surrogateescape")
1696 return path
1697 except UnicodeError as e:
1698 raise ValueError("Path could not be safely encoded as %s: %s" % (encoding, str(e)))
1699
1706 """
1707 Attempts to portably return the null device on this system.
1708
1709 The null device is something like C{/dev/null} on a UNIX system. The name
1710 varies on other platforms.
1711 """
1712 return os.devnull
1713
1720 """
1721 Converts English day name to numeric day of week as from C{time.localtime}.
1722
1723 For instance, the day C{monday} would be converted to the number C{0}.
1724
1725 @param dayName: Day of week to convert
1726 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1727
1728 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible.
1729 """
1730 if dayName.lower() == "monday":
1731 return 0
1732 elif dayName.lower() == "tuesday":
1733 return 1
1734 elif dayName.lower() == "wednesday":
1735 return 2
1736 elif dayName.lower() == "thursday":
1737 return 3
1738 elif dayName.lower() == "friday":
1739 return 4
1740 elif dayName.lower() == "saturday":
1741 return 5
1742 elif dayName.lower() == "sunday":
1743 return 6
1744 else:
1745 return -1
1746
1753 """
1754 Indicates whether "today" is the backup starting day per configuration.
1755
1756 If the current day's English name matches the indicated starting day, then
1757 today is a starting day.
1758
1759 @param startingDay: Configured starting day.
1760 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1761
1762 @return: Boolean indicating whether today is the starting day.
1763 """
1764 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay)
1765 if value:
1766 logger.debug("Today is the start of the week.")
1767 else:
1768 logger.debug("Today is NOT the start of the week.")
1769 return value
1770
1777 """
1778 Returns a "normalized" path based on a path name.
1779
1780 A normalized path is a representation of a path that is also a valid file
1781 name. To make a valid file name out of a complete path, we have to convert
1782 or remove some characters that are significant to the filesystem -- in
1783 particular, the path separator and any leading C{'.'} character (which would
1784 cause the file to be hidden in a file listing).
1785
1786 Note that this is a one-way transformation -- you can't safely derive the
1787 original path from the normalized path.
1788
1789 To normalize a path, we begin by looking at the first character. If the
1790 first character is C{'/'} or C{'\\'}, it gets removed. If the first
1791 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the
1792 rest of the path and convert all remaining C{'/'} or C{'\\'} characters
1793 C{'-'}, and all remaining whitespace characters to C{'_'}.
1794
1795 As a special case, a path consisting only of a single C{'/'} or C{'\\'}
1796 character will be converted to C{'-'}.
1797
1798 @param path: Path to normalize
1799
1800 @return: Normalized path as described above.
1801
1802 @raise ValueError: If the path is None
1803 """
1804 if path is None:
1805 raise ValueError("Cannot normalize path None.")
1806 elif len(path) == 0:
1807 return path
1808 elif path == "/" or path == "\\":
1809 return "-"
1810 else:
1811 normalized = path
1812 normalized = re.sub(r"^\/", "", normalized)
1813 normalized = re.sub(r"^\\", "", normalized)
1814 normalized = re.sub(r"^\.", "_", normalized)
1815 normalized = re.sub(r"\/", "-", normalized)
1816 normalized = re.sub(r"\\", "-", normalized)
1817 normalized = re.sub(r"\s", "_", normalized)
1818 return normalized
1819
1826 """
1827 Sanitizes the operating system environment.
1828
1829 The operating system environment is contained in C{os.environ}. This method
1830 sanitizes the contents of that dictionary.
1831
1832 Currently, all it does is reset the locale (removing C{$LC_*}) and set the
1833 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count
1834 on consistent localization regardless of what the end-user has configured.
1835 This is important for code that needs to parse program output.
1836
1837 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already
1838 set to the proper value, it is not re-set, so we can avoid the memory leaks
1839 that are documented to occur on BSD-based systems.
1840
1841 @return: Copy of the sanitized environment.
1842 """
1843 for var in LOCALE_VARS:
1844 if var in os.environ:
1845 del os.environ[var]
1846 if LANG_VAR in os.environ:
1847 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE:
1848 os.environ[LANG_VAR] = DEFAULT_LANGUAGE
1849 return os.environ.copy()
1850
1857 """
1858 Deference a soft link, optionally normalizing it to an absolute path.
1859 @param path: Path of link to dereference
1860 @param absolute: Whether to normalize the result to an absolute path
1861 @return: Dereferenced path, or original path if original is not a link.
1862 """
1863 if os.path.islink(path):
1864 result = os.readlink(path)
1865 if absolute and not os.path.isabs(result):
1866 result = os.path.abspath(os.path.join(os.path.dirname(path), result))
1867 return result
1868 return path
1869
1870
1871
1872
1873
1874
1875 -def checkUnique(prefix, values):
1876 """
1877 Checks that all values are unique.
1878
1879 The values list is checked for duplicate values. If there are
1880 duplicates, an exception is thrown. All duplicate values are listed in
1881 the exception.
1882
1883 @param prefix: Prefix to use in the thrown exception
1884 @param values: List of values to check
1885
1886 @raise ValueError: If there are duplicates in the list
1887 """
1888 values.sort()
1889 duplicates = []
1890 for i in range(1, len(values)):
1891 if values[i-1] == values[i]:
1892 duplicates.append(values[i])
1893 if duplicates:
1894 raise ValueError("%s %s" % (prefix, duplicates))
1895
1902 """
1903 Parses a list of values out of a comma-separated string.
1904
1905 The items in the list are split by comma, and then have whitespace
1906 stripped. As a special case, if C{commaString} is C{None}, then C{None}
1907 will be returned.
1908
1909 @param commaString: List of values in comma-separated string format.
1910 @return: Values from commaString split into a list, or C{None}.
1911 """
1912 if commaString is None:
1913 return None
1914 else:
1915 pass1 = commaString.split(",")
1916 pass2 = []
1917 for item in pass1:
1918 item = item.strip()
1919 if len(item) > 0:
1920 pass2.append(item)
1921 return pass2
1922