1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41 """
42 Provides general-purpose utilities.
43
44 @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList,
45 RegexList, _Vertex, DirectedGraph, PathResolverSingleton,
46 sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine,
47 resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice,
48 deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,
49 ISO_SECTOR_SIZE, BYTES_PER_SECTOR,
50 BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,
51 SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,
52 UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS
53
54 @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes.
55 @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector.
56 @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB).
57 @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB).
58 @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB).
59 @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB).
60 @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB).
61 @var SECONDS_PER_MINUTE: Number of seconds per minute.
62 @var MINUTES_PER_HOUR: Number of minutes per hour.
63 @var HOURS_PER_DAY: Number of hours per day.
64 @var SECONDS_PER_DAY: Number of seconds per day.
65 @var UNIT_BYTES: Constant representing the byte (B) unit for conversion.
66 @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion.
67 @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion.
68 @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion.
69 @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion.
70
71 @author: Kenneth J. Pronovici <pronovic@ieee.org>
72 """
73
74
75
76
77
78
79 import sys
80 import math
81 import os
82 import re
83 import time
84 import logging
85 from subprocess import Popen, STDOUT, PIPE
86 from functools import total_ordering
87 from numbers import Real
88 from decimal import Decimal
89 import collections
90
91 try:
92 import pwd
93 import grp
94 _UID_GID_AVAILABLE = True
95 except ImportError:
96 _UID_GID_AVAILABLE = False
97
98 from CedarBackup3.release import VERSION, DATE
99
100
101
102
103
104
105 logger = logging.getLogger("CedarBackup3.log.util")
106 outputLogger = logging.getLogger("CedarBackup3.output")
107
108 ISO_SECTOR_SIZE = 2048.0
109 BYTES_PER_SECTOR = ISO_SECTOR_SIZE
110
111 BYTES_PER_KBYTE = 1024.0
112 KBYTES_PER_MBYTE = 1024.0
113 MBYTES_PER_GBYTE = 1024.0
114 BYTES_PER_MBYTE = BYTES_PER_KBYTE * KBYTES_PER_MBYTE
115 BYTES_PER_GBYTE = BYTES_PER_MBYTE * MBYTES_PER_GBYTE
116
117 SECONDS_PER_MINUTE = 60.0
118 MINUTES_PER_HOUR = 60.0
119 HOURS_PER_DAY = 24.0
120 SECONDS_PER_DAY = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY
121
122 UNIT_BYTES = 0
123 UNIT_KBYTES = 1
124 UNIT_MBYTES = 2
125 UNIT_GBYTES = 4
126 UNIT_SECTORS = 3
127
128 MTAB_FILE = "/etc/mtab"
129
130 MOUNT_COMMAND = [ "mount", ]
131 UMOUNT_COMMAND = [ "umount", ]
132
133 DEFAULT_LANGUAGE = "C"
134 LANG_VAR = "LANG"
135 LOCALE_VARS = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE",
136 "LC_CTYPE", "LC_IDENTIFICATION",
137 "LC_MEASUREMENT", "LC_MESSAGES",
138 "LC_MONETARY", "LC_NAME", "LC_NUMERIC",
139 "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ]
147
148 """
149 Class representing an "unordered list".
150
151 An "unordered list" is a list in which only the contents matter, not the
152 order in which the contents appear in the list.
153
154 For instance, we might be keeping track of set of paths in a list, because
155 it's convenient to have them in that form. However, for comparison
156 purposes, we would only care that the lists contain exactly the same
157 contents, regardless of order.
158
159 I have come up with two reasonable ways of doing this, plus a couple more
160 that would work but would be a pain to implement. My first method is to
161 copy and sort each list, comparing the sorted versions. This will only work
162 if two lists with exactly the same members are guaranteed to sort in exactly
163 the same order. The second way would be to create two Sets and then compare
164 the sets. However, this would lose information about any duplicates in
165 either list. I've decided to go with option #1 for now. I'll modify this
166 code if I run into problems in the future.
167
168 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__},
169 C{__le__} and C{__lt__} list methods to change the definition of the various
170 comparison operators. In all cases, the comparison is changed to return the
171 result of the original operation I{but instead comparing sorted lists}.
172 This is going to be quite a bit slower than a normal list, so you probably
173 only want to use it on small lists.
174 """
175
177 """
178 Definition of C{==} operator for this class.
179 @param other: Other object to compare to.
180 @return: True/false depending on whether C{self == other}.
181 """
182 if other is None:
183 return False
184 selfSorted = UnorderedList.mixedsort(self[:])
185 otherSorted = UnorderedList.mixedsort(other[:])
186 return selfSorted.__eq__(otherSorted)
187
189 """
190 Definition of C{!=} operator for this class.
191 @param other: Other object to compare to.
192 @return: True/false depending on whether C{self != other}.
193 """
194 if other is None:
195 return True
196 selfSorted = UnorderedList.mixedsort(self[:])
197 otherSorted = UnorderedList.mixedsort(other[:])
198 return selfSorted.__ne__(otherSorted)
199
201 """
202 Definition of S{>=} operator for this class.
203 @param other: Other object to compare to.
204 @return: True/false depending on whether C{self >= other}.
205 """
206 if other is None:
207 return True
208 selfSorted = UnorderedList.mixedsort(self[:])
209 otherSorted = UnorderedList.mixedsort(other[:])
210 return selfSorted.__ge__(otherSorted)
211
213 """
214 Definition of C{>} operator for this class.
215 @param other: Other object to compare to.
216 @return: True/false depending on whether C{self > other}.
217 """
218 if other is None:
219 return True
220 selfSorted = UnorderedList.mixedsort(self[:])
221 otherSorted = UnorderedList.mixedsort(other[:])
222 return selfSorted.__gt__(otherSorted)
223
225 """
226 Definition of S{<=} operator for this class.
227 @param other: Other object to compare to.
228 @return: True/false depending on whether C{self <= other}.
229 """
230 if other is None:
231 return False
232 selfSorted = UnorderedList.mixedsort(self[:])
233 otherSorted = UnorderedList.mixedsort(other[:])
234 return selfSorted.__le__(otherSorted)
235
237 """
238 Definition of C{<} operator for this class.
239 @param other: Other object to compare to.
240 @return: True/false depending on whether C{self < other}.
241 """
242 if other is None:
243 return False
244 selfSorted = UnorderedList.mixedsort(self[:])
245 otherSorted = UnorderedList.mixedsort(other[:])
246 return selfSorted.__lt__(otherSorted)
247
248 @staticmethod
250 """
251 Sort a list, making sure we don't blow up if the list happens to include mixed values.
252 @see: http://stackoverflow.com/questions/26575183/how-can-i-get-2-x-like-sorting-behaviour-in-python-3-x
253 """
254 return sorted(value, key=UnorderedList.mixedkey)
255
256 @staticmethod
257
259 """Provide a key for use by mixedsort()"""
260 numeric = Real, Decimal
261 if isinstance(value, numeric):
262 typeinfo = numeric
263 else:
264 typeinfo = type(value)
265 try:
266 x = value < value
267 except TypeError:
268 value = repr(value)
269 return repr(typeinfo), value
270
277
278 """
279 Class representing a list of absolute paths.
280
281 This is an unordered list.
282
283 We override the C{append}, C{insert} and C{extend} methods to ensure that
284 any item added to the list is an absolute path.
285
286 Each item added to the list is encoded using L{encodePath}. If we don't do
287 this, we have problems trying certain operations between strings and unicode
288 objects, particularly for "odd" filenames that can't be encoded in standard
289 ASCII.
290 """
291
293 """
294 Overrides the standard C{append} method.
295 @raise ValueError: If item is not an absolute path.
296 """
297 if not os.path.isabs(item):
298 raise ValueError("Not an absolute path: [%s]" % item)
299 list.append(self, encodePath(item))
300
301 - def insert(self, index, item):
302 """
303 Overrides the standard C{insert} method.
304 @raise ValueError: If item is not an absolute path.
305 """
306 if not os.path.isabs(item):
307 raise ValueError("Not an absolute path: [%s]" % item)
308 list.insert(self, index, encodePath(item))
309
311 """
312 Overrides the standard C{insert} method.
313 @raise ValueError: If any item is not an absolute path.
314 """
315 for item in seq:
316 if not os.path.isabs(item):
317 raise ValueError("Not an absolute path: [%s]" % item)
318 for item in seq:
319 list.append(self, encodePath(item))
320
327
328 """
329 Class representing a list containing only objects with a certain type.
330
331 This is an unordered list.
332
333 We override the C{append}, C{insert} and C{extend} methods to ensure that
334 any item added to the list matches the type that is requested. The
335 comparison uses the built-in C{isinstance}, which should allow subclasses of
336 of the requested type to be added to the list as well.
337
338 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a
339 CollectDir object."} if C{objectName} is C{"CollectDir"}.
340 """
341
342 - def __init__(self, objectType, objectName):
343 """
344 Initializes a typed list for a particular type.
345 @param objectType: Type that the list elements must match.
346 @param objectName: Short string containing the "name" of the type.
347 """
348 super(ObjectTypeList, self).__init__()
349 self.objectType = objectType
350 self.objectName = objectName
351
353 """
354 Overrides the standard C{append} method.
355 @raise ValueError: If item does not match requested type.
356 """
357 if not isinstance(item, self.objectType):
358 raise ValueError("Item must be a %s object." % self.objectName)
359 list.append(self, item)
360
361 - def insert(self, index, item):
362 """
363 Overrides the standard C{insert} method.
364 @raise ValueError: If item does not match requested type.
365 """
366 if not isinstance(item, self.objectType):
367 raise ValueError("Item must be a %s object." % self.objectName)
368 list.insert(self, index, item)
369
371 """
372 Overrides the standard C{insert} method.
373 @raise ValueError: If item does not match requested type.
374 """
375 for item in seq:
376 if not isinstance(item, self.objectType):
377 raise ValueError("All items must be %s objects." % self.objectName)
378 list.extend(self, seq)
379
380
381
382
383
384
385 -class RestrictedContentList(UnorderedList):
386
387 """
388 Class representing a list containing only object with certain values.
389
390 This is an unordered list.
391
392 We override the C{append}, C{insert} and C{extend} methods to ensure that
393 any item added to the list is among the valid values. We use a standard
394 comparison, so pretty much anything can be in the list of valid values.
395
396 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be
397 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}.
398
399 @note: This class doesn't make any attempt to trap for nonsensical
400 arguments. All of the values in the values list should be of the same type
401 (i.e. strings). Then, all list operations also need to be of that type
402 (i.e. you should always insert or append just strings). If you mix types --
403 for instance lists and strings -- you will likely see AttributeError
404 exceptions or other problems.
405 """
406
407 - def __init__(self, valuesList, valuesDescr, prefix=None):
408 """
409 Initializes a list restricted to containing certain values.
410 @param valuesList: List of valid values.
411 @param valuesDescr: Short string describing list of values.
412 @param prefix: Prefix to use in error messages (None results in prefix "Item")
413 """
414 super(RestrictedContentList, self).__init__()
415 self.prefix = "Item"
416 if prefix is not None: self.prefix = prefix
417 self.valuesList = valuesList
418 self.valuesDescr = valuesDescr
419
420 - def append(self, item):
421 """
422 Overrides the standard C{append} method.
423 @raise ValueError: If item is not in the values list.
424 """
425 if item not in self.valuesList:
426 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
427 list.append(self, item)
428
429 - def insert(self, index, item):
430 """
431 Overrides the standard C{insert} method.
432 @raise ValueError: If item is not in the values list.
433 """
434 if item not in self.valuesList:
435 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
436 list.insert(self, index, item)
437
438 - def extend(self, seq):
439 """
440 Overrides the standard C{insert} method.
441 @raise ValueError: If item is not in the values list.
442 """
443 for item in seq:
444 if item not in self.valuesList:
445 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr))
446 list.extend(self, seq)
447
454
455 """
456 Class representing a list containing only strings that match a regular expression.
457
458 If C{emptyAllowed} is passed in as C{False}, then empty strings are
459 explicitly disallowed, even if they happen to match the regular expression.
460 (C{None} values are always disallowed, since string operations are not
461 permitted on C{None}.)
462
463 This is an unordered list.
464
465 We override the C{append}, C{insert} and C{extend} methods to ensure that
466 any item added to the list matches the indicated regular expression.
467
468 @note: If you try to put values that are not strings into the list, you will
469 likely get either TypeError or AttributeError exceptions as a result.
470 """
471
472 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
473 """
474 Initializes a list restricted to containing certain values.
475 @param valuesRegex: Regular expression that must be matched, as a string
476 @param emptyAllowed: Indicates whether empty or None values are allowed.
477 @param prefix: Prefix to use in error messages (None results in prefix "Item")
478 """
479 super(RegexMatchList, self).__init__()
480 self.prefix = "Item"
481 if prefix is not None: self.prefix = prefix
482 self.valuesRegex = valuesRegex
483 self.emptyAllowed = emptyAllowed
484 self.pattern = re.compile(self.valuesRegex)
485
487 """
488 Overrides the standard C{append} method.
489 @raise ValueError: If item is None
490 @raise ValueError: If item is empty and empty values are not allowed
491 @raise ValueError: If item does not match the configured regular expression
492 """
493 if item is None or (not self.emptyAllowed and item == ""):
494 raise ValueError("%s cannot be empty." % self.prefix)
495 if not self.pattern.search(item):
496 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
497 list.append(self, item)
498
499 - def insert(self, index, item):
500 """
501 Overrides the standard C{insert} method.
502 @raise ValueError: If item is None
503 @raise ValueError: If item is empty and empty values are not allowed
504 @raise ValueError: If item does not match the configured regular expression
505 """
506 if item is None or (not self.emptyAllowed and item == ""):
507 raise ValueError("%s cannot be empty." % self.prefix)
508 if not self.pattern.search(item):
509 raise ValueError("%s is not valid [%s]" % (self.prefix, item))
510 list.insert(self, index, item)
511
513 """
514 Overrides the standard C{insert} method.
515 @raise ValueError: If any item is None
516 @raise ValueError: If any item is empty and empty values are not allowed
517 @raise ValueError: If any item does not match the configured regular expression
518 """
519 for item in seq:
520 if item is None or (not self.emptyAllowed and item == ""):
521 raise ValueError("%s cannot be empty." % self.prefix)
522 if not self.pattern.search(item):
523 raise ValueError("%s is not valid: [%s]" % (self.prefix, item))
524 list.extend(self, seq)
525
526
527
528
529
530
531 -class RegexList(UnorderedList):
532
533 """
534 Class representing a list of valid regular expression strings.
535
536 This is an unordered list.
537
538 We override the C{append}, C{insert} and C{extend} methods to ensure that
539 any item added to the list is a valid regular expression.
540 """
541
543 """
544 Overrides the standard C{append} method.
545 @raise ValueError: If item is not an absolute path.
546 """
547 try:
548 re.compile(item)
549 except re.error:
550 raise ValueError("Not a valid regular expression: [%s]" % item)
551 list.append(self, item)
552
553 - def insert(self, index, item):
554 """
555 Overrides the standard C{insert} method.
556 @raise ValueError: If item is not an absolute path.
557 """
558 try:
559 re.compile(item)
560 except re.error:
561 raise ValueError("Not a valid regular expression: [%s]" % item)
562 list.insert(self, index, item)
563
565 """
566 Overrides the standard C{insert} method.
567 @raise ValueError: If any item is not an absolute path.
568 """
569 for item in seq:
570 try:
571 re.compile(item)
572 except re.error:
573 raise ValueError("Not a valid regular expression: [%s]" % item)
574 for item in seq:
575 list.append(self, item)
576
577
578
579
580
581
582 -class _Vertex(object):
583
584 """
585 Represents a vertex (or node) in a directed graph.
586 """
587
589 """
590 Constructor.
591 @param name: Name of this graph vertex.
592 @type name: String value.
593 """
594 self.name = name
595 self.endpoints = []
596 self.state = None
597
600
601 """
602 Represents a directed graph.
603
604 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set
605 B{E} of vertex pairs or edges. In a directed graph, each edge also has an
606 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph}
607 object provides a way to construct a directed graph and execute a depth-
608 first search.
609
610 This data structure was designed based on the graphing chapter in
611 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>},
612 by Steven S. Skiena.
613
614 This class is intended to be used by Cedar Backup for dependency ordering.
615 Because of this, it's not quite general-purpose. Unlike a "general" graph,
616 every vertex in this graph has at least one edge pointing to it, from a
617 special "start" vertex. This is so no vertices get "lost" either because
618 they have no dependencies or because nothing depends on them.
619 """
620
621 _UNDISCOVERED = 0
622 _DISCOVERED = 1
623 _EXPLORED = 2
624
626 """
627 Directed graph constructor.
628
629 @param name: Name of this graph.
630 @type name: String value.
631 """
632 if name is None or name == "":
633 raise ValueError("Graph name must be non-empty.")
634 self._name = name
635 self._vertices = {}
636 self._startVertex = _Vertex(None)
637
639 """
640 Official string representation for class instance.
641 """
642 return "DirectedGraph(%s)" % self.name
643
645 """
646 Informal string representation for class instance.
647 """
648 return self.__repr__()
649
651 """Equals operator, implemented in terms of original Python 2 compare operator."""
652 return self.__cmp__(other) == 0
653
655 """Less-than operator, implemented in terms of original Python 2 compare operator."""
656 return self.__cmp__(other) < 0
657
659 """Greater-than operator, implemented in terms of original Python 2 compare operator."""
660 return self.__cmp__(other) > 0
661
663 """
664 Original Python 2 comparison operator.
665 @param other: Other object to compare to.
666 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other.
667 """
668
669 if other is None:
670 return 1
671 if self.name != other.name:
672 if str(self.name or "") < str(other.name or ""):
673 return -1
674 else:
675 return 1
676 if self._vertices != other._vertices:
677 if self._vertices < other._vertices:
678 return -1
679 else:
680 return 1
681 return 0
682
684 """
685 Property target used to get the graph name.
686 """
687 return self._name
688
689 name = property(_getName, None, None, "Name of the graph.")
690
692 """
693 Creates a named vertex.
694 @param name: vertex name
695 @raise ValueError: If the vertex name is C{None} or empty.
696 """
697 if name is None or name == "":
698 raise ValueError("Vertex name must be non-empty.")
699 vertex = _Vertex(name)
700 self._startVertex.endpoints.append(vertex)
701 self._vertices[name] = vertex
702
704 """
705 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex.
706 @param start: Name of start vertex.
707 @param finish: Name of finish vertex.
708 @raise ValueError: If one of the named vertices is unknown.
709 """
710 try:
711 startVertex = self._vertices[start]
712 finishVertex = self._vertices[finish]
713 startVertex.endpoints.append(finishVertex)
714 except KeyError as e:
715 raise ValueError("Vertex [%s] could not be found." % e)
716
718 """
719 Implements a topological sort of the graph.
720
721 This method also enforces that the graph is a directed acyclic graph,
722 which is a requirement of a topological sort.
723
724 A directed acyclic graph (or "DAG") is a directed graph with no directed
725 cycles. A topological sort of a DAG is an ordering on the vertices such
726 that all edges go from left to right. Only an acyclic graph can have a
727 topological sort, but any DAG has at least one topological sort.
728
729 Since a topological sort only makes sense for an acyclic graph, this
730 method throws an exception if a cycle is found.
731
732 A depth-first search only makes sense if the graph is acyclic. If the
733 graph contains any cycles, it is not possible to determine a consistent
734 ordering for the vertices.
735
736 @note: If a particular vertex has no edges, then its position in the
737 final list depends on the order in which the vertices were created in the
738 graph. If you're using this method to determine a dependency order, this
739 makes sense: a vertex with no dependencies can go anywhere (and will).
740
741 @return: Ordering on the vertices so that all edges go from left to right.
742
743 @raise ValueError: If a cycle is found in the graph.
744 """
745 ordering = []
746 for key in self._vertices:
747 vertex = self._vertices[key]
748 vertex.state = self._UNDISCOVERED
749 for key in self._vertices:
750 vertex = self._vertices[key]
751 if vertex.state == self._UNDISCOVERED:
752 self._topologicalSort(self._startVertex, ordering)
753 return ordering
754
756 """
757 Recursive depth first search function implementing topological sort.
758 @param vertex: Vertex to search
759 @param ordering: List of vertices in proper order
760 """
761 vertex.state = self._DISCOVERED
762 for endpoint in vertex.endpoints:
763 if endpoint.state == self._UNDISCOVERED:
764 self._topologicalSort(endpoint, ordering)
765 elif endpoint.state != self._EXPLORED:
766 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name))
767 if vertex.name is not None:
768 ordering.insert(0, vertex.name)
769 vertex.state = self._EXPLORED
770
777
778 """
779 Singleton used for resolving executable paths.
780
781 Various functions throughout Cedar Backup (including extensions) need a way
782 to resolve the path of executables that they use. For instance, the image
783 functionality needs to find the C{mkisofs} executable, and the Subversion
784 extension needs to find the C{svnlook} executable. Cedar Backup's original
785 behavior was to assume that the simple name (C{"svnlook"} or whatever) was
786 available on the caller's C{$PATH}, and to fail otherwise. However, this
787 turns out to be less than ideal, since for instance the root user might not
788 always have executables like C{svnlook} in its path.
789
790 One solution is to specify a path (either via an absolute path or some sort
791 of path insertion or path appending mechanism) that would apply to the
792 C{executeCommand()} function. This is not difficult to implement, but it
793 seem like kind of a "big hammer" solution. Besides that, it might also
794 represent a security flaw (for instance, I prefer not to mess with root's
795 C{$PATH} on the application level if I don't have to).
796
797 The alternative is to set up some sort of configuration for the path to
798 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or
799 whatever. This PathResolverSingleton aims to provide a good solution to the
800 mapping problem. Callers of all sorts (extensions or not) can get an
801 instance of the singleton. Then, they call the C{lookup} method to try and
802 resolve the executable they are looking for. Through the C{lookup} method,
803 the caller can also specify a default to use if a mapping is not found.
804 This way, with no real effort on the part of the caller, behavior can neatly
805 degrade to something equivalent to the current behavior if there is no
806 special mapping or if the singleton was never initialized in the first
807 place.
808
809 Even better, extensions automagically get access to the same resolver
810 functionality, and they don't even need to understand how the mapping
811 happens. All extension authors need to do is document what executables
812 their code requires, and the standard resolver configuration section will
813 meet their needs.
814
815 The class should be initialized once through the constructor somewhere in
816 the main routine. Then, the main routine should call the L{fill} method to
817 fill in the resolver's internal structures. Everyone else who needs to
818 resolve a path will get an instance of the class using L{getInstance} and
819 will then just call the L{lookup} method.
820
821 @cvar _instance: Holds a reference to the singleton
822 @ivar _mapping: Internal mapping from resource name to path.
823 """
824
825 _instance = None
826
828 """Helper class to provide a singleton factory method."""
837
838 getInstance = _Helper()
839
844
845 - def lookup(self, name, default=None):
846 """
847 Looks up name and returns the resolved path associated with the name.
848 @param name: Name of the path resource to resolve.
849 @param default: Default to return if resource cannot be resolved.
850 @return: Resolved path associated with name, or default if name can't be resolved.
851 """
852 value = default
853 if name in list(self._mapping.keys()):
854 value = self._mapping[name]
855 logger.debug("Resolved command [%s] to [%s].", name, value)
856 return value
857
858 - def fill(self, mapping):
859 """
860 Fills in the singleton's internal mapping from name to resource.
861 @param mapping: Mapping from resource name to path.
862 @type mapping: Dictionary mapping name to path, both as strings.
863 """
864 self._mapping = { }
865 for key in list(mapping.keys()):
866 self._mapping[key] = mapping[key]
867
868
869
870
871
872
873 -class Pipe(Popen):
874 """
875 Specialized pipe class for use by C{executeCommand}.
876
877 The L{executeCommand} function needs a specialized way of interacting
878 with a pipe. First, C{executeCommand} only reads from the pipe, and
879 never writes to it. Second, C{executeCommand} needs a way to discard all
880 output written to C{stderr}, as a means of simulating the shell
881 C{2>/dev/null} construct.
882 """
883 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
884 stderr = STDOUT
885 if ignoreStderr:
886 devnull = nullDevice()
887 stderr = os.open(devnull, os.O_RDWR)
888 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
889
896
897 """
898 Class holding runtime diagnostic information.
899
900 Diagnostic information is information that is useful to get from users for
901 debugging purposes. I'm consolidating it all here into one object.
902
903 @sort: __init__, __repr__, __str__
904 """
905
906
908 """
909 Constructor for the C{Diagnostics} class.
910 """
911
913 """
914 Official string representation for class instance.
915 """
916 return "Diagnostics()"
917
919 """
920 Informal string representation for class instance.
921 """
922 return self.__repr__()
923
925 """
926 Get a map containing all of the diagnostic values.
927 @return: Map from diagnostic name to diagnostic value.
928 """
929 values = {}
930 values['version'] = self.version
931 values['interpreter'] = self.interpreter
932 values['platform'] = self.platform
933 values['encoding'] = self.encoding
934 values['locale'] = self.locale
935 values['timestamp'] = self.timestamp
936 return values
937
939 """
940 Pretty-print diagnostic information to a file descriptor.
941 @param fd: File descriptor used to print information.
942 @param prefix: Prefix string (if any) to place onto printed lines
943 @note: The C{fd} is used rather than C{print} to facilitate unit testing.
944 """
945 lines = self._buildDiagnosticLines(prefix)
946 for line in lines:
947 fd.write("%s\n" % line)
948
950 """
951 Pretty-print diagnostic information using a logger method.
952 @param method: Logger method to use for logging (i.e. logger.info)
953 @param prefix: Prefix string (if any) to place onto printed lines
954 """
955 lines = self._buildDiagnosticLines(prefix)
956 for line in lines:
957 method("%s" % line)
958
960 """
961 Build a set of pretty-printed diagnostic lines.
962 @param prefix: Prefix string (if any) to place onto printed lines
963 @return: List of strings, not terminated by newlines.
964 """
965 values = self.getValues()
966 keys = list(values.keys())
967 keys.sort()
968 tmax = Diagnostics._getMaxLength(keys) + 3
969 lines = []
970 for key in keys:
971 title = key.title()
972 title += (tmax - len(title)) * '.'
973 value = values[key]
974 line = "%s%s: %s" % (prefix, title, value)
975 lines.append(line)
976 return lines
977
978 @staticmethod
980 """
981 Get the maximum length from among a list of strings.
982 """
983 tmax = 0
984 for value in values:
985 if len(value) > tmax:
986 tmax = len(value)
987 return tmax
988
990 """
991 Property target to get the Cedar Backup version.
992 """
993 return "Cedar Backup %s (%s)" % (VERSION, DATE)
994
996 """
997 Property target to get the Python interpreter version.
998 """
999 version = sys.version_info
1000 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
1001
1003 """
1004 Property target to get the filesystem encoding.
1005 """
1006 return sys.getfilesystemencoding() or sys.getdefaultencoding()
1007
1020
1022 """
1023 Property target to get the default locale that is in effect.
1024 """
1025 try:
1026 import locale
1027 return locale.getdefaultlocale()[0]
1028 except:
1029 return "(unknown)"
1030
1032 """
1033 Property target to get a current date/time stamp.
1034 """
1035 try:
1036 import datetime
1037 return datetime.datetime.utcnow().ctime() + " UTC"
1038 except:
1039 return "(unknown)"
1040
1041 version = property(_getVersion, None, None, "Cedar Backup version.")
1042 interpreter = property(_getInterpreter, None, None, "Python interpreter version.")
1043 platform = property(_getPlatform, None, None, "Platform identifying information.")
1044 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.")
1045 locale = property(_getLocale, None, None, "Locale that is in effect.")
1046 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057 -def sortDict(d):
1058 """
1059 Returns the keys of the dictionary sorted by value.
1060 @param d: Dictionary to operate on
1061 @return: List of dictionary keys sorted in order by dictionary value.
1062 """
1063 items = list(d.items())
1064 items.sort(key=lambda x: (x[1], x[0]))
1065 return [key for key, value in items]
1066
1067
1068
1069
1070
1071
1072 -def removeKeys(d, keys):
1073 """
1074 Removes all of the keys from the dictionary.
1075 The dictionary is altered in-place.
1076 Each key must exist in the dictionary.
1077 @param d: Dictionary to operate on
1078 @param keys: List of keys to remove
1079 @raise KeyError: If one of the keys does not exist
1080 """
1081 for key in keys:
1082 del d[key]
1083
1084
1085
1086
1087
1088
1089 -def convertSize(size, fromUnit, toUnit):
1090 """
1091 Converts a size in one unit to a size in another unit.
1092
1093 This is just a convenience function so that the functionality can be
1094 implemented in just one place. Internally, we convert values to bytes and
1095 then to the final unit.
1096
1097 The available units are:
1098
1099 - C{UNIT_BYTES} - Bytes
1100 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B
1101 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB
1102 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB
1103 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B
1104
1105 @param size: Size to convert
1106 @type size: Integer or float value in units of C{fromUnit}
1107
1108 @param fromUnit: Unit to convert from
1109 @type fromUnit: One of the units listed above
1110
1111 @param toUnit: Unit to convert to
1112 @type toUnit: One of the units listed above
1113
1114 @return: Number converted to new unit, as a float.
1115 @raise ValueError: If one of the units is invalid.
1116 """
1117 if size is None:
1118 raise ValueError("Cannot convert size of None.")
1119 if fromUnit == UNIT_BYTES:
1120 byteSize = float(size)
1121 elif fromUnit == UNIT_KBYTES:
1122 byteSize = float(size) * BYTES_PER_KBYTE
1123 elif fromUnit == UNIT_MBYTES:
1124 byteSize = float(size) * BYTES_PER_MBYTE
1125 elif fromUnit == UNIT_GBYTES:
1126 byteSize = float(size) * BYTES_PER_GBYTE
1127 elif fromUnit == UNIT_SECTORS:
1128 byteSize = float(size) * BYTES_PER_SECTOR
1129 else:
1130 raise ValueError("Unknown 'from' unit %s." % fromUnit)
1131 if toUnit == UNIT_BYTES:
1132 return byteSize
1133 elif toUnit == UNIT_KBYTES:
1134 return byteSize / BYTES_PER_KBYTE
1135 elif toUnit == UNIT_MBYTES:
1136 return byteSize / BYTES_PER_MBYTE
1137 elif toUnit == UNIT_GBYTES:
1138 return byteSize / BYTES_PER_GBYTE
1139 elif toUnit == UNIT_SECTORS:
1140 return byteSize / BYTES_PER_SECTOR
1141 else:
1142 raise ValueError("Unknown 'to' unit %s." % toUnit)
1143
1144
1145
1146
1147
1148
1149 -def displayBytes(bytes, digits=2):
1150 """
1151 Format a byte quantity so it can be sensibly displayed.
1152
1153 It's rather difficult to look at a number like "72372224 bytes" and get any
1154 meaningful information out of it. It would be more useful to see something
1155 like "69.02 MB". That's what this function does. Any time you want to display
1156 a byte value, i.e.::
1157
1158 print "Size: %s bytes" % bytes
1159
1160 Call this function instead::
1161
1162 print "Size: %s" % displayBytes(bytes)
1163
1164 What comes out will be sensibly formatted. The indicated number of digits
1165 will be listed after the decimal point, rounded based on whatever rules are
1166 used by Python's standard C{%f} string format specifier. (Values less than 1
1167 kB will be listed in bytes and will not have a decimal point, since the
1168 concept of a fractional byte is nonsensical.)
1169
1170 @param bytes: Byte quantity.
1171 @type bytes: Integer number of bytes.
1172
1173 @param digits: Number of digits to display after the decimal point.
1174 @type digits: Integer value, typically 2-5.
1175
1176 @return: String, formatted for sensible display.
1177 """
1178 if bytes is None:
1179 raise ValueError("Cannot display byte value of None.")
1180 bytes = float(bytes)
1181 if math.fabs(bytes) < BYTES_PER_KBYTE:
1182 fmt = "%.0f bytes"
1183 value = bytes
1184 elif math.fabs(bytes) < BYTES_PER_MBYTE:
1185 fmt = "%." + "%d" % digits + "f kB"
1186 value = bytes / BYTES_PER_KBYTE
1187 elif math.fabs(bytes) < BYTES_PER_GBYTE:
1188 fmt = "%." + "%d" % digits + "f MB"
1189 value = bytes / BYTES_PER_MBYTE
1190 else:
1191 fmt = "%." + "%d" % digits + "f GB"
1192 value = bytes / BYTES_PER_GBYTE
1193 return fmt % value
1194
1201 """
1202 Gets a reference to a named function.
1203
1204 This does some hokey-pokey to get back a reference to a dynamically named
1205 function. For instance, say you wanted to get a reference to the
1206 C{os.path.isdir} function. You could use::
1207
1208 myfunc = getFunctionReference("os.path", "isdir")
1209
1210 Although we won't bomb out directly, behavior is pretty much undefined if
1211 you pass in C{None} or C{""} for either C{module} or C{function}.
1212
1213 The only validation we enforce is that whatever we get back must be
1214 callable.
1215
1216 I derived this code based on the internals of the Python unittest
1217 implementation. I don't claim to completely understand how it works.
1218
1219 @param module: Name of module associated with function.
1220 @type module: Something like "os.path" or "CedarBackup3.util"
1221
1222 @param function: Name of function
1223 @type function: Something like "isdir" or "getUidGid"
1224
1225 @return: Reference to function associated with name.
1226
1227 @raise ImportError: If the function cannot be found.
1228 @raise ValueError: If the resulting reference is not callable.
1229
1230 @copyright: Some of this code, prior to customization, was originally part
1231 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python
1232 Software Foundation; All Rights Reserved.
1233 """
1234 parts = []
1235 if module is not None and module != "":
1236 parts = module.split(".")
1237 if function is not None and function != "":
1238 parts.append(function)
1239 copy = parts[:]
1240 while copy:
1241 try:
1242 module = __import__(".".join(copy))
1243 break
1244 except ImportError:
1245 del copy[-1]
1246 if not copy: raise
1247 parts = parts[1:]
1248 obj = module
1249 for part in parts:
1250 obj = getattr(obj, part)
1251 if not isinstance(obj, collections.Callable):
1252 raise ValueError("Reference to %s.%s is not callable." % (module, function))
1253 return obj
1254
1255
1256
1257
1258
1259
1260 -def getUidGid(user, group):
1261 """
1262 Get the uid/gid associated with a user/group pair
1263
1264 This is a no-op if user/group functionality is not available on the platform.
1265
1266 @param user: User name
1267 @type user: User name as a string
1268
1269 @param group: Group name
1270 @type group: Group name as a string
1271
1272 @return: Tuple C{(uid, gid)} matching passed-in user and group.
1273 @raise ValueError: If the ownership user/group values are invalid
1274 """
1275 if _UID_GID_AVAILABLE:
1276 try:
1277 uid = pwd.getpwnam(user)[2]
1278 gid = grp.getgrnam(group)[2]
1279 return (uid, gid)
1280 except Exception as e:
1281 logger.debug("Error looking up uid and gid for [%s:%s]: %s", user, group, e)
1282 raise ValueError("Unable to lookup up uid and gid for passed in user/group.")
1283 else:
1284 return (0, 0)
1285
1292 """
1293 Changes ownership of path to match the user and group.
1294
1295 This is a no-op if user/group functionality is not available on the
1296 platform, or if the either passed-in user or group is C{None}. Further, we
1297 won't even try to do it unless running as root, since it's unlikely to work.
1298
1299 @param path: Path whose ownership to change.
1300 @param user: User which owns file.
1301 @param group: Group which owns file.
1302 """
1303 if _UID_GID_AVAILABLE:
1304 if user is None or group is None:
1305 logger.debug("User or group is None, so not attempting to change owner on [%s].", path)
1306 elif not isRunningAsRoot():
1307 logger.debug("Not root, so not attempting to change owner on [%s].", path)
1308 else:
1309 try:
1310 (uid, gid) = getUidGid(user, group)
1311 os.chown(path, uid, gid)
1312 except Exception as e:
1313 logger.error("Error changing ownership of [%s]: %s", path, e)
1314
1321 """
1322 Indicates whether the program is running as the root user.
1323 """
1324 return os.getuid() == 0
1325
1332 """
1333 Splits a command line string into a list of arguments.
1334
1335 Unfortunately, there is no "standard" way to parse a command line string,
1336 and it's actually not an easy problem to solve portably (essentially, we
1337 have to emulate the shell argument-processing logic). This code only
1338 respects double quotes (C{"}) for grouping arguments, not single quotes
1339 (C{'}). Make sure you take this into account when building your command
1340 line.
1341
1342 Incidentally, I found this particular parsing method while digging around in
1343 Google Groups, and I tweaked it for my own use.
1344
1345 @param commandLine: Command line string
1346 @type commandLine: String, i.e. "cback3 --verbose stage store"
1347
1348 @return: List of arguments, suitable for passing to C{popen2}.
1349
1350 @raise ValueError: If the command line is None.
1351 """
1352 if commandLine is None:
1353 raise ValueError("Cannot split command line of None.")
1354 fields = re.findall('[^ "]+|"[^"]+"', commandLine)
1355 fields = [field.replace('"', '') for field in fields]
1356 return fields
1357
1364 """
1365 Resolves the real path to a command through the path resolver mechanism.
1366
1367 Both extensions and standard Cedar Backup functionality need a way to
1368 resolve the "real" location of various executables. Normally, they assume
1369 that these executables are on the system path, but some callers need to
1370 specify an alternate location.
1371
1372 Ideally, we want to handle this configuration in a central location. The
1373 Cedar Backup path resolver mechanism (a singleton called
1374 L{PathResolverSingleton}) provides the central location to store the
1375 mappings. This function wraps access to the singleton, and is what all
1376 functions (extensions or standard functionality) should call if they need to
1377 find a command.
1378
1379 The passed-in command must actually be a list, in the standard form used by
1380 all existing Cedar Backup code (something like C{["svnlook", ]}). The
1381 lookup will actually be done on the first element in the list, and the
1382 returned command will always be in list form as well.
1383
1384 If the passed-in command can't be resolved or no mapping exists, then the
1385 command itself will be returned unchanged. This way, we neatly fall back on
1386 default behavior if we have no sensible alternative.
1387
1388 @param command: Command to resolve.
1389 @type command: List form of command, i.e. C{["svnlook", ]}.
1390
1391 @return: Path to command or just command itself if no mapping exists.
1392 """
1393 singleton = PathResolverSingleton.getInstance()
1394 name = command[0]
1395 result = command[:]
1396 result[0] = singleton.lookup(name, name)
1397 return result
1398
1399
1400
1401
1402
1403
1404 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1405 """
1406 Executes a shell command, hopefully in a safe way.
1407
1408 This function exists to replace direct calls to C{os.popen} in the Cedar
1409 Backup code. It's not safe to call a function such as C{os.popen()} with
1410 untrusted arguments, since that can cause problems if the string contains
1411 non-safe variables or other constructs (imagine that the argument is
1412 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/;
1413 echo"} in the current environment).
1414
1415 Instead, it's safer to pass a list of arguments in the style supported bt
1416 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe}
1417 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}.
1418
1419 Under the normal case, this function will return a tuple of C{(status,
1420 None)} where the status is the wait-encoded return status of the call per
1421 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as
1422 C{True}, the function will return a tuple of C{(status, output)} where
1423 C{output} is a list of strings, one entry per line in the output from the
1424 command. Output is always logged to the C{outputLogger.info()} target,
1425 regardless of whether it's returned.
1426
1427 By default, C{stdout} and C{stderr} will be intermingled in the output.
1428 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be
1429 included in the output.
1430
1431 The C{doNotLog} parameter exists so that callers can force the function to
1432 not log command output to the debug log. Normally, you would want to log.
1433 However, if you're using this function to write huge output files (i.e.
1434 database backups written to C{stdout}) then you might want to avoid putting
1435 all that information into the debug log.
1436
1437 The C{outputFile} parameter exists to make it easier for a caller to push
1438 output into a file, i.e. as a substitute for redirection to a file. If this
1439 value is passed in, each time a line of output is generated, it will be
1440 written to the file using C{outputFile.write()}. At the end, the file
1441 descriptor will be flushed using C{outputFile.flush()}. The caller
1442 maintains responsibility for closing the file object appropriately.
1443
1444 @note: I know that it's a bit confusing that the command and the arguments
1445 are both lists. I could have just required the caller to pass in one big
1446 list. However, I think it makes some sense to keep the command (the
1447 constant part of what we're executing, i.e. C{"scp -B"}) separate from its
1448 arguments, even if they both end up looking kind of similar.
1449
1450 @note: You cannot redirect output via shell constructs (i.e. C{>file},
1451 C{2>/dev/null}, etc.) using this function. The redirection string would be
1452 passed to the command just like any other argument. However, you can
1453 implement the equivalent to redirection using C{ignoreStderr} and
1454 C{outputFile}, as discussed above.
1455
1456 @note: The operating system environment is partially sanitized before
1457 the command is invoked. See L{sanitizeEnvironment} for details.
1458
1459 @param command: Shell command to execute
1460 @type command: List of individual arguments that make up the command
1461
1462 @param args: List of arguments to the command
1463 @type args: List of additional arguments to the command
1464
1465 @param returnOutput: Indicates whether to return the output of the command
1466 @type returnOutput: Boolean C{True} or C{False}
1467
1468 @param ignoreStderr: Whether stderr should be discarded
1469 @type ignoreStderr: Boolean True or False
1470
1471 @param doNotLog: Indicates that output should not be logged.
1472 @type doNotLog: Boolean C{True} or C{False}
1473
1474 @param outputFile: File object that all output should be written to.
1475 @type outputFile: File object as returned from C{open()} or C{file()}, configured for binary write
1476
1477 @return: Tuple of C{(result, output)} as described above.
1478 """
1479 logger.debug("Executing command %s with args %s.", command, args)
1480 outputLogger.info("Executing command %s with args %s.", command, args)
1481 if doNotLog:
1482 logger.debug("Note: output will not be logged, per the doNotLog flag.")
1483 outputLogger.info("Note: output will not be logged, per the doNotLog flag.")
1484 output = []
1485 fields = command[:]
1486 fields.extend(args)
1487 try:
1488 sanitizeEnvironment()
1489 try:
1490 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1491 except OSError:
1492
1493
1494
1495 pipe = Pipe(fields, ignoreStderr=ignoreStderr)
1496 while True:
1497 line = pipe.stdout.readline()
1498 if not line: break
1499 if returnOutput: output.append(line.decode("utf-8"))
1500 if outputFile is not None: outputFile.write(line)
1501 if not doNotLog: outputLogger.info(line.decode("utf-8")[:-1])
1502 if outputFile is not None:
1503 try:
1504 outputFile.flush()
1505 except: pass
1506 if returnOutput:
1507 return (pipe.wait(), output)
1508 else:
1509 return (pipe.wait(), None)
1510 except OSError as e:
1511 try:
1512 if returnOutput:
1513 if output != []:
1514 return (pipe.wait(), output)
1515 else:
1516 return (pipe.wait(), [ e, ])
1517 else:
1518 return (pipe.wait(), None)
1519 except UnboundLocalError:
1520 if returnOutput:
1521 return (256, [])
1522 else:
1523 return (256, None)
1524
1531 """
1532 Calculates the age (in days) of a file.
1533
1534 The "age" of a file is the amount of time since the file was last used, per
1535 the most recent of the file's C{st_atime} and C{st_mtime} values.
1536
1537 Technically, we only intend this function to work with files, but it will
1538 probably work with anything on the filesystem.
1539
1540 @param path: Path to a file on disk.
1541
1542 @return: Age of the file in days (possibly fractional).
1543 @raise OSError: If the file doesn't exist.
1544 """
1545 currentTime = int(time.time())
1546 fileStats = os.stat(path)
1547 lastUse = max(fileStats.st_atime, fileStats.st_mtime)
1548 ageInSeconds = currentTime - lastUse
1549 ageInDays = ageInSeconds / SECONDS_PER_DAY
1550 return ageInDays
1551
1552
1553
1554
1555
1556
1557 -def mount(devicePath, mountPoint, fsType):
1558 """
1559 Mounts the indicated device at the indicated mount point.
1560
1561 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount
1562 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any
1563 filesystem type that is supported by C{mount} on your platform. If the type
1564 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may
1565 not work on all systems.
1566
1567 @note: This only works on platforms that have a concept of "mounting" a
1568 filesystem through a command-line C{"mount"} command, like UNIXes. It
1569 won't work on Windows.
1570
1571 @param devicePath: Path of device to be mounted.
1572 @param mountPoint: Path that device should be mounted at.
1573 @param fsType: Type of the filesystem assumed to be available via the device.
1574
1575 @raise IOError: If the device cannot be mounted.
1576 """
1577 if fsType is None:
1578 args = [ devicePath, mountPoint ]
1579 else:
1580 args = [ "-t", fsType, devicePath, mountPoint ]
1581 command = resolveCommand(MOUNT_COMMAND)
1582 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0]
1583 if result != 0:
1584 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1585
1586
1587
1588
1589
1590
1591 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1592 """
1593 Unmounts whatever device is mounted at the indicated mount point.
1594
1595 Sometimes, it might not be possible to unmount the mount point immediately,
1596 if there are still files open there. Use the C{attempts} and C{waitSeconds}
1597 arguments to indicate how many unmount attempts to make and how many seconds
1598 to wait between attempts. If you pass in zero attempts, no attempts will be
1599 made (duh).
1600
1601 If the indicated mount point is not really a mount point per
1602 C{os.path.ismount()}, then it will be ignored. This seems to be a safer
1603 check then looking through C{/etc/mtab}, since C{ismount()} is already in
1604 the Python standard library and is documented as working on all POSIX
1605 systems.
1606
1607 If C{removeAfter} is C{True}, then the mount point will be removed using
1608 C{os.rmdir()} after the unmount action succeeds. If for some reason the
1609 mount point is not a directory, then it will not be removed.
1610
1611 @note: This only works on platforms that have a concept of "mounting" a
1612 filesystem through a command-line C{"mount"} command, like UNIXes. It
1613 won't work on Windows.
1614
1615 @param mountPoint: Mount point to be unmounted.
1616 @param removeAfter: Remove the mount point after unmounting it.
1617 @param attempts: Number of times to attempt the unmount.
1618 @param waitSeconds: Number of seconds to wait between repeated attempts.
1619
1620 @raise IOError: If the mount point is still mounted after attempts are exhausted.
1621 """
1622 if os.path.ismount(mountPoint):
1623 for attempt in range(0, attempts):
1624 logger.debug("Making attempt %d to unmount [%s].", attempt, mountPoint)
1625 command = resolveCommand(UMOUNT_COMMAND)
1626 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0]
1627 if result != 0:
1628 logger.error("Error [%d] unmounting [%s] on attempt %d.", result, mountPoint, attempt)
1629 elif os.path.ismount(mountPoint):
1630 logger.error("After attempt %d, [%s] is still mounted.", attempt, mountPoint)
1631 else:
1632 logger.debug("Successfully unmounted [%s] on attempt %d.", mountPoint, attempt)
1633 break
1634 if attempt+1 < attempts:
1635 if waitSeconds > 0:
1636 logger.info("Sleeping %d second(s) before next unmount attempt.", waitSeconds)
1637 time.sleep(waitSeconds)
1638 else:
1639 if os.path.ismount(mountPoint):
1640 raise IOError("Unable to unmount [%s] after %d attempts.", mountPoint, attempts)
1641 logger.info("Mount point [%s] seems to have finally gone away.", mountPoint)
1642 if os.path.isdir(mountPoint) and removeAfter:
1643 logger.debug("Removing mount point [%s].", mountPoint)
1644 os.rmdir(mountPoint)
1645
1652 """
1653 Indicates whether a specific filesystem device is currently mounted.
1654
1655 We determine whether the device is mounted by looking through the system's
1656 C{mtab} file. This file shows every currently-mounted filesystem, ordered
1657 by device. We only do the check if the C{mtab} file exists and is readable.
1658 Otherwise, we assume that the device is not mounted.
1659
1660 @note: This only works on platforms that have a concept of an mtab file
1661 to show mounted volumes, like UNIXes. It won't work on Windows.
1662
1663 @param devicePath: Path of device to be checked
1664
1665 @return: True if device is mounted, false otherwise.
1666 """
1667 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK):
1668 realPath = os.path.realpath(devicePath)
1669 with open(MTAB_FILE) as f:
1670 lines = f.readlines()
1671 for line in lines:
1672 (mountDevice, mountPoint, remainder) = line.split(None, 2)
1673 if mountDevice in [ devicePath, realPath, ]:
1674 logger.debug("Device [%s] is mounted at [%s].", devicePath, mountPoint)
1675 return True
1676 return False
1677
1684 """
1685 Safely encodes a filesystem path as a Unicode string, converting bytes to fileystem encoding if necessary.
1686 @param path: Path to encode
1687 @return: Path, as a string, encoded appropriately
1688 @raise ValueError: If the path cannot be encoded properly.
1689 @see: http://lucumr.pocoo.org/2013/7/2/the-updated-guide-to-unicode/
1690 """
1691 if path is None:
1692 return path
1693 try:
1694 if isinstance(path, bytes):
1695 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding()
1696 path = path.decode(encoding, "surrogateescape")
1697 return path
1698 except UnicodeError as e:
1699 raise ValueError("Path could not be safely encoded as %s: %s" % (encoding, str(e)))
1700
1707 """
1708 Attempts to portably return the null device on this system.
1709
1710 The null device is something like C{/dev/null} on a UNIX system. The name
1711 varies on other platforms.
1712 """
1713 return os.devnull
1714
1721 """
1722 Converts English day name to numeric day of week as from C{time.localtime}.
1723
1724 For instance, the day C{monday} would be converted to the number C{0}.
1725
1726 @param dayName: Day of week to convert
1727 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1728
1729 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible.
1730 """
1731 if dayName.lower() == "monday":
1732 return 0
1733 elif dayName.lower() == "tuesday":
1734 return 1
1735 elif dayName.lower() == "wednesday":
1736 return 2
1737 elif dayName.lower() == "thursday":
1738 return 3
1739 elif dayName.lower() == "friday":
1740 return 4
1741 elif dayName.lower() == "saturday":
1742 return 5
1743 elif dayName.lower() == "sunday":
1744 return 6
1745 else:
1746 return -1
1747
1754 """
1755 Indicates whether "today" is the backup starting day per configuration.
1756
1757 If the current day's English name matches the indicated starting day, then
1758 today is a starting day.
1759
1760 @param startingDay: Configured starting day.
1761 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc.
1762
1763 @return: Boolean indicating whether today is the starting day.
1764 """
1765 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay)
1766 if value:
1767 logger.debug("Today is the start of the week.")
1768 else:
1769 logger.debug("Today is NOT the start of the week.")
1770 return value
1771
1778 """
1779 Returns a "normalized" path based on a path name.
1780
1781 A normalized path is a representation of a path that is also a valid file
1782 name. To make a valid file name out of a complete path, we have to convert
1783 or remove some characters that are significant to the filesystem -- in
1784 particular, the path separator and any leading C{'.'} character (which would
1785 cause the file to be hidden in a file listing).
1786
1787 Note that this is a one-way transformation -- you can't safely derive the
1788 original path from the normalized path.
1789
1790 To normalize a path, we begin by looking at the first character. If the
1791 first character is C{'/'} or C{'\\'}, it gets removed. If the first
1792 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the
1793 rest of the path and convert all remaining C{'/'} or C{'\\'} characters
1794 C{'-'}, and all remaining whitespace characters to C{'_'}.
1795
1796 As a special case, a path consisting only of a single C{'/'} or C{'\\'}
1797 character will be converted to C{'-'}.
1798
1799 @param path: Path to normalize
1800
1801 @return: Normalized path as described above.
1802
1803 @raise ValueError: If the path is None
1804 """
1805 if path is None:
1806 raise ValueError("Cannot normalize path None.")
1807 elif len(path) == 0:
1808 return path
1809 elif path == "/" or path == "\\":
1810 return "-"
1811 else:
1812 normalized = path
1813 normalized = re.sub(r"^\/", "", normalized)
1814 normalized = re.sub(r"^\\", "", normalized)
1815 normalized = re.sub(r"^\.", "_", normalized)
1816 normalized = re.sub(r"\/", "-", normalized)
1817 normalized = re.sub(r"\\", "-", normalized)
1818 normalized = re.sub(r"\s", "_", normalized)
1819 return normalized
1820
1827 """
1828 Sanitizes the operating system environment.
1829
1830 The operating system environment is contained in C{os.environ}. This method
1831 sanitizes the contents of that dictionary.
1832
1833 Currently, all it does is reset the locale (removing C{$LC_*}) and set the
1834 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count
1835 on consistent localization regardless of what the end-user has configured.
1836 This is important for code that needs to parse program output.
1837
1838 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already
1839 set to the proper value, it is not re-set, so we can avoid the memory leaks
1840 that are documented to occur on BSD-based systems.
1841
1842 @return: Copy of the sanitized environment.
1843 """
1844 for var in LOCALE_VARS:
1845 if var in os.environ:
1846 del os.environ[var]
1847 if LANG_VAR in os.environ:
1848 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE:
1849 os.environ[LANG_VAR] = DEFAULT_LANGUAGE
1850 return os.environ.copy()
1851
1858 """
1859 Deference a soft link, optionally normalizing it to an absolute path.
1860 @param path: Path of link to dereference
1861 @param absolute: Whether to normalize the result to an absolute path
1862 @return: Dereferenced path, or original path if original is not a link.
1863 """
1864 if os.path.islink(path):
1865 result = os.readlink(path)
1866 if absolute and not os.path.isabs(result):
1867 result = os.path.abspath(os.path.join(os.path.dirname(path), result))
1868 return result
1869 return path
1870
1871
1872
1873
1874
1875
1876 -def checkUnique(prefix, values):
1877 """
1878 Checks that all values are unique.
1879
1880 The values list is checked for duplicate values. If there are
1881 duplicates, an exception is thrown. All duplicate values are listed in
1882 the exception.
1883
1884 @param prefix: Prefix to use in the thrown exception
1885 @param values: List of values to check
1886
1887 @raise ValueError: If there are duplicates in the list
1888 """
1889 values.sort()
1890 duplicates = []
1891 for i in range(1, len(values)):
1892 if values[i-1] == values[i]:
1893 duplicates.append(values[i])
1894 if duplicates:
1895 raise ValueError("%s %s" % (prefix, duplicates))
1896
1903 """
1904 Parses a list of values out of a comma-separated string.
1905
1906 The items in the list are split by comma, and then have whitespace
1907 stripped. As a special case, if C{commaString} is C{None}, then C{None}
1908 will be returned.
1909
1910 @param commaString: List of values in comma-separated string format.
1911 @return: Values from commaString split into a list, or C{None}.
1912 """
1913 if commaString is None:
1914 return None
1915 else:
1916 pass1 = commaString.split(",")
1917 pass2 = []
1918 for item in pass1:
1919 item = item.strip()
1920 if len(item) > 0:
1921 pass2.append(item)
1922 return pass2
1923