1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38 """
39 Provides backup peer-related objects and utility functions.
40
41 @sort: LocalPeer, RemotePeer
42
43 @var DEF_COLLECT_INDICATOR: Name of the default collect indicator file.
44 @var DEF_STAGE_INDICATOR: Name of the default stage indicator file.
45
46 @author: Kenneth J. Pronovici <pronovic@ieee.org>
47 """
48
49
50
51
52
53
54
55 import os
56 import logging
57 import shutil
58
59
60 from CedarBackup2.filesystem import FilesystemList
61 from CedarBackup2.util import resolveCommand, executeCommand, isRunningAsRoot
62 from CedarBackup2.util import splitCommandLine, encodePath
63 from CedarBackup2.config import VALID_FAILURE_MODES
64
65
66
67
68
69
70 logger = logging.getLogger("CedarBackup2.log.peer")
71
72 DEF_RCP_COMMAND = [ "/usr/bin/scp", "-B", "-q", "-C" ]
73 DEF_RSH_COMMAND = [ "/usr/bin/ssh", ]
74 DEF_CBACK_COMMAND = "/usr/bin/cback"
75
76 DEF_COLLECT_INDICATOR = "cback.collect"
77 DEF_STAGE_INDICATOR = "cback.stage"
78
79 SU_COMMAND = [ "su" ]
87
88
89
90
91
92 """
93 Backup peer representing a local peer in a backup pool.
94
95 This is a class representing a local (non-network) peer in a backup pool.
96 Local peers are backed up by simple filesystem copy operations. A local
97 peer has associated with it a name (typically, but not necessarily, a
98 hostname) and a collect directory.
99
100 The public methods other than the constructor are part of a "backup peer"
101 interface shared with the C{RemotePeer} class.
102
103 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
104 _copyLocalDir, _copyLocalFile, name, collectDir
105 """
106
107
108
109
110
111 - def __init__(self, name, collectDir, ignoreFailureMode=None):
112 """
113 Initializes a local backup peer.
114
115 Note that the collect directory must be an absolute path, but does not
116 have to exist when the object is instantiated. We do a lazy validation
117 on this value since we could (potentially) be creating peer objects
118 before an ongoing backup completed.
119
120 @param name: Name of the backup peer
121 @type name: String, typically a hostname
122
123 @param collectDir: Path to the peer's collect directory
124 @type collectDir: String representing an absolute local path on disk
125
126 @param ignoreFailureMode: Ignore failure mode for this peer
127 @type ignoreFailureMode: One of VALID_FAILURE_MODES
128
129 @raise ValueError: If the name is empty.
130 @raise ValueError: If collect directory is not an absolute path.
131 """
132 self._name = None
133 self._collectDir = None
134 self._ignoreFailureMode = None
135 self.name = name
136 self.collectDir = collectDir
137 self.ignoreFailureMode = ignoreFailureMode
138
139
140
141
142
143
145 """
146 Property target used to set the peer name.
147 The value must be a non-empty string and cannot be C{None}.
148 @raise ValueError: If the value is an empty string or C{None}.
149 """
150 if value is None or len(value) < 1:
151 raise ValueError("Peer name must be a non-empty string.")
152 self._name = value
153
155 """
156 Property target used to get the peer name.
157 """
158 return self._name
159
161 """
162 Property target used to set the collect directory.
163 The value must be an absolute path and cannot be C{None}.
164 It does not have to exist on disk at the time of assignment.
165 @raise ValueError: If the value is C{None} or is not an absolute path.
166 @raise ValueError: If a path cannot be encoded properly.
167 """
168 if value is None or not os.path.isabs(value):
169 raise ValueError("Collect directory must be an absolute path.")
170 self._collectDir = encodePath(value)
171
173 """
174 Property target used to get the collect directory.
175 """
176 return self._collectDir
177
179 """
180 Property target used to set the ignoreFailure mode.
181 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
182 @raise ValueError: If the value is not valid.
183 """
184 if value is not None:
185 if value not in VALID_FAILURE_MODES:
186 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
187 self._ignoreFailureMode = value
188
190 """
191 Property target used to get the ignoreFailure mode.
192 """
193 return self._ignoreFailureMode
194
195 name = property(_getName, _setName, None, "Name of the peer.")
196 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
197 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
198
199
200
201
202
203
204 - def stagePeer(self, targetDir, ownership=None, permissions=None):
205 """
206 Stages data from the peer into the indicated local target directory.
207
208 The collect and target directories must both already exist before this
209 method is called. If passed in, ownership and permissions will be
210 applied to the files that are copied.
211
212 @note: The caller is responsible for checking that the indicator exists,
213 if they care. This function only stages the files within the directory.
214
215 @note: If you have user/group as strings, call the L{util.getUidGid} function
216 to get the associated uid/gid as an ownership tuple.
217
218 @param targetDir: Target directory to write data into
219 @type targetDir: String representing a directory on disk
220
221 @param ownership: Owner and group that the staged files should have
222 @type ownership: Tuple of numeric ids C{(uid, gid)}
223
224 @param permissions: Permissions that the staged files should have
225 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
226
227 @return: Number of files copied from the source directory to the target directory.
228
229 @raise ValueError: If collect directory is not a directory or does not exist
230 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
231 @raise ValueError: If a path cannot be encoded properly.
232 @raise IOError: If there were no files to stage (i.e. the directory was empty)
233 @raise IOError: If there is an IO error copying a file.
234 @raise OSError: If there is an OS error copying or changing permissions on a file
235 """
236 targetDir = encodePath(targetDir)
237 if not os.path.isabs(targetDir):
238 logger.debug("Target directory [%s] not an absolute path.", targetDir)
239 raise ValueError("Target directory must be an absolute path.")
240 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
241 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir)
242 raise ValueError("Collect directory is not a directory or does not exist on disk.")
243 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
244 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir)
245 raise ValueError("Target directory is not a directory or does not exist on disk.")
246 count = LocalPeer._copyLocalDir(self.collectDir, targetDir, ownership, permissions)
247 if count == 0:
248 raise IOError("Did not copy any files from local peer.")
249 return count
250
252 """
253 Checks the collect indicator in the peer's staging directory.
254
255 When a peer has completed collecting its backup files, it will write an
256 empty indicator file into its collect directory. This method checks to
257 see whether that indicator has been written. We're "stupid" here - if
258 the collect directory doesn't exist, you'll naturally get back C{False}.
259
260 If you need to, you can override the name of the collect indicator file
261 by passing in a different name.
262
263 @param collectIndicator: Name of the collect indicator file to check
264 @type collectIndicator: String representing name of a file in the collect directory
265
266 @return: Boolean true/false depending on whether the indicator exists.
267 @raise ValueError: If a path cannot be encoded properly.
268 """
269 collectIndicator = encodePath(collectIndicator)
270 if collectIndicator is None:
271 return os.path.exists(os.path.join(self.collectDir, DEF_COLLECT_INDICATOR))
272 else:
273 return os.path.exists(os.path.join(self.collectDir, collectIndicator))
274
276 """
277 Writes the stage indicator in the peer's staging directory.
278
279 When the master has completed collecting its backup files, it will write
280 an empty indicator file into the peer's collect directory. The presence
281 of this file implies that the staging process is complete.
282
283 If you need to, you can override the name of the stage indicator file by
284 passing in a different name.
285
286 @note: If you have user/group as strings, call the L{util.getUidGid}
287 function to get the associated uid/gid as an ownership tuple.
288
289 @param stageIndicator: Name of the indicator file to write
290 @type stageIndicator: String representing name of a file in the collect directory
291
292 @param ownership: Owner and group that the indicator file should have
293 @type ownership: Tuple of numeric ids C{(uid, gid)}
294
295 @param permissions: Permissions that the indicator file should have
296 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
297
298 @raise ValueError: If collect directory is not a directory or does not exist
299 @raise ValueError: If a path cannot be encoded properly.
300 @raise IOError: If there is an IO error creating the file.
301 @raise OSError: If there is an OS error creating or changing permissions on the file
302 """
303 stageIndicator = encodePath(stageIndicator)
304 if not os.path.exists(self.collectDir) or not os.path.isdir(self.collectDir):
305 logger.debug("Collect directory [%s] is not a directory or does not exist on disk.", self.collectDir)
306 raise ValueError("Collect directory is not a directory or does not exist on disk.")
307 if stageIndicator is None:
308 fileName = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
309 else:
310 fileName = os.path.join(self.collectDir, stageIndicator)
311 LocalPeer._copyLocalFile(None, fileName, ownership, permissions)
312
313
314
315
316
317
318 @staticmethod
319 - def _copyLocalDir(sourceDir, targetDir, ownership=None, permissions=None):
320 """
321 Copies files from the source directory to the target directory.
322
323 This function is not recursive. Only the files in the directory will be
324 copied. Ownership and permissions will be left at their default values
325 if new values are not specified. The source and target directories are
326 allowed to be soft links to a directory, but besides that soft links are
327 ignored.
328
329 @note: If you have user/group as strings, call the L{util.getUidGid}
330 function to get the associated uid/gid as an ownership tuple.
331
332 @param sourceDir: Source directory
333 @type sourceDir: String representing a directory on disk
334
335 @param targetDir: Target directory
336 @type targetDir: String representing a directory on disk
337
338 @param ownership: Owner and group that the copied files should have
339 @type ownership: Tuple of numeric ids C{(uid, gid)}
340
341 @param permissions: Permissions that the staged files should have
342 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
343
344 @return: Number of files copied from the source directory to the target directory.
345
346 @raise ValueError: If source or target is not a directory or does not exist.
347 @raise ValueError: If a path cannot be encoded properly.
348 @raise IOError: If there is an IO error copying the files.
349 @raise OSError: If there is an OS error copying or changing permissions on a files
350 """
351 filesCopied = 0
352 sourceDir = encodePath(sourceDir)
353 targetDir = encodePath(targetDir)
354 for fileName in os.listdir(sourceDir):
355 sourceFile = os.path.join(sourceDir, fileName)
356 targetFile = os.path.join(targetDir, fileName)
357 LocalPeer._copyLocalFile(sourceFile, targetFile, ownership, permissions)
358 filesCopied += 1
359 return filesCopied
360
361 @staticmethod
362 - def _copyLocalFile(sourceFile=None, targetFile=None, ownership=None, permissions=None, overwrite=True):
363 """
364 Copies a source file to a target file.
365
366 If the source file is C{None} then the target file will be created or
367 overwritten as an empty file. If the target file is C{None}, this method
368 is a no-op. Attempting to copy a soft link or a directory will result in
369 an exception.
370
371 @note: If you have user/group as strings, call the L{util.getUidGid}
372 function to get the associated uid/gid as an ownership tuple.
373
374 @note: We will not overwrite a target file that exists when this method
375 is invoked. If the target already exists, we'll raise an exception.
376
377 @param sourceFile: Source file to copy
378 @type sourceFile: String representing a file on disk, as an absolute path
379
380 @param targetFile: Target file to create
381 @type targetFile: String representing a file on disk, as an absolute path
382
383 @param ownership: Owner and group that the copied should have
384 @type ownership: Tuple of numeric ids C{(uid, gid)}
385
386 @param permissions: Permissions that the staged files should have
387 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
388
389 @param overwrite: Indicates whether it's OK to overwrite the target file.
390 @type overwrite: Boolean true/false.
391
392 @raise ValueError: If the passed-in source file is not a regular file.
393 @raise ValueError: If a path cannot be encoded properly.
394 @raise IOError: If the target file already exists.
395 @raise IOError: If there is an IO error copying the file
396 @raise OSError: If there is an OS error copying or changing permissions on a file
397 """
398 targetFile = encodePath(targetFile)
399 sourceFile = encodePath(sourceFile)
400 if targetFile is None:
401 return
402 if not overwrite:
403 if os.path.exists(targetFile):
404 raise IOError("Target file [%s] already exists." % targetFile)
405 if sourceFile is None:
406 open(targetFile, "w").write("")
407 else:
408 if os.path.isfile(sourceFile) and not os.path.islink(sourceFile):
409 shutil.copy(sourceFile, targetFile)
410 else:
411 logger.debug("Source [%s] is not a regular file.", sourceFile)
412 raise ValueError("Source is not a regular file.")
413 if ownership is not None:
414 os.chown(targetFile, ownership[0], ownership[1])
415 if permissions is not None:
416 os.chmod(targetFile, permissions)
417
424
425
426
427
428
429 """
430 Backup peer representing a remote peer in a backup pool.
431
432 This is a class representing a remote (networked) peer in a backup pool.
433 Remote peers are backed up using an rcp-compatible copy command. A remote
434 peer has associated with it a name (which must be a valid hostname), a
435 collect directory, a working directory and a copy method (an rcp-compatible
436 command).
437
438 You can also set an optional local user value. This username will be used
439 as the local user for any remote copies that are required. It can only be
440 used if the root user is executing the backup. The root user will C{su} to
441 the local user and execute the remote copies as that user.
442
443 The copy method is associated with the peer and not with the actual request
444 to copy, because we can envision that each remote host might have a
445 different connect method.
446
447 The public methods other than the constructor are part of a "backup peer"
448 interface shared with the C{LocalPeer} class.
449
450 @sort: __init__, stagePeer, checkCollectIndicator, writeStageIndicator,
451 executeRemoteCommand, executeManagedAction, _getDirContents,
452 _copyRemoteDir, _copyRemoteFile, _pushLocalFile, name, collectDir,
453 remoteUser, rcpCommand, rshCommand, cbackCommand
454 """
455
456
457
458
459
460 - def __init__(self, name=None, collectDir=None, workingDir=None, remoteUser=None,
461 rcpCommand=None, localUser=None, rshCommand=None, cbackCommand=None,
462 ignoreFailureMode=None):
463 """
464 Initializes a remote backup peer.
465
466 @note: If provided, each command will eventually be parsed into a list of
467 strings suitable for passing to C{util.executeCommand} in order to avoid
468 security holes related to shell interpolation. This parsing will be
469 done by the L{util.splitCommandLine} function. See the documentation for
470 that function for some important notes about its limitations.
471
472 @param name: Name of the backup peer
473 @type name: String, must be a valid DNS hostname
474
475 @param collectDir: Path to the peer's collect directory
476 @type collectDir: String representing an absolute path on the remote peer
477
478 @param workingDir: Working directory that can be used to create temporary files, etc.
479 @type workingDir: String representing an absolute path on the current host.
480
481 @param remoteUser: Name of the Cedar Backup user on the remote peer
482 @type remoteUser: String representing a username, valid via remote shell to the peer
483
484 @param localUser: Name of the Cedar Backup user on the current host
485 @type localUser: String representing a username, valid on the current host
486
487 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
488 @type rcpCommand: String representing a system command including required arguments
489
490 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
491 @type rshCommand: String representing a system command including required arguments
492
493 @param cbackCommand: A chack-compatible command to use for executing managed actions
494 @type cbackCommand: String representing a system command including required arguments
495
496 @param ignoreFailureMode: Ignore failure mode for this peer
497 @type ignoreFailureMode: One of VALID_FAILURE_MODES
498
499 @raise ValueError: If collect directory is not an absolute path
500 """
501 self._name = None
502 self._collectDir = None
503 self._workingDir = None
504 self._remoteUser = None
505 self._localUser = None
506 self._rcpCommand = None
507 self._rcpCommandList = None
508 self._rshCommand = None
509 self._rshCommandList = None
510 self._cbackCommand = None
511 self._ignoreFailureMode = None
512 self.name = name
513 self.collectDir = collectDir
514 self.workingDir = workingDir
515 self.remoteUser = remoteUser
516 self.localUser = localUser
517 self.rcpCommand = rcpCommand
518 self.rshCommand = rshCommand
519 self.cbackCommand = cbackCommand
520 self.ignoreFailureMode = ignoreFailureMode
521
522
523
524
525
526
528 """
529 Property target used to set the peer name.
530 The value must be a non-empty string and cannot be C{None}.
531 @raise ValueError: If the value is an empty string or C{None}.
532 """
533 if value is None or len(value) < 1:
534 raise ValueError("Peer name must be a non-empty string.")
535 self._name = value
536
538 """
539 Property target used to get the peer name.
540 """
541 return self._name
542
544 """
545 Property target used to set the collect directory.
546 The value must be an absolute path and cannot be C{None}.
547 It does not have to exist on disk at the time of assignment.
548 @raise ValueError: If the value is C{None} or is not an absolute path.
549 @raise ValueError: If the value cannot be encoded properly.
550 """
551 if value is not None:
552 if not os.path.isabs(value):
553 raise ValueError("Collect directory must be an absolute path.")
554 self._collectDir = encodePath(value)
555
557 """
558 Property target used to get the collect directory.
559 """
560 return self._collectDir
561
563 """
564 Property target used to set the working directory.
565 The value must be an absolute path and cannot be C{None}.
566 @raise ValueError: If the value is C{None} or is not an absolute path.
567 @raise ValueError: If the value cannot be encoded properly.
568 """
569 if value is not None:
570 if not os.path.isabs(value):
571 raise ValueError("Working directory must be an absolute path.")
572 self._workingDir = encodePath(value)
573
575 """
576 Property target used to get the working directory.
577 """
578 return self._workingDir
579
581 """
582 Property target used to set the remote user.
583 The value must be a non-empty string and cannot be C{None}.
584 @raise ValueError: If the value is an empty string or C{None}.
585 """
586 if value is None or len(value) < 1:
587 raise ValueError("Peer remote user must be a non-empty string.")
588 self._remoteUser = value
589
591 """
592 Property target used to get the remote user.
593 """
594 return self._remoteUser
595
597 """
598 Property target used to set the local user.
599 The value must be a non-empty string if it is not C{None}.
600 @raise ValueError: If the value is an empty string.
601 """
602 if value is not None:
603 if len(value) < 1:
604 raise ValueError("Peer local user must be a non-empty string.")
605 self._localUser = value
606
608 """
609 Property target used to get the local user.
610 """
611 return self._localUser
612
614 """
615 Property target to set the rcp command.
616
617 The value must be a non-empty string or C{None}. Its value is stored in
618 the two forms: "raw" as provided by the client, and "parsed" into a list
619 suitable for being passed to L{util.executeCommand} via
620 L{util.splitCommandLine}.
621
622 However, all the caller will ever see via the property is the actual
623 value they set (which includes seeing C{None}, even if we translate that
624 internally to C{DEF_RCP_COMMAND}). Internally, we should always use
625 C{self._rcpCommandList} if we want the actual command list.
626
627 @raise ValueError: If the value is an empty string.
628 """
629 if value is None:
630 self._rcpCommand = None
631 self._rcpCommandList = DEF_RCP_COMMAND
632 else:
633 if len(value) >= 1:
634 self._rcpCommand = value
635 self._rcpCommandList = splitCommandLine(self._rcpCommand)
636 else:
637 raise ValueError("The rcp command must be a non-empty string.")
638
640 """
641 Property target used to get the rcp command.
642 """
643 return self._rcpCommand
644
646 """
647 Property target to set the rsh command.
648
649 The value must be a non-empty string or C{None}. Its value is stored in
650 the two forms: "raw" as provided by the client, and "parsed" into a list
651 suitable for being passed to L{util.executeCommand} via
652 L{util.splitCommandLine}.
653
654 However, all the caller will ever see via the property is the actual
655 value they set (which includes seeing C{None}, even if we translate that
656 internally to C{DEF_RSH_COMMAND}). Internally, we should always use
657 C{self._rshCommandList} if we want the actual command list.
658
659 @raise ValueError: If the value is an empty string.
660 """
661 if value is None:
662 self._rshCommand = None
663 self._rshCommandList = DEF_RSH_COMMAND
664 else:
665 if len(value) >= 1:
666 self._rshCommand = value
667 self._rshCommandList = splitCommandLine(self._rshCommand)
668 else:
669 raise ValueError("The rsh command must be a non-empty string.")
670
672 """
673 Property target used to get the rsh command.
674 """
675 return self._rshCommand
676
678 """
679 Property target to set the cback command.
680
681 The value must be a non-empty string or C{None}. Unlike the other
682 command, this value is only stored in the "raw" form provided by the
683 client.
684
685 @raise ValueError: If the value is an empty string.
686 """
687 if value is None:
688 self._cbackCommand = None
689 else:
690 if len(value) >= 1:
691 self._cbackCommand = value
692 else:
693 raise ValueError("The cback command must be a non-empty string.")
694
696 """
697 Property target used to get the cback command.
698 """
699 return self._cbackCommand
700
702 """
703 Property target used to set the ignoreFailure mode.
704 If not C{None}, the mode must be one of the values in L{VALID_FAILURE_MODES}.
705 @raise ValueError: If the value is not valid.
706 """
707 if value is not None:
708 if value not in VALID_FAILURE_MODES:
709 raise ValueError("Ignore failure mode must be one of %s." % VALID_FAILURE_MODES)
710 self._ignoreFailureMode = value
711
713 """
714 Property target used to get the ignoreFailure mode.
715 """
716 return self._ignoreFailureMode
717
718 name = property(_getName, _setName, None, "Name of the peer (a valid DNS hostname).")
719 collectDir = property(_getCollectDir, _setCollectDir, None, "Path to the peer's collect directory (an absolute local path).")
720 workingDir = property(_getWorkingDir, _setWorkingDir, None, "Path to the peer's working directory (an absolute local path).")
721 remoteUser = property(_getRemoteUser, _setRemoteUser, None, "Name of the Cedar Backup user on the remote peer.")
722 localUser = property(_getLocalUser, _setLocalUser, None, "Name of the Cedar Backup user on the current host.")
723 rcpCommand = property(_getRcpCommand, _setRcpCommand, None, "An rcp-compatible copy command to use for copying files.")
724 rshCommand = property(_getRshCommand, _setRshCommand, None, "An rsh-compatible command to use for remote shells to the peer.")
725 cbackCommand = property(_getCbackCommand, _setCbackCommand, None, "A chack-compatible command to use for executing managed actions.")
726 ignoreFailureMode = property(_getIgnoreFailureMode, _setIgnoreFailureMode, None, "Ignore failure mode for peer.")
727
728
729
730
731
732
733 - def stagePeer(self, targetDir, ownership=None, permissions=None):
734 """
735 Stages data from the peer into the indicated local target directory.
736
737 The target directory must already exist before this method is called. If
738 passed in, ownership and permissions will be applied to the files that
739 are copied.
740
741 @note: The returned count of copied files might be inaccurate if some of
742 the copied files already existed in the staging directory prior to the
743 copy taking place. We don't clear the staging directory first, because
744 some extension might also be using it.
745
746 @note: If you have user/group as strings, call the L{util.getUidGid} function
747 to get the associated uid/gid as an ownership tuple.
748
749 @note: Unlike the local peer version of this method, an I/O error might
750 or might not be raised if the directory is empty. Since we're using a
751 remote copy method, we just don't have the fine-grained control over our
752 exceptions that's available when we can look directly at the filesystem,
753 and we can't control whether the remote copy method thinks an empty
754 directory is an error.
755
756 @param targetDir: Target directory to write data into
757 @type targetDir: String representing a directory on disk
758
759 @param ownership: Owner and group that the staged files should have
760 @type ownership: Tuple of numeric ids C{(uid, gid)}
761
762 @param permissions: Permissions that the staged files should have
763 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
764
765 @return: Number of files copied from the source directory to the target directory.
766
767 @raise ValueError: If target directory is not a directory, does not exist or is not absolute.
768 @raise ValueError: If a path cannot be encoded properly.
769 @raise IOError: If there were no files to stage (i.e. the directory was empty)
770 @raise IOError: If there is an IO error copying a file.
771 @raise OSError: If there is an OS error copying or changing permissions on a file
772 """
773 targetDir = encodePath(targetDir)
774 if not os.path.isabs(targetDir):
775 logger.debug("Target directory [%s] not an absolute path.", targetDir)
776 raise ValueError("Target directory must be an absolute path.")
777 if not os.path.exists(targetDir) or not os.path.isdir(targetDir):
778 logger.debug("Target directory [%s] is not a directory or does not exist on disk.", targetDir)
779 raise ValueError("Target directory is not a directory or does not exist on disk.")
780 count = RemotePeer._copyRemoteDir(self.remoteUser, self.localUser, self.name,
781 self._rcpCommand, self._rcpCommandList,
782 self.collectDir, targetDir,
783 ownership, permissions)
784 if count == 0:
785 raise IOError("Did not copy any files from local peer.")
786 return count
787
789 """
790 Checks the collect indicator in the peer's staging directory.
791
792 When a peer has completed collecting its backup files, it will write an
793 empty indicator file into its collect directory. This method checks to
794 see whether that indicator has been written. If the remote copy command
795 fails, we return C{False} as if the file weren't there.
796
797 If you need to, you can override the name of the collect indicator file
798 by passing in a different name.
799
800 @note: Apparently, we can't count on all rcp-compatible implementations
801 to return sensible errors for some error conditions. As an example, the
802 C{scp} command in Debian 'woody' returns a zero (normal) status even when
803 it can't find a host or if the login or path is invalid. Because of
804 this, the implementation of this method is rather convoluted.
805
806 @param collectIndicator: Name of the collect indicator file to check
807 @type collectIndicator: String representing name of a file in the collect directory
808
809 @return: Boolean true/false depending on whether the indicator exists.
810 @raise ValueError: If a path cannot be encoded properly.
811 """
812 try:
813 if collectIndicator is None:
814 sourceFile = os.path.join(self.collectDir, DEF_COLLECT_INDICATOR)
815 targetFile = os.path.join(self.workingDir, DEF_COLLECT_INDICATOR)
816 else:
817 collectIndicator = encodePath(collectIndicator)
818 sourceFile = os.path.join(self.collectDir, collectIndicator)
819 targetFile = os.path.join(self.workingDir, collectIndicator)
820 logger.debug("Fetch remote [%s] into [%s].", sourceFile, targetFile)
821 if os.path.exists(targetFile):
822 try:
823 os.remove(targetFile)
824 except:
825 raise Exception("Error: collect indicator [%s] already exists!" % targetFile)
826 try:
827 RemotePeer._copyRemoteFile(self.remoteUser, self.localUser, self.name,
828 self._rcpCommand, self._rcpCommandList,
829 sourceFile, targetFile,
830 overwrite=False)
831 if os.path.exists(targetFile):
832 return True
833 else:
834 return False
835 except Exception, e:
836 logger.info("Failed looking for collect indicator: %s", e)
837 return False
838 finally:
839 if os.path.exists(targetFile):
840 try:
841 os.remove(targetFile)
842 except: pass
843
845 """
846 Writes the stage indicator in the peer's staging directory.
847
848 When the master has completed collecting its backup files, it will write
849 an empty indicator file into the peer's collect directory. The presence
850 of this file implies that the staging process is complete.
851
852 If you need to, you can override the name of the stage indicator file by
853 passing in a different name.
854
855 @note: If you have user/group as strings, call the L{util.getUidGid} function
856 to get the associated uid/gid as an ownership tuple.
857
858 @param stageIndicator: Name of the indicator file to write
859 @type stageIndicator: String representing name of a file in the collect directory
860
861 @raise ValueError: If a path cannot be encoded properly.
862 @raise IOError: If there is an IO error creating the file.
863 @raise OSError: If there is an OS error creating or changing permissions on the file
864 """
865 stageIndicator = encodePath(stageIndicator)
866 if stageIndicator is None:
867 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
868 targetFile = os.path.join(self.collectDir, DEF_STAGE_INDICATOR)
869 else:
870 sourceFile = os.path.join(self.workingDir, DEF_STAGE_INDICATOR)
871 targetFile = os.path.join(self.collectDir, stageIndicator)
872 try:
873 if not os.path.exists(sourceFile):
874 open(sourceFile, "w").write("")
875 RemotePeer._pushLocalFile(self.remoteUser, self.localUser, self.name,
876 self._rcpCommand, self._rcpCommandList,
877 sourceFile, targetFile)
878 finally:
879 if os.path.exists(sourceFile):
880 try:
881 os.remove(sourceFile)
882 except: pass
883
885 """
886 Executes a command on the peer via remote shell.
887
888 @param command: Command to execute
889 @type command: String command-line suitable for use with rsh.
890
891 @raise IOError: If there is an error executing the command on the remote peer.
892 """
893 RemotePeer._executeRemoteCommand(self.remoteUser, self.localUser,
894 self.name, self._rshCommand,
895 self._rshCommandList, command)
896
898 """
899 Executes a managed action on this peer.
900
901 @param action: Name of the action to execute.
902 @param fullBackup: Whether a full backup should be executed.
903
904 @raise IOError: If there is an error executing the action on the remote peer.
905 """
906 try:
907 command = RemotePeer._buildCbackCommand(self.cbackCommand, action, fullBackup)
908 self.executeRemoteCommand(command)
909 except IOError, e:
910 logger.info(e)
911 raise IOError("Failed to execute action [%s] on managed client [%s]." % (action, self.name))
912
913
914
915
916
917
918 @staticmethod
919 - def _getDirContents(path):
920 """
921 Returns the contents of a directory in terms of a Set.
922
923 The directory's contents are read as a L{FilesystemList} containing only
924 files, and then the list is converted into a set object for later use.
925
926 @param path: Directory path to get contents for
927 @type path: String representing a path on disk
928
929 @return: Set of files in the directory
930 @raise ValueError: If path is not a directory or does not exist.
931 """
932 contents = FilesystemList()
933 contents.excludeDirs = True
934 contents.excludeLinks = True
935 contents.addDirContents(path)
936 try:
937 return set(contents)
938 except:
939 import sets
940 return sets.Set(contents)
941
942 @staticmethod
943 - def _copyRemoteDir(remoteUser, localUser, remoteHost, rcpCommand, rcpCommandList,
944 sourceDir, targetDir, ownership=None, permissions=None):
945 """
946 Copies files from the source directory to the target directory.
947
948 This function is not recursive. Only the files in the directory will be
949 copied. Ownership and permissions will be left at their default values
950 if new values are not specified. Behavior when copying soft links from
951 the collect directory is dependent on the behavior of the specified rcp
952 command.
953
954 @note: The returned count of copied files might be inaccurate if some of
955 the copied files already existed in the staging directory prior to the
956 copy taking place. We don't clear the staging directory first, because
957 some extension might also be using it.
958
959 @note: If you have user/group as strings, call the L{util.getUidGid} function
960 to get the associated uid/gid as an ownership tuple.
961
962 @note: We don't have a good way of knowing exactly what files we copied
963 down from the remote peer, unless we want to parse the output of the rcp
964 command (ugh). We could change permissions on everything in the target
965 directory, but that's kind of ugly too. Instead, we use Python's set
966 functionality to figure out what files were added while we executed the
967 rcp command. This isn't perfect - for instance, it's not correct if
968 someone else is messing with the directory at the same time we're doing
969 the remote copy - but it's about as good as we're going to get.
970
971 @note: Apparently, we can't count on all rcp-compatible implementations
972 to return sensible errors for some error conditions. As an example, the
973 C{scp} command in Debian 'woody' returns a zero (normal) status even
974 when it can't find a host or if the login or path is invalid. We try
975 to work around this by issuing C{IOError} if we don't copy any files from
976 the remote host.
977
978 @param remoteUser: Name of the Cedar Backup user on the remote peer
979 @type remoteUser: String representing a username, valid via the copy command
980
981 @param localUser: Name of the Cedar Backup user on the current host
982 @type localUser: String representing a username, valid on the current host
983
984 @param remoteHost: Hostname of the remote peer
985 @type remoteHost: String representing a hostname, accessible via the copy command
986
987 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
988 @type rcpCommand: String representing a system command including required arguments
989
990 @param rcpCommandList: An rcp-compatible copy command to use for copying files
991 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
992
993 @param sourceDir: Source directory
994 @type sourceDir: String representing a directory on disk
995
996 @param targetDir: Target directory
997 @type targetDir: String representing a directory on disk
998
999 @param ownership: Owner and group that the copied files should have
1000 @type ownership: Tuple of numeric ids C{(uid, gid)}
1001
1002 @param permissions: Permissions that the staged files should have
1003 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1004
1005 @return: Number of files copied from the source directory to the target directory.
1006
1007 @raise ValueError: If source or target is not a directory or does not exist.
1008 @raise IOError: If there is an IO error copying the files.
1009 """
1010 beforeSet = RemotePeer._getDirContents(targetDir)
1011 if localUser is not None:
1012 try:
1013 if not isRunningAsRoot():
1014 raise IOError("Only root can remote copy as another user.")
1015 except AttributeError: pass
1016 actualCommand = "%s %s@%s:%s/* %s" % (rcpCommand, remoteUser, remoteHost, sourceDir, targetDir)
1017 command = resolveCommand(SU_COMMAND)
1018 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1019 if result != 0:
1020 raise IOError("Error (%d) copying files from remote host as local user [%s]." % (result, localUser))
1021 else:
1022 copySource = "%s@%s:%s/*" % (remoteUser, remoteHost, sourceDir)
1023 command = resolveCommand(rcpCommandList)
1024 result = executeCommand(command, [copySource, targetDir])[0]
1025 if result != 0:
1026 raise IOError("Error (%d) copying files from remote host." % result)
1027 afterSet = RemotePeer._getDirContents(targetDir)
1028 if len(afterSet) == 0:
1029 raise IOError("Did not copy any files from remote peer.")
1030 differenceSet = afterSet.difference(beforeSet)
1031 if len(differenceSet) == 0:
1032 raise IOError("Apparently did not copy any new files from remote peer.")
1033 for targetFile in differenceSet:
1034 if ownership is not None:
1035 os.chown(targetFile, ownership[0], ownership[1])
1036 if permissions is not None:
1037 os.chmod(targetFile, permissions)
1038 return len(differenceSet)
1039
1040 @staticmethod
1041 - def _copyRemoteFile(remoteUser, localUser, remoteHost,
1042 rcpCommand, rcpCommandList,
1043 sourceFile, targetFile, ownership=None,
1044 permissions=None, overwrite=True):
1045 """
1046 Copies a remote source file to a target file.
1047
1048 @note: Internally, we have to go through and escape any spaces in the
1049 source path with double-backslash, otherwise things get screwed up. It
1050 doesn't seem to be required in the target path. I hope this is portable
1051 to various different rcp methods, but I guess it might not be (all I have
1052 to test with is OpenSSH).
1053
1054 @note: If you have user/group as strings, call the L{util.getUidGid} function
1055 to get the associated uid/gid as an ownership tuple.
1056
1057 @note: We will not overwrite a target file that exists when this method
1058 is invoked. If the target already exists, we'll raise an exception.
1059
1060 @note: Apparently, we can't count on all rcp-compatible implementations
1061 to return sensible errors for some error conditions. As an example, the
1062 C{scp} command in Debian 'woody' returns a zero (normal) status even when
1063 it can't find a host or if the login or path is invalid. We try to work
1064 around this by issuing C{IOError} the target file does not exist when
1065 we're done.
1066
1067 @param remoteUser: Name of the Cedar Backup user on the remote peer
1068 @type remoteUser: String representing a username, valid via the copy command
1069
1070 @param remoteHost: Hostname of the remote peer
1071 @type remoteHost: String representing a hostname, accessible via the copy command
1072
1073 @param localUser: Name of the Cedar Backup user on the current host
1074 @type localUser: String representing a username, valid on the current host
1075
1076 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1077 @type rcpCommand: String representing a system command including required arguments
1078
1079 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1080 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1081
1082 @param sourceFile: Source file to copy
1083 @type sourceFile: String representing a file on disk, as an absolute path
1084
1085 @param targetFile: Target file to create
1086 @type targetFile: String representing a file on disk, as an absolute path
1087
1088 @param ownership: Owner and group that the copied should have
1089 @type ownership: Tuple of numeric ids C{(uid, gid)}
1090
1091 @param permissions: Permissions that the staged files should have
1092 @type permissions: UNIX permissions mode, specified in octal (i.e. C{0640}).
1093
1094 @param overwrite: Indicates whether it's OK to overwrite the target file.
1095 @type overwrite: Boolean true/false.
1096
1097 @raise IOError: If the target file already exists.
1098 @raise IOError: If there is an IO error copying the file
1099 @raise OSError: If there is an OS error changing permissions on the file
1100 """
1101 if not overwrite:
1102 if os.path.exists(targetFile):
1103 raise IOError("Target file [%s] already exists." % targetFile)
1104 if localUser is not None:
1105 try:
1106 if not isRunningAsRoot():
1107 raise IOError("Only root can remote copy as another user.")
1108 except AttributeError: pass
1109 actualCommand = "%s %s@%s:%s %s" % (rcpCommand, remoteUser, remoteHost, sourceFile.replace(" ", "\\ "), targetFile)
1110 command = resolveCommand(SU_COMMAND)
1111 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1112 if result != 0:
1113 raise IOError("Error (%d) copying [%s] from remote host as local user [%s]." % (result, sourceFile, localUser))
1114 else:
1115 copySource = "%s@%s:%s" % (remoteUser, remoteHost, sourceFile.replace(" ", "\\ "))
1116 command = resolveCommand(rcpCommandList)
1117 result = executeCommand(command, [copySource, targetFile])[0]
1118 if result != 0:
1119 raise IOError("Error (%d) copying [%s] from remote host." % (result, sourceFile))
1120 if not os.path.exists(targetFile):
1121 raise IOError("Apparently unable to copy file from remote host.")
1122 if ownership is not None:
1123 os.chown(targetFile, ownership[0], ownership[1])
1124 if permissions is not None:
1125 os.chmod(targetFile, permissions)
1126
1127 @staticmethod
1128 - def _pushLocalFile(remoteUser, localUser, remoteHost,
1129 rcpCommand, rcpCommandList,
1130 sourceFile, targetFile, overwrite=True):
1131 """
1132 Copies a local source file to a remote host.
1133
1134 @note: We will not overwrite a target file that exists when this method
1135 is invoked. If the target already exists, we'll raise an exception.
1136
1137 @note: Internally, we have to go through and escape any spaces in the
1138 source and target paths with double-backslash, otherwise things get
1139 screwed up. I hope this is portable to various different rcp methods,
1140 but I guess it might not be (all I have to test with is OpenSSH).
1141
1142 @note: If you have user/group as strings, call the L{util.getUidGid} function
1143 to get the associated uid/gid as an ownership tuple.
1144
1145 @param remoteUser: Name of the Cedar Backup user on the remote peer
1146 @type remoteUser: String representing a username, valid via the copy command
1147
1148 @param localUser: Name of the Cedar Backup user on the current host
1149 @type localUser: String representing a username, valid on the current host
1150
1151 @param remoteHost: Hostname of the remote peer
1152 @type remoteHost: String representing a hostname, accessible via the copy command
1153
1154 @param rcpCommand: An rcp-compatible copy command to use for copying files from the peer
1155 @type rcpCommand: String representing a system command including required arguments
1156
1157 @param rcpCommandList: An rcp-compatible copy command to use for copying files
1158 @type rcpCommandList: Command as a list to be passed to L{util.executeCommand}
1159
1160 @param sourceFile: Source file to copy
1161 @type sourceFile: String representing a file on disk, as an absolute path
1162
1163 @param targetFile: Target file to create
1164 @type targetFile: String representing a file on disk, as an absolute path
1165
1166 @param overwrite: Indicates whether it's OK to overwrite the target file.
1167 @type overwrite: Boolean true/false.
1168
1169 @raise IOError: If there is an IO error copying the file
1170 @raise OSError: If there is an OS error changing permissions on the file
1171 """
1172 if not overwrite:
1173 if os.path.exists(targetFile):
1174 raise IOError("Target file [%s] already exists." % targetFile)
1175 if localUser is not None:
1176 try:
1177 if not isRunningAsRoot():
1178 raise IOError("Only root can remote copy as another user.")
1179 except AttributeError: pass
1180 actualCommand = '%s "%s" "%s@%s:%s"' % (rcpCommand, sourceFile, remoteUser, remoteHost, targetFile)
1181 command = resolveCommand(SU_COMMAND)
1182 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1183 if result != 0:
1184 raise IOError("Error (%d) copying [%s] to remote host as local user [%s]." % (result, sourceFile, localUser))
1185 else:
1186 copyTarget = "%s@%s:%s" % (remoteUser, remoteHost, targetFile.replace(" ", "\\ "))
1187 command = resolveCommand(rcpCommandList)
1188 result = executeCommand(command, [sourceFile.replace(" ", "\\ "), copyTarget])[0]
1189 if result != 0:
1190 raise IOError("Error (%d) copying [%s] to remote host." % (result, sourceFile))
1191
1192 @staticmethod
1193 - def _executeRemoteCommand(remoteUser, localUser, remoteHost, rshCommand, rshCommandList, remoteCommand):
1194 """
1195 Executes a command on the peer via remote shell.
1196
1197 @param remoteUser: Name of the Cedar Backup user on the remote peer
1198 @type remoteUser: String representing a username, valid on the remote host
1199
1200 @param localUser: Name of the Cedar Backup user on the current host
1201 @type localUser: String representing a username, valid on the current host
1202
1203 @param remoteHost: Hostname of the remote peer
1204 @type remoteHost: String representing a hostname, accessible via the copy command
1205
1206 @param rshCommand: An rsh-compatible copy command to use for remote shells to the peer
1207 @type rshCommand: String representing a system command including required arguments
1208
1209 @param rshCommandList: An rsh-compatible copy command to use for remote shells to the peer
1210 @type rshCommandList: Command as a list to be passed to L{util.executeCommand}
1211
1212 @param remoteCommand: The command to be executed on the remote host
1213 @type remoteCommand: String command-line, with no special shell characters ($, <, etc.)
1214
1215 @raise IOError: If there is an error executing the remote command
1216 """
1217 actualCommand = "%s %s@%s '%s'" % (rshCommand, remoteUser, remoteHost, remoteCommand)
1218 if localUser is not None:
1219 try:
1220 if not isRunningAsRoot():
1221 raise IOError("Only root can remote shell as another user.")
1222 except AttributeError: pass
1223 command = resolveCommand(SU_COMMAND)
1224 result = executeCommand(command, [localUser, "-c", actualCommand])[0]
1225 if result != 0:
1226 raise IOError("Command failed [su -c %s \"%s\"]" % (localUser, actualCommand))
1227 else:
1228 command = resolveCommand(rshCommandList)
1229 result = executeCommand(command, ["%s@%s" % (remoteUser, remoteHost), "%s" % remoteCommand])[0]
1230 if result != 0:
1231 raise IOError("Command failed [%s]" % (actualCommand))
1232
1233 @staticmethod
1235 """
1236 Builds a Cedar Backup command line for the named action.
1237
1238 @note: If the cback command is None, then DEF_CBACK_COMMAND is used.
1239
1240 @param cbackCommand: cback command to execute, including required options
1241 @param action: Name of the action to execute.
1242 @param fullBackup: Whether a full backup should be executed.
1243
1244 @return: String suitable for passing to L{_executeRemoteCommand} as remoteCommand.
1245 @raise ValueError: If action is None.
1246 """
1247 if action is None:
1248 raise ValueError("Action cannot be None.")
1249 if cbackCommand is None:
1250 cbackCommand = DEF_CBACK_COMMAND
1251 if fullBackup:
1252 return "%s --full %s" % (cbackCommand, action)
1253 else:
1254 return "%s %s" % (cbackCommand, action)
1255