001    /*
002     * CDDL HEADER START
003     *
004     * The contents of this file are subject to the terms of the
005     * Common Development and Distribution License, Version 1.0 only
006     * (the "License").  You may not use this file except in compliance
007     * with the License.
008     *
009     * You can obtain a copy of the license at
010     * trunk/opends/resource/legal-notices/OpenDS.LICENSE
011     * or https://OpenDS.dev.java.net/OpenDS.LICENSE.
012     * See the License for the specific language governing permissions
013     * and limitations under the License.
014     *
015     * When distributing Covered Code, include this CDDL HEADER in each
016     * file and include the License file at
017     * trunk/opends/resource/legal-notices/OpenDS.LICENSE.  If applicable,
018     * add the following below this CDDL HEADER, with the fields enclosed
019     * by brackets "[]" replaced with your own identifying information:
020     *      Portions Copyright [yyyy] [name of copyright owner]
021     *
022     * CDDL HEADER END
023     *
024     *
025     *      Copyright 2006-2008 Sun Microsystems, Inc.
026     */
027    package org.opends.server.backends.jeb;
028    import org.opends.messages.Message;
029    
030    import org.opends.server.config.ConfigException;
031    import org.opends.server.core.DirectoryServer;
032    import org.opends.server.util.DynamicConstants;
033    import org.opends.server.types.CryptoManagerException;
034    
035    import javax.crypto.Mac;
036    import java.io.BufferedReader;
037    import java.io.File;
038    import java.io.FileInputStream;
039    import java.io.FileNotFoundException;
040    import java.io.FileOutputStream;
041    import java.io.FilenameFilter;
042    import java.io.InputStream;
043    import java.io.InputStreamReader;
044    import java.io.IOException;
045    import java.io.OutputStream;
046    import java.io.OutputStreamWriter;
047    import java.io.Writer;
048    import java.security.MessageDigest;
049    import java.util.ArrayList;
050    import java.util.Arrays;
051    import java.util.Collections;
052    import java.util.Date;
053    import java.util.HashMap;
054    import java.util.HashSet;
055    import java.util.List;
056    import java.util.Set;
057    import java.util.zip.Deflater;
058    import java.util.zip.ZipEntry;
059    import java.util.zip.ZipInputStream;
060    import java.util.zip.ZipOutputStream;
061    
062    import org.opends.server.types.*;
063    import static org.opends.server.loggers.ErrorLogger.logError;
064    import static org.opends.server.loggers.debug.DebugLogger.*;
065    import org.opends.server.loggers.debug.DebugTracer;
066    import static org.opends.messages.JebMessages.*;
067    import static org.opends.server.util.ServerConstants.*;
068    import static org.opends.server.util.StaticUtils.*;
069    
070    
071    
072    /**
073     * A backup manager for JE backends.
074     */
075    public class BackupManager
076    {
077      /**
078       * The tracer object for the debug logger.
079       */
080      private static final DebugTracer TRACER = getTracer();
081    
082      /**
083       * The common prefix for archive files.
084       */
085      public static final String BACKUP_BASE_FILENAME = "backup-";
086    
087      /**
088       * The name of the property that holds the name of the latest log file
089       * at the time the backup was created.
090       */
091      public static final String PROPERTY_LAST_LOGFILE_NAME = "last_logfile_name";
092    
093      /**
094       * The name of the property that holds the size of the latest log file
095       * at the time the backup was created.
096       */
097      public static final String PROPERTY_LAST_LOGFILE_SIZE = "last_logfile_size";
098    
099    
100      /**
101       * The name of the entry in an incremental backup archive file
102       * containing a list of log files that are unchanged since the
103       * previous backup.
104       */
105      public static final String ZIPENTRY_UNCHANGED_LOGFILES = "unchanged.txt";
106    
107      /**
108       * The name of a dummy entry in the backup archive file that will act
109       * as a placeholder in case a backup is done on an empty backend.
110       */
111      public static final String ZIPENTRY_EMPTY_PLACEHOLDER = "empty.placeholder";
112    
113    
114      /**
115       * The backend ID.
116       */
117      private String backendID;
118    
119    
120      /**
121       * Construct a backup manager for a JE backend.
122       * @param backendID The ID of the backend instance for which a backup
123       * manager is required.
124       */
125      public BackupManager(String backendID)
126      {
127        this.backendID   = backendID;
128      }
129    
130      /**
131       * Create a backup of the JE backend.  The backup is stored in a single zip
132       * file in the backup directory.  If the backup is incremental, then the
133       * first entry in the zip is a text file containing a list of all the JE
134       * log files that are unchanged since the previous backup.  The remaining
135       * zip entries are the JE log files themselves, which, for an incremental,
136       * only include those files that have changed.
137       * @param backendDir The directory of the backend instance for
138       * which the backup is required.
139       * @param  backupConfig  The configuration to use when performing the backup.
140       * @throws DirectoryException If a Directory Server error occurs.
141       */
142      public void createBackup(File backendDir, BackupConfig backupConfig)
143           throws DirectoryException
144      {
145        // Get the properties to use for the backup.
146        String          backupID        = backupConfig.getBackupID();
147        BackupDirectory backupDir       = backupConfig.getBackupDirectory();
148        boolean         incremental     = backupConfig.isIncremental();
149        String          incrBaseID      = backupConfig.getIncrementalBaseID();
150        boolean         compress        = backupConfig.compressData();
151        boolean         encrypt         = backupConfig.encryptData();
152        boolean         hash            = backupConfig.hashData();
153        boolean         signHash        = backupConfig.signHash();
154    
155    
156        // Create a hash map that will hold the extra backup property information
157        // for this backup.
158        HashMap<String,String> backupProperties = new HashMap<String,String>();
159    
160    
161        // Get the crypto manager and use it to obtain references to the message
162        // digest and/or MAC to use for hashing and/or signing.
163        CryptoManager cryptoManager   = DirectoryServer.getCryptoManager();
164        Mac           mac             = null;
165        MessageDigest digest          = null;
166        String        digestAlgorithm = null;
167        String        macKeyID    = null;
168    
169        if (hash)
170        {
171          if (signHash)
172          {
173            try
174            {
175              macKeyID = cryptoManager.getMacEngineKeyEntryID();
176              backupProperties.put(BACKUP_PROPERTY_MAC_KEY_ID, macKeyID);
177    
178              mac = cryptoManager.getMacEngine(macKeyID);
179            }
180            catch (Exception e)
181            {
182              if (debugEnabled())
183              {
184                TRACER.debugCaught(DebugLogLevel.ERROR, e);
185              }
186    
187              Message message = ERR_JEB_BACKUP_CANNOT_GET_MAC.get(
188                  macKeyID, stackTraceToSingleLineString(e));
189              throw new DirectoryException(
190                   DirectoryServer.getServerErrorResultCode(), message, e);
191            }
192          }
193          else
194          {
195            digestAlgorithm = cryptoManager.getPreferredMessageDigestAlgorithm();
196            backupProperties.put(BACKUP_PROPERTY_DIGEST_ALGORITHM, digestAlgorithm);
197    
198            try
199            {
200              digest = cryptoManager.getPreferredMessageDigest();
201            }
202            catch (Exception e)
203            {
204              if (debugEnabled())
205              {
206                TRACER.debugCaught(DebugLogLevel.ERROR, e);
207              }
208    
209              Message message = ERR_JEB_BACKUP_CANNOT_GET_DIGEST.get(
210                  digestAlgorithm, stackTraceToSingleLineString(e));
211              throw new DirectoryException(
212                   DirectoryServer.getServerErrorResultCode(), message, e);
213            }
214          }
215        }
216    
217    
218        // Date the backup.
219        Date backupDate = new Date();
220    
221        // If this is an incremental, determine the base backup for this backup.
222        HashSet<String> dependencies = new HashSet<String>();
223        BackupInfo baseBackup = null;
224    /*
225        FilenameFilter backupTagFilter = new FilenameFilter()
226        {
227          public boolean accept(File dir, String name)
228          {
229            return name.startsWith(BackupInfo.PROPERTY_BACKUP_ID);
230          }
231        };
232    */
233        if (incremental)
234        {
235          if (incrBaseID == null)
236          {
237            // The default is to use the latest backup as base.
238            if (backupDir.getLatestBackup() != null)
239            {
240              incrBaseID = backupDir.getLatestBackup().getBackupID();
241            }
242          }
243    
244          // Get the set of possible base backups from the current database.
245    /*
246          String[] files = backendDir.list(backupTagFilter);
247          if (files == null || files.length == 0)
248          {
249            // Incremental not allowed until after a full.
250            Message msg = ERR_JEB_INCR_BACKUP_REQUIRES_FULL.get();
251            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
252                                         msg);
253          }
254          HashSet<String> backups = new HashSet<String>();
255          int prefixLen = BackupInfo.PROPERTY_BACKUP_ID.length()+1;
256          for (String s : files)
257          {
258            String actualBaseID = s.substring(prefixLen);
259            backups.add(actualBaseID);
260          }
261    
262          // Check that it makes sense to do this incremental.
263          if (incrBaseID == null || !backups.contains(incrBaseID))
264          {
265            Message msg =
266                ERR_JEB_INCR_BACKUP_FROM_WRONG_BASE.get(backups.toString());
267            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
268                                         msg);
269          }
270    */
271    
272          baseBackup = getBackupInfo(backupDir, incrBaseID);
273        }
274    
275        // Get information about the latest log file from the base backup.
276        String latestFileName = null;
277        long latestFileSize = 0;
278        if (baseBackup != null)
279        {
280          HashMap<String,String> properties = baseBackup.getBackupProperties();
281          latestFileName = properties.get(PROPERTY_LAST_LOGFILE_NAME);
282          latestFileSize = Long.parseLong(
283               properties.get(PROPERTY_LAST_LOGFILE_SIZE));
284        }
285    
286        // Create an output stream that will be used to write the archive file.  At
287        // its core, it will be a file output stream to put a file on the disk.  If
288        // we are to encrypt the data, then that file output stream will be wrapped
289        // in a cipher output stream.  The resulting output stream will then be
290        // wrapped by a zip output stream (which may or may not actually use
291        // compression).
292        String archiveFilename = null;
293        OutputStream outputStream;
294        File archiveFile;
295        try
296        {
297          archiveFilename = BACKUP_BASE_FILENAME + backendID + "-" + backupID;
298          archiveFile = new File(backupDir.getPath(), archiveFilename);
299          if (archiveFile.exists())
300          {
301            int i=1;
302            while (true)
303            {
304              archiveFile = new File(backupDir.getPath(),
305                                     archiveFilename  + "." + i);
306              if (archiveFile.exists())
307              {
308                i++;
309              }
310              else
311              {
312                archiveFilename = archiveFilename + "." + i;
313                break;
314              }
315            }
316          }
317    
318          outputStream = new FileOutputStream(archiveFile, false);
319          backupProperties.put(BACKUP_PROPERTY_ARCHIVE_FILENAME, archiveFilename);
320        }
321        catch (Exception e)
322        {
323          if (debugEnabled())
324          {
325            TRACER.debugCaught(DebugLogLevel.ERROR, e);
326          }
327    
328          Message message = ERR_JEB_BACKUP_CANNOT_CREATE_ARCHIVE_FILE.
329              get(String.valueOf(archiveFilename), backupDir.getPath(),
330                  stackTraceToSingleLineString(e));
331          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
332                                       message, e);
333        }
334    
335    
336        // If we should encrypt the data, then wrap the output stream in a cipher
337        // output stream.
338        if (encrypt)
339        {
340          try
341          {
342            outputStream
343                    = cryptoManager.getCipherOutputStream(outputStream);
344          }
345          catch (CryptoManagerException e)
346          {
347            if (debugEnabled())
348            {
349              TRACER.debugCaught(DebugLogLevel.ERROR, e);
350            }
351    
352            Message message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
353                    stackTraceToSingleLineString(e));
354            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
355                                         message, e);
356          }
357        }
358    
359    
360        // Wrap the file output stream in a zip output stream.
361        ZipOutputStream zipStream = new ZipOutputStream(outputStream);
362    
363        Message message = ERR_JEB_BACKUP_ZIP_COMMENT.get(
364                DynamicConstants.PRODUCT_NAME,
365                backupID, backendID);
366        zipStream.setComment(message.toString());
367    
368        if (compress)
369        {
370          zipStream.setLevel(Deflater.DEFAULT_COMPRESSION);
371        }
372        else
373        {
374          zipStream.setLevel(Deflater.NO_COMPRESSION);
375        }
376    
377        // Record this backup in the database itself.
378    /*
379        String backupTag = BackupInfo.PROPERTY_BACKUP_ID + "_" + backupID;
380        File tagFile = new File(backendDir, backupTag);
381        try
382        {
383          tagFile.createNewFile();
384        }
385        catch (IOException e)
386        {
387          assert debugException(CLASS_NAME, "createBackup", e);
388          Message msg = ERR_JEB_CANNOT_CREATE_BACKUP_TAG_FILE.get(
389              backupTag, backendDir.getPath());
390          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
391                                       msg);
392        }
393    */
394    
395        // Get a list of all the log files comprising the database.
396        FilenameFilter filenameFilter = new FilenameFilter()
397        {
398          public boolean accept(File d, String name)
399          {
400            return name.endsWith(".jdb");
401          }
402        };
403    
404        File[] logFiles;
405        try
406        {
407          logFiles = backendDir.listFiles(filenameFilter);
408        }
409        catch (Exception e)
410        {
411          if (debugEnabled())
412          {
413            TRACER.debugCaught(DebugLogLevel.ERROR, e);
414          }
415    
416          message = ERR_JEB_BACKUP_CANNOT_LIST_LOG_FILES.get(
417              backendDir.getAbsolutePath(), stackTraceToSingleLineString(e));
418          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
419              message, e);
420        }
421    
422        // Check to see if backend is empty. If so, insert placeholder entry into
423        // archive
424        if(logFiles.length <= 0)
425        {
426          try
427          {
428            ZipEntry emptyPlaceholder = new ZipEntry(ZIPENTRY_EMPTY_PLACEHOLDER);
429            zipStream.putNextEntry(emptyPlaceholder);
430          }
431          catch (IOException e)
432          {
433            if (debugEnabled())
434            {
435              TRACER.debugCaught(DebugLogLevel.ERROR, e);
436            }
437            message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
438                ZIPENTRY_EMPTY_PLACEHOLDER, stackTraceToSingleLineString(e));
439            throw new DirectoryException(
440                DirectoryServer.getServerErrorResultCode(), message, e);
441          }
442        }
443    
444        // Sort the log files from oldest to youngest since this is the order
445        // in which they must be copied.
446        // This is easy since the files are created in alphabetical order by JE.
447        Arrays.sort(logFiles);
448    
449        try
450        {
451          // Archive the backup tag files.
452    /*
453          File[] tagFiles = backendDir.listFiles(backupTagFilter);
454          if (tagFiles != null)
455          {
456            for (File f : tagFiles)
457            {
458              try
459              {
460                archiveFile(zipStream, mac, digest, f);
461              }
462              catch (IOException e)
463              {
464                assert debugException(CLASS_NAME, "createBackup", e);
465                Message message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
466                    backupTag, stackTraceToSingleLineString(e));
467                throw new DirectoryException(
468                     DirectoryServer.getServerErrorResultCode(), message, e);
469              }
470            }
471          }
472    */
473    
474          // Process log files that are unchanged from the base backup.
475          int indexCurrent = 0;
476          if (latestFileName != null)
477          {
478            ArrayList<String> unchangedList = new ArrayList<String>();
479            while (indexCurrent < logFiles.length &&
480                    !backupConfig.isCancelled())
481            {
482              File logFile = logFiles[indexCurrent];
483              String logFileName = logFile.getName();
484    
485              // Stop when we get to the first log file that has been
486              // written since the base backup.
487              int compareResult = logFileName.compareTo(latestFileName);
488              if (compareResult > 0 ||
489                   (compareResult == 0 && logFile.length() != latestFileSize))
490              {
491                break;
492              }
493    
494              message = NOTE_JEB_BACKUP_FILE_UNCHANGED.get(logFileName);
495              logError(message);
496    
497              unchangedList.add(logFileName);
498    
499              indexCurrent++;
500            }
501    
502            // Write a file containing the list of unchanged log files.
503            if (!unchangedList.isEmpty())
504            {
505              String zipEntryName = ZIPENTRY_UNCHANGED_LOGFILES;
506              try
507              {
508                archiveList(zipStream, mac, digest, zipEntryName, unchangedList);
509              }
510              catch (IOException e)
511              {
512                if (debugEnabled())
513                {
514                  TRACER.debugCaught(DebugLogLevel.ERROR, e);
515                }
516                message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
517                    zipEntryName, stackTraceToSingleLineString(e));
518                throw new DirectoryException(
519                     DirectoryServer.getServerErrorResultCode(), message, e);
520              }
521    
522              // Set the dependency.
523              dependencies.add(baseBackup.getBackupID());
524            }
525          }
526    
527          // Write the new log files to the zip file.
528          do
529          {
530            boolean deletedFiles = false;
531    
532            while (indexCurrent < logFiles.length &&
533                    !backupConfig.isCancelled())
534            {
535              File logFile = logFiles[indexCurrent];
536    
537              try
538              {
539                latestFileSize = archiveFile(zipStream, mac, digest,
540                                             logFile, backupConfig);
541                latestFileName = logFile.getName();
542              }
543              catch (FileNotFoundException e)
544              {
545                if (debugEnabled())
546                {
547                  TRACER.debugCaught(DebugLogLevel.ERROR, e);
548                }
549    
550                // A log file has been deleted by the cleaner since we started.
551                deletedFiles = true;
552              }
553              catch (IOException e)
554              {
555                if (debugEnabled())
556                {
557                  TRACER.debugCaught(DebugLogLevel.ERROR, e);
558                }
559                message = ERR_JEB_BACKUP_CANNOT_WRITE_ARCHIVE_FILE.get(
560                    logFile.getName(), stackTraceToSingleLineString(e));
561                throw new DirectoryException(
562                     DirectoryServer.getServerErrorResultCode(), message, e);
563              }
564    
565              indexCurrent++;
566            }
567    
568            if (deletedFiles)
569            {
570              // The cleaner is active and has deleted one or more of the log files
571              // since we started.  The in-use data from those log files will have
572              // been written to new log files, so we must include those new files.
573              final String latest = logFiles[logFiles.length-1].getName();
574              final long latestSize = latestFileSize;
575              FilenameFilter filter = new FilenameFilter()
576              {
577                public boolean accept(File d, String name)
578                {
579                  if (!name.endsWith(".jdb")) return false;
580                  int compareTo = name.compareTo(latest);
581                  if (compareTo > 0) return true;
582                  if (compareTo == 0 && d.length() > latestSize) return true;
583                  return false;
584                }
585              };
586    
587              try
588              {
589                logFiles = backendDir.listFiles(filter);
590                indexCurrent = 0;
591              }
592              catch (Exception e)
593              {
594                if (debugEnabled())
595                {
596                  TRACER.debugCaught(DebugLogLevel.ERROR, e);
597                }
598    
599                message = ERR_JEB_BACKUP_CANNOT_LIST_LOG_FILES.get(
600                    backendDir.getAbsolutePath(), stackTraceToSingleLineString(e));
601                throw new DirectoryException(
602                     DirectoryServer.getServerErrorResultCode(), message, e);
603              }
604    
605              if (logFiles == null)
606              {
607                break;
608              }
609    
610              Arrays.sort(logFiles);
611    
612              message = NOTE_JEB_BACKUP_CLEANER_ACTIVITY.get(
613                      String.valueOf(logFiles.length));
614              logError(message);
615            }
616            else
617            {
618              // We are done.
619              break;
620            }
621          }
622          while (true);
623    
624        }
625        catch (DirectoryException e)
626        {
627          if (debugEnabled())
628          {
629            TRACER.debugCaught(DebugLogLevel.ERROR, e);
630          }
631    
632          try
633          {
634            zipStream.close();
635          } catch (Exception e2) {}
636        }
637    
638        // We're done writing the file, so close the zip stream (which should also
639        // close the underlying stream).
640        try
641        {
642          zipStream.close();
643        }
644        catch (Exception e)
645        {
646          if (debugEnabled())
647          {
648            TRACER.debugCaught(DebugLogLevel.ERROR, e);
649          }
650    
651          message = ERR_JEB_BACKUP_CANNOT_CLOSE_ZIP_STREAM.
652              get(archiveFilename, backupDir.getPath(),
653                  stackTraceToSingleLineString(e));
654          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
655                                       message, e);
656        }
657    
658    
659        // Get the digest or MAC bytes if appropriate.
660        byte[] digestBytes = null;
661        byte[] macBytes    = null;
662        if (hash)
663        {
664          if (signHash)
665          {
666            macBytes = mac.doFinal();
667          }
668          else
669          {
670            digestBytes = digest.digest();
671          }
672        }
673    
674    
675        // Create a descriptor for this backup.
676        backupProperties.put(PROPERTY_LAST_LOGFILE_NAME, latestFileName);
677        backupProperties.put(PROPERTY_LAST_LOGFILE_SIZE,
678                             String.valueOf(latestFileSize));
679        BackupInfo backupInfo = new BackupInfo(backupDir, backupID,
680                                               backupDate, incremental, compress,
681                                               encrypt, digestBytes, macBytes,
682                                               dependencies, backupProperties);
683    
684        try
685        {
686          backupDir.addBackup(backupInfo);
687          backupDir.writeBackupDirectoryDescriptor();
688        }
689        catch (Exception e)
690        {
691          if (debugEnabled())
692          {
693            TRACER.debugCaught(DebugLogLevel.ERROR, e);
694          }
695    
696          message = ERR_JEB_BACKUP_CANNOT_UPDATE_BACKUP_DESCRIPTOR.get(
697              backupDir.getDescriptorPath(), stackTraceToSingleLineString(e));
698          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
699                                       message, e);
700        }
701    
702        // Remove the backup if this operation was cancelled since the
703        // backup may be incomplete
704        if (backupConfig.isCancelled())
705        {
706          removeBackup(backupDir, backupID);
707        }
708      }
709    
710    
711    
712      /**
713       * Restore a JE backend from backup, or verify the backup.
714       * @param backendDir The configuration of the backend instance to be
715       * restored.
716       * @param  restoreConfig The configuration to use when performing the restore.
717       * @throws DirectoryException If a Directory Server error occurs.
718       */
719      public void restoreBackup(File backendDir,
720                                RestoreConfig restoreConfig)
721           throws DirectoryException
722      {
723        // Get the properties to use for the restore.
724        String          backupID        = restoreConfig.getBackupID();
725        BackupDirectory backupDir       = restoreConfig.getBackupDirectory();
726        boolean         verifyOnly      = restoreConfig.verifyOnly();
727    
728        BackupInfo backupInfo = getBackupInfo(backupDir, backupID);
729    
730        // Create a restore directory with a different name to the backend
731        // directory.
732        File restoreDir = new File(backendDir.getPath() + "-restore-" + backupID);
733        if (!verifyOnly)
734        {
735          File[] files = restoreDir.listFiles();
736          if (files != null)
737          {
738            for (File f : files)
739            {
740              f.delete();
741            }
742          }
743          restoreDir.mkdir();
744        }
745    
746        // Get the set of restore files that are in dependencies.
747        Set<String> includeFiles;
748        try
749        {
750          includeFiles = getUnchanged(backupDir, backupInfo);
751        }
752        catch (IOException e)
753        {
754          if (debugEnabled())
755          {
756            TRACER.debugCaught(DebugLogLevel.ERROR, e);
757          }
758          Message message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
759              backupInfo.getBackupID(), stackTraceToSingleLineString(e));
760          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
761                                       message, e);
762        }
763    
764        // Restore any dependencies.
765        List<BackupInfo> dependents = getDependents(backupDir, backupInfo);
766        for (BackupInfo dependent : dependents)
767        {
768          try
769          {
770            restoreArchive(restoreDir, restoreConfig, dependent, includeFiles);
771          }
772          catch (IOException e)
773          {
774            if (debugEnabled())
775            {
776              TRACER.debugCaught(DebugLogLevel.ERROR, e);
777            }
778            Message message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
779                dependent.getBackupID(), stackTraceToSingleLineString(e));
780            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
781                                         message, e);
782          }
783        }
784    
785        // Restore the final archive file.
786        try
787        {
788          restoreArchive(restoreDir, restoreConfig, backupInfo, null);
789        }
790        catch (IOException e)
791        {
792          if (debugEnabled())
793          {
794            TRACER.debugCaught(DebugLogLevel.ERROR, e);
795          }
796          Message message = ERR_JEB_BACKUP_CANNOT_RESTORE.get(
797              backupInfo.getBackupID(), stackTraceToSingleLineString(e));
798          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
799                                       message, e);
800        }
801    
802        // Delete the current backend directory and rename the restore directory.
803        if (!verifyOnly)
804        {
805          File[] files = backendDir.listFiles();
806          if (files != null)
807          {
808            for (File f : files)
809            {
810              f.delete();
811            }
812          }
813          backendDir.delete();
814          if (!restoreDir.renameTo(backendDir))
815          {
816            Message msg = ERR_JEB_CANNOT_RENAME_RESTORE_DIRECTORY.get(
817                restoreDir.getPath(), backendDir.getPath());
818            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
819                                         msg);
820          }
821        }
822      }
823    
824      /**
825       * Removes the specified backup if it is possible to do so.
826       *
827       * @param  backupDir  The backup directory structure with which the
828       *                          specified backup is associated.
829       * @param  backupID         The backup ID for the backup to be removed.
830       *
831       * @throws  DirectoryException  If it is not possible to remove the specified
832       *                              backup for some reason (e.g., no such backup
833       *                              exists or there are other backups that are
834       *                              dependent upon it).
835       */
836      public void removeBackup(BackupDirectory backupDir,
837                               String backupID)
838             throws DirectoryException
839      {
840        BackupInfo backupInfo = getBackupInfo(backupDir, backupID);
841        HashMap<String,String> backupProperties = backupInfo.getBackupProperties();
842    
843        String archiveFilename =
844             backupProperties.get(BACKUP_PROPERTY_ARCHIVE_FILENAME);
845        File archiveFile = new File(backupDir.getPath(), archiveFilename);
846    
847        try
848        {
849          backupDir.removeBackup(backupID);
850        }
851        catch (ConfigException e)
852        {
853          if (debugEnabled())
854          {
855            TRACER.debugCaught(DebugLogLevel.ERROR, e);
856          }
857          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
858                                       e.getMessageObject());
859        }
860    
861        try
862        {
863          backupDir.writeBackupDirectoryDescriptor();
864        }
865        catch (Exception e)
866        {
867          if (debugEnabled())
868          {
869            TRACER.debugCaught(DebugLogLevel.ERROR, e);
870          }
871    
872          Message message = ERR_JEB_BACKUP_CANNOT_UPDATE_BACKUP_DESCRIPTOR.get(
873              backupDir.getDescriptorPath(), stackTraceToSingleLineString(e));
874          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
875                                       message, e);
876        }
877    
878        // Remove the archive file.
879        archiveFile.delete();
880    
881      }
882    
883    
884    
885      /**
886       * Restore the contents of an archive file.  If the archive is being
887       * restored as a dependency, then only files in the specified set
888       * are restored, and the restored files are removed from the set.  Otherwise
889       * all files from the archive are restored, and files that are to be found
890       * in dependencies are added to the set.
891       *
892       * @param restoreDir     The directory in which files are to be restored.
893       * @param restoreConfig  The restore configuration.
894       * @param backupInfo     The backup containing the files to be restored.
895       * @param includeFiles   The set of files to be restored.  If null, then
896       *                       all files are restored.
897       * @throws DirectoryException If a Directory Server error occurs.
898       * @throws IOException   If an I/O exception occurs during the restore.
899       */
900      private void restoreArchive(File restoreDir,
901                                  RestoreConfig restoreConfig,
902                                  BackupInfo backupInfo,
903                                  Set<String> includeFiles)
904           throws DirectoryException,IOException
905      {
906        BackupDirectory backupDir       = restoreConfig.getBackupDirectory();
907        boolean verifyOnly              = restoreConfig.verifyOnly();
908    
909        String          backupID        = backupInfo.getBackupID();
910        boolean         encrypt         = backupInfo.isEncrypted();
911        byte[]          hash            = backupInfo.getUnsignedHash();
912        byte[]          signHash        = backupInfo.getSignedHash();
913    
914        HashMap<String,String> backupProperties = backupInfo.getBackupProperties();
915    
916        String archiveFilename =
917             backupProperties.get(BACKUP_PROPERTY_ARCHIVE_FILENAME);
918        File archiveFile = new File(backupDir.getPath(), archiveFilename);
919    
920        InputStream inputStream = new FileInputStream(archiveFile);
921    
922        // Get the crypto manager and use it to obtain references to the message
923        // digest and/or MAC to use for hashing and/or signing.
924        CryptoManager cryptoManager   = DirectoryServer.getCryptoManager();
925        Mac           mac             = null;
926        MessageDigest digest          = null;
927        String        digestAlgorithm = null;
928        String        macKeyID        = null;
929    
930        if (signHash != null)
931        {
932          macKeyID = backupProperties.get(BACKUP_PROPERTY_MAC_KEY_ID);
933    
934          try
935          {
936            mac = cryptoManager.getMacEngine(macKeyID);
937          }
938          catch (Exception e)
939          {
940            if (debugEnabled())
941            {
942              TRACER.debugCaught(DebugLogLevel.ERROR, e);
943            }
944    
945            Message message = ERR_JEB_BACKUP_CANNOT_GET_MAC.get(
946                macKeyID, stackTraceToSingleLineString(e));
947            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
948                                         message, e);
949          }
950        }
951    
952        if (hash != null)
953        {
954          digestAlgorithm = backupProperties.get(BACKUP_PROPERTY_DIGEST_ALGORITHM);
955    
956          try
957          {
958            digest = cryptoManager.getMessageDigest(digestAlgorithm);
959          }
960          catch (Exception e)
961          {
962            if (debugEnabled())
963            {
964              TRACER.debugCaught(DebugLogLevel.ERROR, e);
965            }
966    
967            Message message = ERR_JEB_BACKUP_CANNOT_GET_DIGEST.get(
968                digestAlgorithm, stackTraceToSingleLineString(e));
969            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
970                                         message, e);
971          }
972        }
973    
974    
975        // If the data is encrypted, then wrap the input stream in a cipher
976        // input stream.
977        if (encrypt)
978        {
979          try
980          {
981            inputStream = cryptoManager.getCipherInputStream(inputStream);
982          }
983          catch (CryptoManagerException e)
984          {
985            if (debugEnabled())
986            {
987              TRACER.debugCaught(DebugLogLevel.ERROR, e);
988            }
989    
990            Message message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
991                stackTraceToSingleLineString(e));
992            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
993                                         message, e);
994          }
995        }
996    
997    
998        // Wrap the file input stream in a zip input stream.
999        ZipInputStream zipStream = new ZipInputStream(inputStream);
1000    
1001        // Iterate through the entries in the zip file.
1002        ZipEntry zipEntry = zipStream.getNextEntry();
1003        while (zipEntry != null && !restoreConfig.isCancelled())
1004        {
1005          String name = zipEntry.getName();
1006    
1007          if (name.equals(ZIPENTRY_EMPTY_PLACEHOLDER))
1008          {
1009            // This entry is treated specially to indicate a backup of an empty
1010            // backend was attempted.
1011    
1012            zipEntry = zipStream.getNextEntry();
1013            continue;
1014          }
1015    
1016          if (name.equals(ZIPENTRY_UNCHANGED_LOGFILES))
1017          {
1018            // This entry is treated specially. It is never restored,
1019            // and its hash is computed on the strings, not the bytes.
1020            if (mac != null || digest != null)
1021            {
1022              // The file name is part of the hash.
1023              if (mac != null)
1024              {
1025                mac.update(getBytes(name));
1026              }
1027    
1028              if (digest != null)
1029              {
1030                digest.update(getBytes(name));
1031              }
1032    
1033              InputStreamReader reader = new InputStreamReader(zipStream);
1034              BufferedReader bufferedReader = new BufferedReader(reader);
1035              String line = bufferedReader.readLine();
1036              while (line != null)
1037              {
1038                if (mac != null)
1039                {
1040                  mac.update(getBytes(line));
1041                }
1042    
1043                if (digest != null)
1044                {
1045                  digest.update(getBytes(line));
1046                }
1047    
1048                line = bufferedReader.readLine();
1049              }
1050            }
1051    
1052            zipEntry = zipStream.getNextEntry();
1053            continue;
1054          }
1055    
1056          // See if we need to restore the file.
1057          File file = new File(restoreDir, name);
1058          OutputStream outputStream = null;
1059          if (includeFiles == null || includeFiles.contains(zipEntry.getName()))
1060          {
1061            if (!verifyOnly)
1062            {
1063              outputStream = new FileOutputStream(file);
1064            }
1065          }
1066    
1067          if (outputStream != null || mac != null || digest != null)
1068          {
1069            if (verifyOnly)
1070            {
1071              Message message = NOTE_JEB_BACKUP_VERIFY_FILE.get(zipEntry.getName());
1072              logError(message);
1073            }
1074    
1075            // The file name is part of the hash.
1076            if (mac != null)
1077            {
1078              mac.update(getBytes(name));
1079            }
1080    
1081            if (digest != null)
1082            {
1083              digest.update(getBytes(name));
1084            }
1085    
1086            // Process the file.
1087            long totalBytesRead = 0;
1088            byte[] buffer = new byte[8192];
1089            int bytesRead = zipStream.read(buffer);
1090            while (bytesRead > 0 && !restoreConfig.isCancelled())
1091            {
1092              totalBytesRead += bytesRead;
1093    
1094              if (mac != null)
1095              {
1096                mac.update(buffer, 0, bytesRead);
1097              }
1098    
1099              if (digest != null)
1100              {
1101                digest.update(buffer, 0, bytesRead);
1102              }
1103    
1104              if (outputStream != null)
1105              {
1106                outputStream.write(buffer, 0, bytesRead);
1107              }
1108    
1109              bytesRead = zipStream.read(buffer);
1110            }
1111    
1112            if (outputStream != null)
1113            {
1114              outputStream.close();
1115    
1116              Message message = NOTE_JEB_BACKUP_RESTORED_FILE.get(
1117                  zipEntry.getName(), totalBytesRead);
1118              logError(message);
1119            }
1120          }
1121    
1122          zipEntry = zipStream.getNextEntry();
1123        }
1124    
1125        zipStream.close();
1126    
1127        // Check the hash.
1128        if (digest != null)
1129        {
1130          if (!Arrays.equals(digest.digest(), hash))
1131          {
1132            Message message = ERR_JEB_BACKUP_UNSIGNED_HASH_ERROR.get(backupID);
1133            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1134                                         message);
1135          }
1136        }
1137    
1138        if (mac != null)
1139        {
1140          byte[] computedSignHash = mac.doFinal();
1141    
1142          if (!Arrays.equals(computedSignHash, signHash))
1143          {
1144            Message message = ERR_JEB_BACKUP_SIGNED_HASH_ERROR.get(backupID);
1145            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1146                                         message);
1147          }
1148        }
1149      }
1150    
1151    
1152    
1153      /**
1154       * Writes a file to an entry in the archive file.
1155       * @param zipStream The zip output stream to which the file is to be
1156       *                  written.
1157       * @param mac A message authentication code to be updated, if not null.
1158       * @param digest A message digest to be updated, if not null.
1159       * @param file The file to be written.
1160       * @return The number of bytes written from the file.
1161       * @throws FileNotFoundException If the file to be archived does not exist.
1162       * @throws IOException If an I/O error occurs while archiving the file.
1163       */
1164      private long archiveFile(ZipOutputStream zipStream,
1165                               Mac mac, MessageDigest digest, File file,
1166                               BackupConfig backupConfig)
1167           throws IOException, FileNotFoundException
1168      {
1169        ZipEntry zipEntry = new ZipEntry(file.getName());
1170    
1171        // Open the file for reading.
1172        InputStream inputStream = new FileInputStream(file);
1173    
1174        // Start the zip entry.
1175        zipStream.putNextEntry(zipEntry);
1176    
1177        // Put the name in the hash.
1178        if (mac != null)
1179        {
1180          mac.update(getBytes(file.getName()));
1181        }
1182    
1183        if (digest != null)
1184        {
1185          digest.update(getBytes(file.getName()));
1186        }
1187    
1188        // Write the file.
1189        long totalBytesRead = 0;
1190        byte[] buffer = new byte[8192];
1191        int bytesRead = inputStream.read(buffer);
1192        while (bytesRead > 0 && !backupConfig.isCancelled())
1193        {
1194          if (mac != null)
1195          {
1196            mac.update(buffer, 0, bytesRead);
1197          }
1198    
1199          if (digest != null)
1200          {
1201            digest.update(buffer, 0, bytesRead);
1202          }
1203    
1204          zipStream.write(buffer, 0, bytesRead);
1205          totalBytesRead += bytesRead;
1206          bytesRead = inputStream.read(buffer);
1207        }
1208        inputStream.close();
1209    
1210        // Finish the zip entry.
1211        zipStream.closeEntry();
1212    
1213        Message message = NOTE_JEB_BACKUP_ARCHIVED_FILE.get(zipEntry.getName());
1214        logError(message);
1215    
1216        return totalBytesRead;
1217      }
1218    
1219      /**
1220       * Write a list of strings to an entry in the archive file.
1221       * @param zipStream The zip output stream to which the entry is to be
1222       *                  written.
1223       * @param mac An optional MAC to be updated.
1224       * @param digest An optional message digest to be updated.
1225       * @param fileName The name of the zip entry to be written.
1226       * @param list A list of strings to be written.  The strings must not
1227       *             contain newlines.
1228       * @throws IOException If an I/O error occurs while writing the archive entry.
1229       */
1230      private void archiveList(ZipOutputStream zipStream,
1231                               Mac mac, MessageDigest digest, String fileName,
1232                               List<String> list)
1233           throws IOException
1234      {
1235        ZipEntry zipEntry = new ZipEntry(fileName);
1236    
1237        // Start the zip entry.
1238        zipStream.putNextEntry(zipEntry);
1239    
1240        // Put the name in the hash.
1241        if (mac != null)
1242        {
1243          mac.update(getBytes(fileName));
1244        }
1245    
1246        if (digest != null)
1247        {
1248          digest.update(getBytes(fileName));
1249        }
1250    
1251        Writer writer = new OutputStreamWriter(zipStream);
1252        for (String s : list)
1253        {
1254          if (mac != null)
1255          {
1256            mac.update(getBytes(s));
1257          }
1258    
1259          if (digest != null)
1260          {
1261            digest.update(getBytes(s));
1262          }
1263    
1264          writer.write(s);
1265          writer.write(EOL);
1266        }
1267        writer.flush();
1268    
1269        // Finish the zip entry.
1270        zipStream.closeEntry();
1271      }
1272    
1273      /**
1274       * Obtains the set of files in a backup that are unchanged from its
1275       * dependent backup or backups.  This list is stored as the first entry
1276       * in the archive file.
1277       * @param backupDir The backup directory.
1278       * @param backupInfo The backup info.
1279       * @return The set of files that were unchanged.
1280       * @throws DirectoryException If an error occurs while trying to get the
1281       * appropriate cipher algorithm for an encrypted backup.
1282       * @throws IOException If an I/O error occurs while reading the backup
1283       * archive file.
1284       */
1285      private Set<String> getUnchanged(BackupDirectory backupDir,
1286                                       BackupInfo backupInfo)
1287           throws DirectoryException, IOException
1288      {
1289        HashSet<String> hashSet = new HashSet<String>();
1290    
1291        boolean         encrypt         = backupInfo.isEncrypted();
1292    
1293        HashMap<String,String> backupProperties = backupInfo.getBackupProperties();
1294    
1295        String archiveFilename =
1296             backupProperties.get(BACKUP_PROPERTY_ARCHIVE_FILENAME);
1297        File archiveFile = new File(backupDir.getPath(), archiveFilename);
1298    
1299        InputStream inputStream = new FileInputStream(archiveFile);
1300    
1301        // Get the crypto manager and use it to obtain references to the message
1302        // digest and/or MAC to use for hashing and/or signing.
1303        CryptoManager cryptoManager   = DirectoryServer.getCryptoManager();
1304    
1305        // If the data is encrypted, then wrap the input stream in a cipher
1306        // input stream.
1307        if (encrypt)
1308        {
1309          try
1310          {
1311            inputStream = cryptoManager.getCipherInputStream(inputStream);
1312          }
1313          catch (CryptoManagerException e)
1314          {
1315            if (debugEnabled())
1316            {
1317              TRACER.debugCaught(DebugLogLevel.ERROR, e);
1318            }
1319    
1320            Message message = ERR_JEB_BACKUP_CANNOT_GET_CIPHER.get(
1321                    stackTraceToSingleLineString(e));
1322            throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1323                                         message, e);
1324          }
1325        }
1326    
1327    
1328        // Wrap the file input stream in a zip input stream.
1329        ZipInputStream zipStream = new ZipInputStream(inputStream);
1330    
1331        // Iterate through the entries in the zip file.
1332        ZipEntry zipEntry = zipStream.getNextEntry();
1333        while (zipEntry != null)
1334        {
1335          // We are looking for the entry containing the list of unchanged files.
1336          if (zipEntry.getName().equals(ZIPENTRY_UNCHANGED_LOGFILES))
1337          {
1338            InputStreamReader reader = new InputStreamReader(zipStream);
1339            BufferedReader bufferedReader = new BufferedReader(reader);
1340            String line = bufferedReader.readLine();
1341            while (line != null)
1342            {
1343              hashSet.add(line);
1344              line = bufferedReader.readLine();
1345            }
1346            break;
1347          }
1348    
1349          zipEntry = zipStream.getNextEntry();
1350        }
1351    
1352        zipStream.close();
1353        return hashSet;
1354      }
1355    
1356      /**
1357       * Obtains a list of the dependencies of a given backup in order from
1358       * the oldest (the full backup), to the most recent.
1359       * @param backupDir The backup directory.
1360       * @param backupInfo The backup for which dependencies are required.
1361       * @return A list of dependent backups.
1362       * @throws DirectoryException If a Directory Server error occurs.
1363       */
1364      private ArrayList<BackupInfo> getDependents(BackupDirectory backupDir,
1365                                                  BackupInfo backupInfo)
1366           throws DirectoryException
1367      {
1368        ArrayList<BackupInfo> dependents = new ArrayList<BackupInfo>();
1369        while (backupInfo != null && !backupInfo.getDependencies().isEmpty())
1370        {
1371          String backupID = backupInfo.getDependencies().iterator().next();
1372          backupInfo = getBackupInfo(backupDir, backupID);
1373          if (backupInfo != null)
1374          {
1375            dependents.add(backupInfo);
1376          }
1377        }
1378        Collections.reverse(dependents);
1379        return dependents;
1380      }
1381    
1382      /**
1383       * Get the information for a given backup ID from the backup directory.
1384       * @param backupDir The backup directory.
1385       * @param backupID The backup ID.
1386       * @return The backup information, never null.
1387       * @throws DirectoryException If the backup information cannot be found.
1388       */
1389      private BackupInfo getBackupInfo(BackupDirectory backupDir,
1390                                       String backupID) throws DirectoryException
1391      {
1392        BackupInfo backupInfo = backupDir.getBackupInfo(backupID);
1393        if (backupInfo == null)
1394        {
1395          Message message =
1396              ERR_JEB_BACKUP_MISSING_BACKUPID.get(backupDir.getPath(), backupID);
1397          throw new DirectoryException(DirectoryServer.getServerErrorResultCode(),
1398                                       message);
1399        }
1400        return backupInfo;
1401      }
1402    }