001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2015-2017 ForgeRock AS.
015 */
016package org.opends.server.backends.jeb;
017
018import static com.sleepycat.je.EnvironmentConfig.*;
019import static com.sleepycat.je.LockMode.READ_COMMITTED;
020import static com.sleepycat.je.LockMode.RMW;
021import static com.sleepycat.je.OperationStatus.*;
022import static org.forgerock.util.Utils.*;
023import static org.opends.messages.BackendMessages.*;
024import static org.opends.messages.UtilityMessages.*;
025import static org.opends.server.backends.pluggable.spi.StorageUtils.*;
026import static org.opends.server.util.BackupManager.getFiles;
027import static org.opends.server.util.StaticUtils.*;
028
029import java.io.File;
030import java.io.FileFilter;
031import java.io.IOException;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.util.ArrayList;
035import java.util.HashMap;
036import java.util.HashSet;
037import java.util.List;
038import java.util.ListIterator;
039import java.util.Map;
040import java.util.NoSuchElementException;
041import java.util.Objects;
042import java.util.Set;
043import java.util.SortedSet;
044import java.util.TreeSet;
045import java.util.concurrent.ConcurrentHashMap;
046import java.util.concurrent.ConcurrentMap;
047import java.util.concurrent.TimeUnit;
048
049import org.forgerock.i18n.LocalizableMessage;
050import org.forgerock.i18n.slf4j.LocalizedLogger;
051import org.forgerock.opendj.config.server.ConfigChangeResult;
052import org.forgerock.opendj.config.server.ConfigException;
053import org.forgerock.opendj.ldap.ByteSequence;
054import org.forgerock.opendj.ldap.ByteString;
055import org.forgerock.util.Reject;
056import org.forgerock.opendj.config.server.ConfigurationChangeListener;
057import org.forgerock.opendj.server.config.server.JEBackendCfg;
058import org.opends.server.api.Backupable;
059import org.opends.server.api.DiskSpaceMonitorHandler;
060import org.opends.server.backends.pluggable.spi.AccessMode;
061import org.opends.server.backends.pluggable.spi.Cursor;
062import org.opends.server.backends.pluggable.spi.Importer;
063import org.opends.server.backends.pluggable.spi.ReadOnlyStorageException;
064import org.opends.server.backends.pluggable.spi.ReadOperation;
065import org.opends.server.backends.pluggable.spi.SequentialCursor;
066import org.opends.server.backends.pluggable.spi.Storage;
067import org.opends.server.backends.pluggable.spi.StorageRuntimeException;
068import org.opends.server.backends.pluggable.spi.StorageStatus;
069import org.opends.server.backends.pluggable.spi.StorageUtils;
070import org.opends.server.backends.pluggable.spi.TreeName;
071import org.opends.server.backends.pluggable.spi.UpdateFunction;
072import org.opends.server.backends.pluggable.spi.WriteOperation;
073import org.opends.server.backends.pluggable.spi.WriteableTransaction;
074import org.opends.server.core.DirectoryServer;
075import org.opends.server.core.MemoryQuota;
076import org.opends.server.core.ServerContext;
077import org.opends.server.extensions.DiskSpaceMonitor;
078import org.opends.server.types.BackupConfig;
079import org.opends.server.types.BackupDirectory;
080import org.opends.server.types.DirectoryException;
081import org.opends.server.types.RestoreConfig;
082import org.opends.server.util.BackupManager;
083
084import com.sleepycat.je.CursorConfig;
085import com.sleepycat.je.Database;
086import com.sleepycat.je.DatabaseConfig;
087import com.sleepycat.je.DatabaseEntry;
088import com.sleepycat.je.DatabaseException;
089import com.sleepycat.je.DatabaseNotFoundException;
090import com.sleepycat.je.Durability;
091import com.sleepycat.je.Environment;
092import com.sleepycat.je.EnvironmentConfig;
093import com.sleepycat.je.OperationStatus;
094import com.sleepycat.je.Transaction;
095import com.sleepycat.je.TransactionConfig;
096
097/** Berkeley DB Java Edition (JE for short) database implementation of the {@link Storage} engine. */
098public final class JEStorage implements Storage, Backupable, ConfigurationChangeListener<JEBackendCfg>,
099    DiskSpaceMonitorHandler
100{
101  /** JE implementation of the {@link Cursor} interface. */
102  private static final class CursorImpl implements Cursor<ByteString, ByteString>
103  {
104    private ByteString currentKey;
105    private ByteString currentValue;
106    private boolean isDefined;
107    private final com.sleepycat.je.Cursor cursor;
108    private final DatabaseEntry dbKey = new DatabaseEntry();
109    private final DatabaseEntry dbValue = new DatabaseEntry();
110
111    private CursorImpl(com.sleepycat.je.Cursor cursor)
112    {
113      this.cursor = cursor;
114    }
115
116    @Override
117    public void close()
118    {
119      closeSilently(cursor);
120    }
121
122    @Override
123    public boolean isDefined()
124    {
125      return isDefined;
126    }
127
128    @Override
129    public ByteString getKey()
130    {
131      if (currentKey == null)
132      {
133        throwIfNotSuccess();
134        currentKey = ByteString.wrap(dbKey.getData());
135      }
136      return currentKey;
137    }
138
139    @Override
140    public ByteString getValue()
141    {
142      if (currentValue == null)
143      {
144        throwIfNotSuccess();
145        currentValue = ByteString.wrap(dbValue.getData());
146      }
147      return currentValue;
148    }
149
150    @Override
151    public boolean next()
152    {
153      clearCurrentKeyAndValue();
154      try
155      {
156        isDefined = cursor.getNext(dbKey, dbValue, null) == SUCCESS;
157        return isDefined;
158      }
159      catch (DatabaseException e)
160      {
161        throw new StorageRuntimeException(e);
162      }
163    }
164
165    @Override
166    public void delete() throws NoSuchElementException, UnsupportedOperationException
167    {
168      throwIfNotSuccess();
169      try
170      {
171        cursor.delete();
172      }
173      catch (DatabaseException e)
174      {
175        throw new StorageRuntimeException(e);
176      }
177    }
178
179    @Override
180    public boolean positionToKey(final ByteSequence key)
181    {
182      clearCurrentKeyAndValue();
183      setData(dbKey, key);
184      try
185      {
186        isDefined = cursor.getSearchKey(dbKey, dbValue, null) == SUCCESS;
187        return isDefined;
188      }
189      catch (DatabaseException e)
190      {
191        throw new StorageRuntimeException(e);
192      }
193    }
194
195    @Override
196    public boolean positionToKeyOrNext(final ByteSequence key)
197    {
198      clearCurrentKeyAndValue();
199      setData(dbKey, key);
200      try
201      {
202        isDefined = cursor.getSearchKeyRange(dbKey, dbValue, null) == SUCCESS;
203        return isDefined;
204      }
205      catch (DatabaseException e)
206      {
207        throw new StorageRuntimeException(e);
208      }
209    }
210
211    @Override
212    public boolean positionToIndex(int index)
213    {
214      clearCurrentKeyAndValue();
215      try
216      {
217        isDefined = cursor.getFirst(dbKey, dbValue, null) == SUCCESS;
218        if (!isDefined)
219        {
220          return false;
221        }
222        else if (index == 0)
223        {
224          return true;
225        }
226
227        // equivalent to READ_UNCOMMITTED
228        long skipped = cursor.skipNext(index, dbKey, dbValue, null);
229        if (skipped == index)
230        {
231          isDefined = cursor.getCurrent(dbKey, dbValue, null) == SUCCESS;
232        }
233        else
234        {
235          isDefined = false;
236        }
237        return isDefined;
238      }
239      catch (DatabaseException e)
240      {
241        throw new StorageRuntimeException(e);
242      }
243    }
244
245    @Override
246    public boolean positionToLastKey()
247    {
248      clearCurrentKeyAndValue();
249      try
250      {
251        isDefined = cursor.getLast(dbKey, dbValue, null) == SUCCESS;
252        return isDefined;
253      }
254      catch (DatabaseException e)
255      {
256        throw new StorageRuntimeException(e);
257      }
258    }
259
260    private void clearCurrentKeyAndValue()
261    {
262      currentKey = null;
263      currentValue = null;
264    }
265
266    private void throwIfNotSuccess()
267    {
268      if (!isDefined())
269      {
270        throw new NoSuchElementException();
271      }
272    }
273  }
274
275  /** JE implementation of the {@link Importer} interface. */
276  private final class ImporterImpl implements Importer
277  {
278    private final Map<TreeName, Database> trees = new HashMap<>();
279
280    private Database getOrOpenTree(TreeName treeName)
281    {
282      return getOrOpenTree0(trees, treeName);
283    }
284
285    @Override
286    public void put(final TreeName treeName, final ByteSequence key, final ByteSequence value)
287    {
288      try
289      {
290        getOrOpenTree(treeName).put(null, db(key), db(value));
291      }
292      catch (DatabaseException e)
293      {
294        throw new StorageRuntimeException(e);
295      }
296    }
297
298    @Override
299    public ByteString read(final TreeName treeName, final ByteSequence key)
300    {
301      try
302      {
303        DatabaseEntry dbValue = new DatabaseEntry();
304        boolean isDefined = getOrOpenTree(treeName).get(null, db(key), dbValue, null) == SUCCESS;
305        return valueToBytes(dbValue, isDefined);
306      }
307      catch (DatabaseException e)
308      {
309        throw new StorageRuntimeException(e);
310      }
311    }
312
313    @Override
314    public SequentialCursor<ByteString, ByteString> openCursor(TreeName treeName)
315    {
316      try
317      {
318        return new CursorImpl(getOrOpenTree(treeName).openCursor(null, new CursorConfig()));
319      }
320      catch (DatabaseException e)
321      {
322        throw new StorageRuntimeException(e);
323      }
324    }
325
326    @Override
327    public void clearTree(TreeName treeName)
328    {
329      env.truncateDatabase(null, toDatabaseName(treeName), false);
330    }
331
332    @Override
333    public void close()
334    {
335      closeSilently(trees.values());
336      trees.clear();
337      JEStorage.this.close();
338    }
339  }
340
341  /** JE implementation of the {@link WriteableTransaction} interface. */
342  private final class WriteableTransactionImpl implements WriteableTransaction
343  {
344    private final Transaction txn;
345
346    private WriteableTransactionImpl(Transaction txn)
347    {
348      this.txn = txn;
349    }
350
351    /**
352     * This is currently needed for import-ldif:
353     * <ol>
354     * <li>Opening the EntryContainer calls {@link #openTree(TreeName, boolean)} for each index</li>
355     * <li>Then the underlying storage is closed</li>
356     * <li>Then {@link Importer#startImport()} is called</li>
357     * <li>Then ID2Entry#put() is called</li>
358     * <li>Which in turn calls ID2Entry#encodeEntry()</li>
359     * <li>Which in turn finally calls PersistentCompressedSchema#store()</li>
360     * <li>Which uses a reference to the storage (that was closed before calling startImport()) and
361     * uses it as if it was open</li>
362     * </ol>
363     */
364    private Database getOrOpenTree(TreeName treeName)
365    {
366      try
367      {
368        return getOrOpenTree0(trees, treeName);
369      }
370      catch (Exception e)
371      {
372        throw new StorageRuntimeException(e);
373      }
374    }
375
376    @Override
377    public void put(final TreeName treeName, final ByteSequence key, final ByteSequence value)
378    {
379      try
380      {
381        final OperationStatus status = getOrOpenTree(treeName).put(txn, db(key), db(value));
382        if (status != SUCCESS)
383        {
384          throw new StorageRuntimeException(putErrorMsg(treeName, key, value, "did not succeed: " + status));
385        }
386      }
387      catch (DatabaseException e)
388      {
389        throw new StorageRuntimeException(putErrorMsg(treeName, key, value, "threw an exception"), e);
390      }
391    }
392
393    private String putErrorMsg(TreeName treeName, ByteSequence key, ByteSequence value, String msg)
394    {
395      return "put(treeName=" + treeName + ", key=" + key + ", value=" + value + ") " + msg;
396    }
397
398    @Override
399    public boolean delete(final TreeName treeName, final ByteSequence key)
400    {
401      try
402      {
403        return getOrOpenTree(treeName).delete(txn, db(key)) == SUCCESS;
404      }
405      catch (DatabaseException e)
406      {
407        throw new StorageRuntimeException(deleteErrorMsg(treeName, key, "threw an exception"), e);
408      }
409    }
410
411    private String deleteErrorMsg(TreeName treeName, ByteSequence key, String msg)
412    {
413      return "delete(treeName=" + treeName + ", key=" + key + ") " + msg;
414    }
415
416    @Override
417    public long getRecordCount(TreeName treeName)
418    {
419      try
420      {
421        return getOrOpenTree(treeName).count();
422      }
423      catch (DatabaseException e)
424      {
425        throw new StorageRuntimeException(e);
426      }
427    }
428
429    @Override
430    public Cursor<ByteString, ByteString> openCursor(final TreeName treeName)
431    {
432      try
433      {
434        return new CursorImpl(getOrOpenTree(treeName).openCursor(txn, CursorConfig.READ_COMMITTED));
435      }
436      catch (DatabaseException e)
437      {
438        throw new StorageRuntimeException(e);
439      }
440    }
441
442    @Override
443    public ByteString read(final TreeName treeName, final ByteSequence key)
444    {
445      try
446      {
447        DatabaseEntry dbValue = new DatabaseEntry();
448        boolean isDefined = getOrOpenTree(treeName).get(txn, db(key), dbValue, READ_COMMITTED) == SUCCESS;
449        return valueToBytes(dbValue, isDefined);
450      }
451      catch (DatabaseException e)
452      {
453        throw new StorageRuntimeException(e);
454      }
455    }
456
457    @Override
458    public boolean update(final TreeName treeName, final ByteSequence key, final UpdateFunction f)
459    {
460      try
461      {
462        final Database tree = getOrOpenTree(treeName);
463        final DatabaseEntry dbKey = db(key);
464        final DatabaseEntry dbValue = new DatabaseEntry();
465        for (;;)
466        {
467          final boolean isDefined = tree.get(txn, dbKey, dbValue, RMW) == SUCCESS;
468          final ByteSequence oldValue = valueToBytes(dbValue, isDefined);
469          final ByteSequence newValue = f.computeNewValue(oldValue);
470          if (Objects.equals(newValue, oldValue))
471          {
472            return false;
473          }
474          if (newValue == null)
475          {
476            return tree.delete(txn, dbKey) == SUCCESS;
477          }
478          setData(dbValue, newValue);
479          if (isDefined)
480          {
481            return tree.put(txn, dbKey, dbValue) == SUCCESS;
482          }
483          else if (tree.putNoOverwrite(txn, dbKey, dbValue) == SUCCESS)
484          {
485            return true;
486          }
487          // else retry due to phantom read: another thread inserted a record
488        }
489      }
490      catch (DatabaseException e)
491      {
492        throw new StorageRuntimeException(e);
493      }
494    }
495
496    @Override
497    public void openTree(final TreeName treeName, boolean createOnDemand)
498    {
499      getOrOpenTree(treeName);
500    }
501
502    @Override
503    public void deleteTree(final TreeName treeName)
504    {
505      try
506      {
507        synchronized (trees)
508        {
509          closeSilently(trees.remove(treeName));
510          env.removeDatabase(txn, toDatabaseName(treeName));
511        }
512      }
513      catch (DatabaseNotFoundException e)
514      {
515        // This is fine: end result is what we wanted
516      }
517      catch (DatabaseException e)
518      {
519        throw new StorageRuntimeException(e);
520      }
521    }
522  }
523
524  /** JE read-only implementation of {@link StorageImpl} interface. */
525  private final class ReadOnlyTransactionImpl implements WriteableTransaction
526  {
527    private final WriteableTransactionImpl delegate;
528
529    ReadOnlyTransactionImpl(WriteableTransactionImpl delegate)
530    {
531      this.delegate = delegate;
532    }
533
534    @Override
535    public ByteString read(TreeName treeName, ByteSequence key)
536    {
537      return delegate.read(treeName, key);
538    }
539
540    @Override
541    public Cursor<ByteString, ByteString> openCursor(TreeName treeName)
542    {
543      return delegate.openCursor(treeName);
544    }
545
546    @Override
547    public long getRecordCount(TreeName treeName)
548    {
549      return delegate.getRecordCount(treeName);
550    }
551
552    @Override
553    public void openTree(TreeName treeName, boolean createOnDemand)
554    {
555      if (createOnDemand)
556      {
557        throw new ReadOnlyStorageException();
558      }
559      delegate.openTree(treeName, false);
560    }
561
562    @Override
563    public void deleteTree(TreeName name)
564    {
565      throw new ReadOnlyStorageException();
566    }
567
568    @Override
569    public void put(TreeName treeName, ByteSequence key, ByteSequence value)
570    {
571      throw new ReadOnlyStorageException();
572    }
573
574    @Override
575    public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f)
576    {
577      throw new ReadOnlyStorageException();
578    }
579
580    @Override
581    public boolean delete(TreeName treeName, ByteSequence key)
582    {
583      throw new ReadOnlyStorageException();
584    }
585  }
586
587  private WriteableTransaction newWriteableTransaction(Transaction txn)
588  {
589    final WriteableTransactionImpl writeableStorage = new WriteableTransactionImpl(txn);
590    return accessMode.isWriteable() ? writeableStorage : new ReadOnlyTransactionImpl(writeableStorage);
591  }
592
593  private static final int IMPORT_DB_CACHE_SIZE = 32 * MB;
594
595  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
596
597  /** Use read committed isolation instead of the default which is repeatable read. */
598  private static final TransactionConfig TXN_READ_COMMITTED = new TransactionConfig().setReadCommitted(true);
599
600  private final ServerContext serverContext;
601  private final File backendDirectory;
602  private JEBackendCfg config;
603  private AccessMode accessMode;
604
605  private Environment env;
606  private EnvironmentConfig envConfig;
607  private MemoryQuota memQuota;
608  private JEMonitor monitor;
609  private DiskSpaceMonitor diskMonitor;
610  private StorageStatus storageStatus = StorageStatus.working();
611  private final ConcurrentMap<TreeName, Database> trees = new ConcurrentHashMap<>();
612
613  /**
614   * Creates a new JE storage with the provided configuration.
615   *
616   * @param cfg
617   *          The configuration.
618   * @param serverContext
619   *          This server instance context
620   * @throws ConfigException
621   *           if memory cannot be reserved
622   */
623  JEStorage(final JEBackendCfg cfg, ServerContext serverContext) throws ConfigException
624  {
625    this.serverContext = serverContext;
626    backendDirectory = getBackendDirectory(cfg);
627    config = cfg;
628    cfg.addJEChangeListener(this);
629  }
630
631  private Database getOrOpenTree0(Map<TreeName, Database> trees, TreeName treeName)
632  {
633    Database tree = trees.get(treeName);
634    if (tree == null)
635    {
636      synchronized (trees)
637      {
638        tree = trees.get(treeName);
639        if (tree == null)
640        {
641          tree = env.openDatabase(null, toDatabaseName(treeName), dbConfig());
642          trees.put(treeName, tree);
643        }
644      }
645    }
646    return tree;
647  }
648
649  private void buildConfiguration(AccessMode accessMode, boolean isImport) throws ConfigException
650  {
651    this.accessMode = accessMode;
652
653    if (isImport)
654    {
655      envConfig = new EnvironmentConfig();
656      envConfig
657        .setTransactional(false)
658        .setAllowCreate(true)
659        .setLockTimeout(0, TimeUnit.SECONDS)
660        .setTxnTimeout(0, TimeUnit.SECONDS)
661        .setCacheSize(IMPORT_DB_CACHE_SIZE)
662        .setDurability(Durability.COMMIT_NO_SYNC)
663        .setConfigParam(CLEANER_MIN_UTILIZATION, String.valueOf(config.getDBCleanerMinUtilization()))
664        .setConfigParam(LOG_FILE_MAX, String.valueOf(config.getDBLogFileMax()));
665    }
666    else
667    {
668      envConfig = ConfigurableEnvironment.parseConfigEntry(config);
669    }
670
671    diskMonitor = serverContext.getDiskSpaceMonitor();
672    memQuota = serverContext.getMemoryQuota();
673    if (config.getDBCacheSize() > 0)
674    {
675      memQuota.acquireMemory(config.getDBCacheSize());
676    }
677    else
678    {
679      memQuota.acquireMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
680    }
681  }
682
683  private DatabaseConfig dbConfig()
684  {
685    boolean isImport = !envConfig.getTransactional();
686    return new DatabaseConfig()
687      .setKeyPrefixing(true)
688      .setAllowCreate(true)
689      .setTransactional(!isImport)
690      .setDeferredWrite(isImport);
691  }
692
693  @Override
694  public void close()
695  {
696    synchronized (trees)
697    {
698      closeSilently(trees.values());
699      trees.clear();
700    }
701
702    if (env != null)
703    {
704      DirectoryServer.deregisterMonitorProvider(monitor);
705      monitor = null;
706      try
707      {
708        env.close();
709        env = null;
710      }
711      catch (DatabaseException e)
712      {
713        throw new IllegalStateException(e);
714      }
715    }
716
717    if (config.getDBCacheSize() > 0)
718    {
719      memQuota.releaseMemory(config.getDBCacheSize());
720    }
721    else
722    {
723      memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent()));
724    }
725    config.removeJEChangeListener(this);
726    diskMonitor.deregisterMonitoredDirectory(getDirectory(), this);
727  }
728
729  @Override
730  public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException
731  {
732    Reject.ifNull(accessMode, "accessMode must not be null");
733    buildConfiguration(accessMode, false);
734    open0();
735  }
736
737  private void open0() throws ConfigException
738  {
739    setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn());
740    try
741    {
742      if (env != null)
743      {
744        throw new IllegalStateException(
745            "Database is already open, either the backend is enabled or an import is currently running.");
746      }
747      env = new Environment(backendDirectory, envConfig);
748      monitor = new JEMonitor(config.getBackendId() + " JE Database", env);
749      DirectoryServer.registerMonitorProvider(monitor);
750    }
751    catch (DatabaseException e)
752    {
753      throw new StorageRuntimeException(e);
754    }
755    registerMonitoredDirectory(config);
756  }
757
758  @Override
759  public <T> T read(final ReadOperation<T> operation) throws Exception
760  {
761    try
762    {
763      return operation.run(newWriteableTransaction(null));
764    }
765    catch (final StorageRuntimeException e)
766    {
767      if (e.getCause() != null)
768      {
769        throw (Exception) e.getCause();
770      }
771      throw e;
772    }
773  }
774
775  @Override
776  public Importer startImport() throws ConfigException, StorageRuntimeException
777  {
778    buildConfiguration(AccessMode.READ_WRITE, true);
779    open0();
780    return new ImporterImpl();
781  }
782
783  private static String toDatabaseName(final TreeName treeName)
784  {
785    return treeName.toString();
786  }
787
788  @Override
789  public void write(final WriteOperation operation) throws Exception
790  {
791    final Transaction txn = beginTransaction();
792    try
793    {
794      operation.run(newWriteableTransaction(txn));
795      commit(txn);
796    }
797    catch (final StorageRuntimeException e)
798    {
799      if (e.getCause() != null)
800      {
801        throw (Exception) e.getCause();
802      }
803      throw e;
804    }
805    finally
806    {
807      abort(txn);
808    }
809  }
810
811  private Transaction beginTransaction()
812  {
813    if (envConfig.getTransactional())
814    {
815      final Transaction txn = env.beginTransaction(null, TXN_READ_COMMITTED);
816      logger.trace("beginTransaction txnid=%d", txn.getId());
817      return txn;
818    }
819    return null;
820  }
821
822  private void commit(final Transaction txn)
823  {
824    if (txn != null)
825    {
826      txn.commit();
827      logger.trace("commit txnid=%d", txn.getId());
828    }
829  }
830
831  private void abort(final Transaction txn)
832  {
833    if (txn != null)
834    {
835      txn.abort();
836      logger.trace("abort txnid=%d", txn.getId());
837    }
838  }
839
840  @Override
841  public boolean supportsBackupAndRestore()
842  {
843    return true;
844  }
845
846  @Override
847  public File getDirectory()
848  {
849    return getBackendDirectory(config);
850  }
851
852  private static File getBackendDirectory(JEBackendCfg cfg)
853  {
854    return getDBDirectory(cfg.getDBDirectory(), cfg.getBackendId());
855  }
856
857  @Override
858  public ListIterator<Path> getFilesToBackup() throws DirectoryException
859  {
860    return new JELogFilesIterator(getDirectory(), config.getBackendId());
861  }
862
863  /**
864   * Iterator on JE log files to backup.
865   * <p>
866   * The cleaner thread may delete some log files and add new ones during the backup.
867   * The iterator is automatically renewed if at least one file is added.
868   */
869  static class JELogFilesIterator implements ListIterator<Path>
870  {
871    private final File rootDirectory;
872    private final String backendID;
873    private final SortedSet<Path> rememberedFiles = new TreeSet<>();
874    private ListIterator<Path> listIterator;
875    private boolean endOfRescan = false;
876
877    JELogFilesIterator(File rootDirectory, String backendID) throws DirectoryException
878    {
879      this.rootDirectory = rootDirectory;
880      this.backendID = backendID;
881      rescanFiles(0);
882    }
883
884    private void rescanFiles(final int startIndex) throws DirectoryException
885    {
886      final int initialSize = rememberedFiles.size();
887      rememberedFiles.addAll(getFiles(rootDirectory, new JELogFileFilter(), backendID));
888      listIterator = new ArrayList<>(rememberedFiles).listIterator(startIndex);
889      // if there is no new files, then trigger he end of rescanning files
890      endOfRescan = initialSize == rememberedFiles.size();
891    }
892
893    @Override
894    public boolean hasNext()
895    {
896      boolean hasNext = listIterator.hasNext();
897      if (hasNext) {
898        return true;
899      }
900      if (endOfRescan) {
901        return false;
902      }
903      // rescan files to check if there are new ones
904      final int nextIndex = listIterator.nextIndex();
905      try {
906        rescanFiles(nextIndex);
907      } catch (final DirectoryException e) {
908        logger.error(ERR_BACKEND_LIST_FILES_TO_BACKUP.get(backendID, stackTraceToSingleLineString(e)));
909      }
910      return listIterator.hasNext();
911    }
912
913    @Override
914    public Path next()
915    {
916      if (hasNext())
917      {
918        return listIterator.next();
919      }
920      throw new NoSuchElementException();
921    }
922
923    @Override
924    public boolean hasPrevious()
925    {
926      return listIterator.hasPrevious();
927    }
928
929    @Override
930    public Path previous()
931    {
932      return listIterator.previous();
933    }
934
935    @Override
936    public int nextIndex()
937    {
938      return listIterator.nextIndex();
939    }
940
941    @Override
942    public int previousIndex()
943    {
944      return listIterator.previousIndex();
945    }
946
947    @Override
948    public void remove()
949    {
950      throw new UnsupportedOperationException("remove() is not implemented");
951    }
952
953    @Override
954    public void set(Path e)
955    {
956      throw new UnsupportedOperationException("set() is not implemented");
957    }
958
959    @Override
960    public void add(Path e)
961    {
962      throw new UnsupportedOperationException("add() is not implemented");
963    }
964  }
965
966  /**
967   * This class implements a FilenameFilter to detect a JE log file, possibly with a constraint on the file name.
968   */
969  private static class JELogFileFilter implements FileFilter
970  {
971    private final String latestFilename;
972
973    /**
974     * Creates the filter for log files that are newer than provided file name.
975     *
976     * @param latestFilename the latest file name
977     */
978    JELogFileFilter(String latestFilename)
979    {
980      this.latestFilename = latestFilename;
981    }
982
983    /** Creates the filter for any JE log file. */
984    JELogFileFilter()
985    {
986      this("");
987    }
988
989    @Override
990    public boolean accept(File file)
991    {
992      String name = file.getName();
993      return name.endsWith(".jdb") && (name.compareTo(latestFilename) > 0);
994    }
995  }
996
997  @Override
998  public Path beforeRestore() throws DirectoryException
999  {
1000    return null;
1001  }
1002
1003  @Override
1004  public boolean isDirectRestore()
1005  {
1006    // restore is done in an intermediate directory
1007    return false;
1008  }
1009
1010  @Override
1011  public void afterRestore(Path restoreDirectory, Path saveDirectory) throws DirectoryException
1012  {
1013    // intermediate directory content is moved to database directory
1014    File targetDirectory = getDirectory();
1015    recursiveDelete(targetDirectory);
1016    try
1017    {
1018      Files.move(restoreDirectory, targetDirectory.toPath());
1019    }
1020    catch(IOException e)
1021    {
1022      LocalizableMessage msg = ERR_CANNOT_RENAME_RESTORE_DIRECTORY.get(restoreDirectory, targetDirectory.getPath());
1023      throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg);
1024    }
1025  }
1026
1027  @Override
1028  public void createBackup(BackupConfig backupConfig) throws DirectoryException
1029  {
1030    new BackupManager(config.getBackendId()).createBackup(this, backupConfig);
1031  }
1032
1033  @Override
1034  public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException
1035  {
1036    new BackupManager(config.getBackendId()).removeBackup(backupDirectory, backupID);
1037  }
1038
1039  @Override
1040  public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException
1041  {
1042    new BackupManager(config.getBackendId()).restoreBackup(this, restoreConfig);
1043  }
1044
1045  @Override
1046  public Set<TreeName> listTrees()
1047  {
1048    try
1049    {
1050      List<String> treeNames = env.getDatabaseNames();
1051      final Set<TreeName> results = new HashSet<>(treeNames.size());
1052      for (String treeName : treeNames)
1053      {
1054        results.add(TreeName.valueOf(treeName));
1055      }
1056      return results;
1057    }
1058    catch (DatabaseException e)
1059    {
1060      throw new StorageRuntimeException(e);
1061    }
1062  }
1063
1064  @Override
1065  public boolean isConfigurationChangeAcceptable(JEBackendCfg newCfg,
1066      List<LocalizableMessage> unacceptableReasons)
1067  {
1068    long newSize = computeSize(newCfg);
1069    long oldSize = computeSize(config);
1070    return (newSize <= oldSize || memQuota.isMemoryAvailable(newSize - oldSize))
1071        && checkConfigurationDirectories(newCfg, unacceptableReasons);
1072  }
1073
1074  private long computeSize(JEBackendCfg cfg)
1075  {
1076    return cfg.getDBCacheSize() > 0 ? cfg.getDBCacheSize() : memQuota.memPercentToBytes(cfg.getDBCachePercent());
1077  }
1078
1079  /**
1080   * Checks newly created backend has a valid configuration.
1081   * @param cfg the new configuration
1082   * @param unacceptableReasons the list of accumulated errors and their messages
1083   * @param context the server context
1084   * @return true if newly created backend has a valid configuration
1085   */
1086  static boolean isConfigurationAcceptable(JEBackendCfg cfg, List<LocalizableMessage> unacceptableReasons,
1087      ServerContext context)
1088  {
1089    if (context != null)
1090    {
1091      MemoryQuota memQuota = context.getMemoryQuota();
1092      if (cfg.getDBCacheSize() > 0 && !memQuota.isMemoryAvailable(cfg.getDBCacheSize()))
1093      {
1094        unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_SIZE_GREATER_THAN_JVM_HEAP.get(
1095            cfg.getDBCacheSize(), memQuota.getAvailableMemory()));
1096        return false;
1097      }
1098      else if (!memQuota.isMemoryAvailable(memQuota.memPercentToBytes(cfg.getDBCachePercent())))
1099      {
1100        unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_PERCENT_GREATER_THAN_JVM_HEAP.get(
1101            cfg.getDBCachePercent(), memQuota.memBytesToPercent(memQuota.getAvailableMemory())));
1102        return false;
1103      }
1104    }
1105    return checkConfigurationDirectories(cfg, unacceptableReasons);
1106  }
1107
1108  private static boolean checkConfigurationDirectories(JEBackendCfg cfg,
1109    List<LocalizableMessage> unacceptableReasons)
1110  {
1111    final ConfigChangeResult ccr = new ConfigChangeResult();
1112    File newBackendDirectory = getBackendDirectory(cfg);
1113
1114    checkDBDirExistsOrCanCreate(newBackendDirectory, ccr, true);
1115    checkDBDirPermissions(cfg.getDBDirectoryPermissions(), cfg.dn(), ccr);
1116    if (!ccr.getMessages().isEmpty())
1117    {
1118      unacceptableReasons.addAll(ccr.getMessages());
1119      return false;
1120    }
1121    return true;
1122  }
1123
1124  @Override
1125  public ConfigChangeResult applyConfigurationChange(JEBackendCfg cfg)
1126  {
1127    final ConfigChangeResult ccr = new ConfigChangeResult();
1128
1129    try
1130    {
1131      File newBackendDirectory = getBackendDirectory(cfg);
1132
1133      // Create the directory if it doesn't exist.
1134      if (!cfg.getDBDirectory().equals(config.getDBDirectory()))
1135      {
1136        checkDBDirExistsOrCanCreate(newBackendDirectory, ccr, false);
1137        if (!ccr.getMessages().isEmpty())
1138        {
1139          return ccr;
1140        }
1141
1142        ccr.setAdminActionRequired(true);
1143        ccr.addMessage(NOTE_CONFIG_DB_DIR_REQUIRES_RESTART.get(config.getDBDirectory(), cfg.getDBDirectory()));
1144      }
1145
1146      if (!cfg.getDBDirectoryPermissions().equalsIgnoreCase(config.getDBDirectoryPermissions())
1147          || !cfg.getDBDirectory().equals(config.getDBDirectory()))
1148      {
1149        checkDBDirPermissions(cfg.getDBDirectoryPermissions(), cfg.dn(), ccr);
1150        if (!ccr.getMessages().isEmpty())
1151        {
1152          return ccr;
1153        }
1154
1155        setDBDirPermissions(newBackendDirectory, cfg.getDBDirectoryPermissions(), cfg.dn(), ccr);
1156        if (!ccr.getMessages().isEmpty())
1157        {
1158          return ccr;
1159        }
1160      }
1161      registerMonitoredDirectory(cfg);
1162      config = cfg;
1163    }
1164    catch (Exception e)
1165    {
1166      addErrorMessage(ccr, LocalizableMessage.raw(stackTraceToSingleLineString(e)));
1167    }
1168    return ccr;
1169  }
1170
1171  private void registerMonitoredDirectory(JEBackendCfg cfg)
1172  {
1173    diskMonitor.registerMonitoredDirectory(
1174      cfg.getBackendId() + " backend",
1175      getDirectory(),
1176      cfg.getDiskLowThreshold(),
1177      cfg.getDiskFullThreshold(),
1178      this);
1179  }
1180
1181  @Override
1182  public void removeStorageFiles() throws StorageRuntimeException
1183  {
1184    StorageUtils.removeStorageFiles(backendDirectory);
1185  }
1186
1187  @Override
1188  public StorageStatus getStorageStatus()
1189  {
1190    return storageStatus;
1191  }
1192
1193  @Override
1194  public void diskFullThresholdReached(File directory, long thresholdInBytes) {
1195    storageStatus = statusWhenDiskSpaceFull(directory, thresholdInBytes, config.getBackendId());
1196  }
1197
1198  @Override
1199  public void diskLowThresholdReached(File directory, long thresholdInBytes) {
1200    storageStatus = statusWhenDiskSpaceLow(directory, thresholdInBytes, config.getBackendId());
1201  }
1202
1203  @Override
1204  public void diskSpaceRestored(File directory, long lowThresholdInBytes, long fullThresholdInBytes) {
1205    storageStatus = StorageStatus.working();
1206  }
1207
1208  private static void setData(final DatabaseEntry dbEntry, final ByteSequence bs)
1209  {
1210    dbEntry.setData(bs != null ? bs.toByteArray() : null);
1211  }
1212
1213  private static DatabaseEntry db(final ByteSequence bs)
1214  {
1215    return new DatabaseEntry(bs != null ? bs.toByteArray() : null);
1216  }
1217
1218  private static ByteString valueToBytes(final DatabaseEntry dbValue, boolean isDefined)
1219  {
1220    if (isDefined)
1221    {
1222      return ByteString.wrap(dbValue.getData());
1223    }
1224    return null;
1225  }
1226}