001/* 002 * The contents of this file are subject to the terms of the Common Development and 003 * Distribution License (the License). You may not use this file except in compliance with the 004 * License. 005 * 006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the 007 * specific language governing permission and limitations under the License. 008 * 009 * When distributing Covered Software, include this CDDL Header Notice in each file and include 010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL 011 * Header, with the fields enclosed by brackets [] replaced by your own identifying 012 * information: "Portions Copyright [year] [name of copyright owner]". 013 * 014 * Copyright 2015-2017 ForgeRock AS. 015 */ 016package org.opends.server.backends.jeb; 017 018import static com.sleepycat.je.EnvironmentConfig.*; 019import static com.sleepycat.je.LockMode.READ_COMMITTED; 020import static com.sleepycat.je.LockMode.RMW; 021import static com.sleepycat.je.OperationStatus.*; 022import static org.forgerock.util.Utils.*; 023import static org.opends.messages.BackendMessages.*; 024import static org.opends.messages.UtilityMessages.*; 025import static org.opends.server.backends.pluggable.spi.StorageUtils.*; 026import static org.opends.server.util.BackupManager.getFiles; 027import static org.opends.server.util.StaticUtils.*; 028 029import java.io.File; 030import java.io.FileFilter; 031import java.io.IOException; 032import java.nio.file.Files; 033import java.nio.file.Path; 034import java.util.ArrayList; 035import java.util.HashMap; 036import java.util.HashSet; 037import java.util.List; 038import java.util.ListIterator; 039import java.util.Map; 040import java.util.NoSuchElementException; 041import java.util.Objects; 042import java.util.Set; 043import java.util.SortedSet; 044import java.util.TreeSet; 045import java.util.concurrent.ConcurrentHashMap; 046import java.util.concurrent.ConcurrentMap; 047import java.util.concurrent.TimeUnit; 048 049import org.forgerock.i18n.LocalizableMessage; 050import org.forgerock.i18n.slf4j.LocalizedLogger; 051import org.forgerock.opendj.config.server.ConfigChangeResult; 052import org.forgerock.opendj.config.server.ConfigException; 053import org.forgerock.opendj.ldap.ByteSequence; 054import org.forgerock.opendj.ldap.ByteString; 055import org.forgerock.util.Reject; 056import org.forgerock.opendj.config.server.ConfigurationChangeListener; 057import org.forgerock.opendj.server.config.server.JEBackendCfg; 058import org.opends.server.api.Backupable; 059import org.opends.server.api.DiskSpaceMonitorHandler; 060import org.opends.server.backends.pluggable.spi.AccessMode; 061import org.opends.server.backends.pluggable.spi.Cursor; 062import org.opends.server.backends.pluggable.spi.Importer; 063import org.opends.server.backends.pluggable.spi.ReadOnlyStorageException; 064import org.opends.server.backends.pluggable.spi.ReadOperation; 065import org.opends.server.backends.pluggable.spi.SequentialCursor; 066import org.opends.server.backends.pluggable.spi.Storage; 067import org.opends.server.backends.pluggable.spi.StorageRuntimeException; 068import org.opends.server.backends.pluggable.spi.StorageStatus; 069import org.opends.server.backends.pluggable.spi.StorageUtils; 070import org.opends.server.backends.pluggable.spi.TreeName; 071import org.opends.server.backends.pluggable.spi.UpdateFunction; 072import org.opends.server.backends.pluggable.spi.WriteOperation; 073import org.opends.server.backends.pluggable.spi.WriteableTransaction; 074import org.opends.server.core.DirectoryServer; 075import org.opends.server.core.MemoryQuota; 076import org.opends.server.core.ServerContext; 077import org.opends.server.extensions.DiskSpaceMonitor; 078import org.opends.server.types.BackupConfig; 079import org.opends.server.types.BackupDirectory; 080import org.opends.server.types.DirectoryException; 081import org.opends.server.types.RestoreConfig; 082import org.opends.server.util.BackupManager; 083 084import com.sleepycat.je.CursorConfig; 085import com.sleepycat.je.Database; 086import com.sleepycat.je.DatabaseConfig; 087import com.sleepycat.je.DatabaseEntry; 088import com.sleepycat.je.DatabaseException; 089import com.sleepycat.je.DatabaseNotFoundException; 090import com.sleepycat.je.Durability; 091import com.sleepycat.je.Environment; 092import com.sleepycat.je.EnvironmentConfig; 093import com.sleepycat.je.OperationStatus; 094import com.sleepycat.je.Transaction; 095import com.sleepycat.je.TransactionConfig; 096 097/** Berkeley DB Java Edition (JE for short) database implementation of the {@link Storage} engine. */ 098public final class JEStorage implements Storage, Backupable, ConfigurationChangeListener<JEBackendCfg>, 099 DiskSpaceMonitorHandler 100{ 101 /** JE implementation of the {@link Cursor} interface. */ 102 private static final class CursorImpl implements Cursor<ByteString, ByteString> 103 { 104 private ByteString currentKey; 105 private ByteString currentValue; 106 private boolean isDefined; 107 private final com.sleepycat.je.Cursor cursor; 108 private final DatabaseEntry dbKey = new DatabaseEntry(); 109 private final DatabaseEntry dbValue = new DatabaseEntry(); 110 111 private CursorImpl(com.sleepycat.je.Cursor cursor) 112 { 113 this.cursor = cursor; 114 } 115 116 @Override 117 public void close() 118 { 119 closeSilently(cursor); 120 } 121 122 @Override 123 public boolean isDefined() 124 { 125 return isDefined; 126 } 127 128 @Override 129 public ByteString getKey() 130 { 131 if (currentKey == null) 132 { 133 throwIfNotSuccess(); 134 currentKey = ByteString.wrap(dbKey.getData()); 135 } 136 return currentKey; 137 } 138 139 @Override 140 public ByteString getValue() 141 { 142 if (currentValue == null) 143 { 144 throwIfNotSuccess(); 145 currentValue = ByteString.wrap(dbValue.getData()); 146 } 147 return currentValue; 148 } 149 150 @Override 151 public boolean next() 152 { 153 clearCurrentKeyAndValue(); 154 try 155 { 156 isDefined = cursor.getNext(dbKey, dbValue, null) == SUCCESS; 157 return isDefined; 158 } 159 catch (DatabaseException e) 160 { 161 throw new StorageRuntimeException(e); 162 } 163 } 164 165 @Override 166 public void delete() throws NoSuchElementException, UnsupportedOperationException 167 { 168 throwIfNotSuccess(); 169 try 170 { 171 cursor.delete(); 172 } 173 catch (DatabaseException e) 174 { 175 throw new StorageRuntimeException(e); 176 } 177 } 178 179 @Override 180 public boolean positionToKey(final ByteSequence key) 181 { 182 clearCurrentKeyAndValue(); 183 setData(dbKey, key); 184 try 185 { 186 isDefined = cursor.getSearchKey(dbKey, dbValue, null) == SUCCESS; 187 return isDefined; 188 } 189 catch (DatabaseException e) 190 { 191 throw new StorageRuntimeException(e); 192 } 193 } 194 195 @Override 196 public boolean positionToKeyOrNext(final ByteSequence key) 197 { 198 clearCurrentKeyAndValue(); 199 setData(dbKey, key); 200 try 201 { 202 isDefined = cursor.getSearchKeyRange(dbKey, dbValue, null) == SUCCESS; 203 return isDefined; 204 } 205 catch (DatabaseException e) 206 { 207 throw new StorageRuntimeException(e); 208 } 209 } 210 211 @Override 212 public boolean positionToIndex(int index) 213 { 214 clearCurrentKeyAndValue(); 215 try 216 { 217 isDefined = cursor.getFirst(dbKey, dbValue, null) == SUCCESS; 218 if (!isDefined) 219 { 220 return false; 221 } 222 else if (index == 0) 223 { 224 return true; 225 } 226 227 // equivalent to READ_UNCOMMITTED 228 long skipped = cursor.skipNext(index, dbKey, dbValue, null); 229 if (skipped == index) 230 { 231 isDefined = cursor.getCurrent(dbKey, dbValue, null) == SUCCESS; 232 } 233 else 234 { 235 isDefined = false; 236 } 237 return isDefined; 238 } 239 catch (DatabaseException e) 240 { 241 throw new StorageRuntimeException(e); 242 } 243 } 244 245 @Override 246 public boolean positionToLastKey() 247 { 248 clearCurrentKeyAndValue(); 249 try 250 { 251 isDefined = cursor.getLast(dbKey, dbValue, null) == SUCCESS; 252 return isDefined; 253 } 254 catch (DatabaseException e) 255 { 256 throw new StorageRuntimeException(e); 257 } 258 } 259 260 private void clearCurrentKeyAndValue() 261 { 262 currentKey = null; 263 currentValue = null; 264 } 265 266 private void throwIfNotSuccess() 267 { 268 if (!isDefined()) 269 { 270 throw new NoSuchElementException(); 271 } 272 } 273 } 274 275 /** JE implementation of the {@link Importer} interface. */ 276 private final class ImporterImpl implements Importer 277 { 278 private final Map<TreeName, Database> trees = new HashMap<>(); 279 280 private Database getOrOpenTree(TreeName treeName) 281 { 282 return getOrOpenTree0(trees, treeName); 283 } 284 285 @Override 286 public void put(final TreeName treeName, final ByteSequence key, final ByteSequence value) 287 { 288 try 289 { 290 getOrOpenTree(treeName).put(null, db(key), db(value)); 291 } 292 catch (DatabaseException e) 293 { 294 throw new StorageRuntimeException(e); 295 } 296 } 297 298 @Override 299 public ByteString read(final TreeName treeName, final ByteSequence key) 300 { 301 try 302 { 303 DatabaseEntry dbValue = new DatabaseEntry(); 304 boolean isDefined = getOrOpenTree(treeName).get(null, db(key), dbValue, null) == SUCCESS; 305 return valueToBytes(dbValue, isDefined); 306 } 307 catch (DatabaseException e) 308 { 309 throw new StorageRuntimeException(e); 310 } 311 } 312 313 @Override 314 public SequentialCursor<ByteString, ByteString> openCursor(TreeName treeName) 315 { 316 try 317 { 318 return new CursorImpl(getOrOpenTree(treeName).openCursor(null, new CursorConfig())); 319 } 320 catch (DatabaseException e) 321 { 322 throw new StorageRuntimeException(e); 323 } 324 } 325 326 @Override 327 public void clearTree(TreeName treeName) 328 { 329 env.truncateDatabase(null, toDatabaseName(treeName), false); 330 } 331 332 @Override 333 public void close() 334 { 335 closeSilently(trees.values()); 336 trees.clear(); 337 JEStorage.this.close(); 338 } 339 } 340 341 /** JE implementation of the {@link WriteableTransaction} interface. */ 342 private final class WriteableTransactionImpl implements WriteableTransaction 343 { 344 private final Transaction txn; 345 346 private WriteableTransactionImpl(Transaction txn) 347 { 348 this.txn = txn; 349 } 350 351 /** 352 * This is currently needed for import-ldif: 353 * <ol> 354 * <li>Opening the EntryContainer calls {@link #openTree(TreeName, boolean)} for each index</li> 355 * <li>Then the underlying storage is closed</li> 356 * <li>Then {@link Importer#startImport()} is called</li> 357 * <li>Then ID2Entry#put() is called</li> 358 * <li>Which in turn calls ID2Entry#encodeEntry()</li> 359 * <li>Which in turn finally calls PersistentCompressedSchema#store()</li> 360 * <li>Which uses a reference to the storage (that was closed before calling startImport()) and 361 * uses it as if it was open</li> 362 * </ol> 363 */ 364 private Database getOrOpenTree(TreeName treeName) 365 { 366 try 367 { 368 return getOrOpenTree0(trees, treeName); 369 } 370 catch (Exception e) 371 { 372 throw new StorageRuntimeException(e); 373 } 374 } 375 376 @Override 377 public void put(final TreeName treeName, final ByteSequence key, final ByteSequence value) 378 { 379 try 380 { 381 final OperationStatus status = getOrOpenTree(treeName).put(txn, db(key), db(value)); 382 if (status != SUCCESS) 383 { 384 throw new StorageRuntimeException(putErrorMsg(treeName, key, value, "did not succeed: " + status)); 385 } 386 } 387 catch (DatabaseException e) 388 { 389 throw new StorageRuntimeException(putErrorMsg(treeName, key, value, "threw an exception"), e); 390 } 391 } 392 393 private String putErrorMsg(TreeName treeName, ByteSequence key, ByteSequence value, String msg) 394 { 395 return "put(treeName=" + treeName + ", key=" + key + ", value=" + value + ") " + msg; 396 } 397 398 @Override 399 public boolean delete(final TreeName treeName, final ByteSequence key) 400 { 401 try 402 { 403 return getOrOpenTree(treeName).delete(txn, db(key)) == SUCCESS; 404 } 405 catch (DatabaseException e) 406 { 407 throw new StorageRuntimeException(deleteErrorMsg(treeName, key, "threw an exception"), e); 408 } 409 } 410 411 private String deleteErrorMsg(TreeName treeName, ByteSequence key, String msg) 412 { 413 return "delete(treeName=" + treeName + ", key=" + key + ") " + msg; 414 } 415 416 @Override 417 public long getRecordCount(TreeName treeName) 418 { 419 try 420 { 421 return getOrOpenTree(treeName).count(); 422 } 423 catch (DatabaseException e) 424 { 425 throw new StorageRuntimeException(e); 426 } 427 } 428 429 @Override 430 public Cursor<ByteString, ByteString> openCursor(final TreeName treeName) 431 { 432 try 433 { 434 return new CursorImpl(getOrOpenTree(treeName).openCursor(txn, CursorConfig.READ_COMMITTED)); 435 } 436 catch (DatabaseException e) 437 { 438 throw new StorageRuntimeException(e); 439 } 440 } 441 442 @Override 443 public ByteString read(final TreeName treeName, final ByteSequence key) 444 { 445 try 446 { 447 DatabaseEntry dbValue = new DatabaseEntry(); 448 boolean isDefined = getOrOpenTree(treeName).get(txn, db(key), dbValue, READ_COMMITTED) == SUCCESS; 449 return valueToBytes(dbValue, isDefined); 450 } 451 catch (DatabaseException e) 452 { 453 throw new StorageRuntimeException(e); 454 } 455 } 456 457 @Override 458 public boolean update(final TreeName treeName, final ByteSequence key, final UpdateFunction f) 459 { 460 try 461 { 462 final Database tree = getOrOpenTree(treeName); 463 final DatabaseEntry dbKey = db(key); 464 final DatabaseEntry dbValue = new DatabaseEntry(); 465 for (;;) 466 { 467 final boolean isDefined = tree.get(txn, dbKey, dbValue, RMW) == SUCCESS; 468 final ByteSequence oldValue = valueToBytes(dbValue, isDefined); 469 final ByteSequence newValue = f.computeNewValue(oldValue); 470 if (Objects.equals(newValue, oldValue)) 471 { 472 return false; 473 } 474 if (newValue == null) 475 { 476 return tree.delete(txn, dbKey) == SUCCESS; 477 } 478 setData(dbValue, newValue); 479 if (isDefined) 480 { 481 return tree.put(txn, dbKey, dbValue) == SUCCESS; 482 } 483 else if (tree.putNoOverwrite(txn, dbKey, dbValue) == SUCCESS) 484 { 485 return true; 486 } 487 // else retry due to phantom read: another thread inserted a record 488 } 489 } 490 catch (DatabaseException e) 491 { 492 throw new StorageRuntimeException(e); 493 } 494 } 495 496 @Override 497 public void openTree(final TreeName treeName, boolean createOnDemand) 498 { 499 getOrOpenTree(treeName); 500 } 501 502 @Override 503 public void deleteTree(final TreeName treeName) 504 { 505 try 506 { 507 synchronized (trees) 508 { 509 closeSilently(trees.remove(treeName)); 510 env.removeDatabase(txn, toDatabaseName(treeName)); 511 } 512 } 513 catch (DatabaseNotFoundException e) 514 { 515 // This is fine: end result is what we wanted 516 } 517 catch (DatabaseException e) 518 { 519 throw new StorageRuntimeException(e); 520 } 521 } 522 } 523 524 /** JE read-only implementation of {@link StorageImpl} interface. */ 525 private final class ReadOnlyTransactionImpl implements WriteableTransaction 526 { 527 private final WriteableTransactionImpl delegate; 528 529 ReadOnlyTransactionImpl(WriteableTransactionImpl delegate) 530 { 531 this.delegate = delegate; 532 } 533 534 @Override 535 public ByteString read(TreeName treeName, ByteSequence key) 536 { 537 return delegate.read(treeName, key); 538 } 539 540 @Override 541 public Cursor<ByteString, ByteString> openCursor(TreeName treeName) 542 { 543 return delegate.openCursor(treeName); 544 } 545 546 @Override 547 public long getRecordCount(TreeName treeName) 548 { 549 return delegate.getRecordCount(treeName); 550 } 551 552 @Override 553 public void openTree(TreeName treeName, boolean createOnDemand) 554 { 555 if (createOnDemand) 556 { 557 throw new ReadOnlyStorageException(); 558 } 559 delegate.openTree(treeName, false); 560 } 561 562 @Override 563 public void deleteTree(TreeName name) 564 { 565 throw new ReadOnlyStorageException(); 566 } 567 568 @Override 569 public void put(TreeName treeName, ByteSequence key, ByteSequence value) 570 { 571 throw new ReadOnlyStorageException(); 572 } 573 574 @Override 575 public boolean update(TreeName treeName, ByteSequence key, UpdateFunction f) 576 { 577 throw new ReadOnlyStorageException(); 578 } 579 580 @Override 581 public boolean delete(TreeName treeName, ByteSequence key) 582 { 583 throw new ReadOnlyStorageException(); 584 } 585 } 586 587 private WriteableTransaction newWriteableTransaction(Transaction txn) 588 { 589 final WriteableTransactionImpl writeableStorage = new WriteableTransactionImpl(txn); 590 return accessMode.isWriteable() ? writeableStorage : new ReadOnlyTransactionImpl(writeableStorage); 591 } 592 593 private static final int IMPORT_DB_CACHE_SIZE = 32 * MB; 594 595 private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); 596 597 /** Use read committed isolation instead of the default which is repeatable read. */ 598 private static final TransactionConfig TXN_READ_COMMITTED = new TransactionConfig().setReadCommitted(true); 599 600 private final ServerContext serverContext; 601 private final File backendDirectory; 602 private JEBackendCfg config; 603 private AccessMode accessMode; 604 605 private Environment env; 606 private EnvironmentConfig envConfig; 607 private MemoryQuota memQuota; 608 private JEMonitor monitor; 609 private DiskSpaceMonitor diskMonitor; 610 private StorageStatus storageStatus = StorageStatus.working(); 611 private final ConcurrentMap<TreeName, Database> trees = new ConcurrentHashMap<>(); 612 613 /** 614 * Creates a new JE storage with the provided configuration. 615 * 616 * @param cfg 617 * The configuration. 618 * @param serverContext 619 * This server instance context 620 * @throws ConfigException 621 * if memory cannot be reserved 622 */ 623 JEStorage(final JEBackendCfg cfg, ServerContext serverContext) throws ConfigException 624 { 625 this.serverContext = serverContext; 626 backendDirectory = getBackendDirectory(cfg); 627 config = cfg; 628 cfg.addJEChangeListener(this); 629 } 630 631 private Database getOrOpenTree0(Map<TreeName, Database> trees, TreeName treeName) 632 { 633 Database tree = trees.get(treeName); 634 if (tree == null) 635 { 636 synchronized (trees) 637 { 638 tree = trees.get(treeName); 639 if (tree == null) 640 { 641 tree = env.openDatabase(null, toDatabaseName(treeName), dbConfig()); 642 trees.put(treeName, tree); 643 } 644 } 645 } 646 return tree; 647 } 648 649 private void buildConfiguration(AccessMode accessMode, boolean isImport) throws ConfigException 650 { 651 this.accessMode = accessMode; 652 653 if (isImport) 654 { 655 envConfig = new EnvironmentConfig(); 656 envConfig 657 .setTransactional(false) 658 .setAllowCreate(true) 659 .setLockTimeout(0, TimeUnit.SECONDS) 660 .setTxnTimeout(0, TimeUnit.SECONDS) 661 .setCacheSize(IMPORT_DB_CACHE_SIZE) 662 .setDurability(Durability.COMMIT_NO_SYNC) 663 .setConfigParam(CLEANER_MIN_UTILIZATION, String.valueOf(config.getDBCleanerMinUtilization())) 664 .setConfigParam(LOG_FILE_MAX, String.valueOf(config.getDBLogFileMax())); 665 } 666 else 667 { 668 envConfig = ConfigurableEnvironment.parseConfigEntry(config); 669 } 670 671 diskMonitor = serverContext.getDiskSpaceMonitor(); 672 memQuota = serverContext.getMemoryQuota(); 673 if (config.getDBCacheSize() > 0) 674 { 675 memQuota.acquireMemory(config.getDBCacheSize()); 676 } 677 else 678 { 679 memQuota.acquireMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); 680 } 681 } 682 683 private DatabaseConfig dbConfig() 684 { 685 boolean isImport = !envConfig.getTransactional(); 686 return new DatabaseConfig() 687 .setKeyPrefixing(true) 688 .setAllowCreate(true) 689 .setTransactional(!isImport) 690 .setDeferredWrite(isImport); 691 } 692 693 @Override 694 public void close() 695 { 696 synchronized (trees) 697 { 698 closeSilently(trees.values()); 699 trees.clear(); 700 } 701 702 if (env != null) 703 { 704 DirectoryServer.deregisterMonitorProvider(monitor); 705 monitor = null; 706 try 707 { 708 env.close(); 709 env = null; 710 } 711 catch (DatabaseException e) 712 { 713 throw new IllegalStateException(e); 714 } 715 } 716 717 if (config.getDBCacheSize() > 0) 718 { 719 memQuota.releaseMemory(config.getDBCacheSize()); 720 } 721 else 722 { 723 memQuota.releaseMemory(memQuota.memPercentToBytes(config.getDBCachePercent())); 724 } 725 config.removeJEChangeListener(this); 726 diskMonitor.deregisterMonitoredDirectory(getDirectory(), this); 727 } 728 729 @Override 730 public void open(AccessMode accessMode) throws ConfigException, StorageRuntimeException 731 { 732 Reject.ifNull(accessMode, "accessMode must not be null"); 733 buildConfiguration(accessMode, false); 734 open0(); 735 } 736 737 private void open0() throws ConfigException 738 { 739 setupStorageFiles(backendDirectory, config.getDBDirectoryPermissions(), config.dn()); 740 try 741 { 742 if (env != null) 743 { 744 throw new IllegalStateException( 745 "Database is already open, either the backend is enabled or an import is currently running."); 746 } 747 env = new Environment(backendDirectory, envConfig); 748 monitor = new JEMonitor(config.getBackendId() + " JE Database", env); 749 DirectoryServer.registerMonitorProvider(monitor); 750 } 751 catch (DatabaseException e) 752 { 753 throw new StorageRuntimeException(e); 754 } 755 registerMonitoredDirectory(config); 756 } 757 758 @Override 759 public <T> T read(final ReadOperation<T> operation) throws Exception 760 { 761 try 762 { 763 return operation.run(newWriteableTransaction(null)); 764 } 765 catch (final StorageRuntimeException e) 766 { 767 if (e.getCause() != null) 768 { 769 throw (Exception) e.getCause(); 770 } 771 throw e; 772 } 773 } 774 775 @Override 776 public Importer startImport() throws ConfigException, StorageRuntimeException 777 { 778 buildConfiguration(AccessMode.READ_WRITE, true); 779 open0(); 780 return new ImporterImpl(); 781 } 782 783 private static String toDatabaseName(final TreeName treeName) 784 { 785 return treeName.toString(); 786 } 787 788 @Override 789 public void write(final WriteOperation operation) throws Exception 790 { 791 final Transaction txn = beginTransaction(); 792 try 793 { 794 operation.run(newWriteableTransaction(txn)); 795 commit(txn); 796 } 797 catch (final StorageRuntimeException e) 798 { 799 if (e.getCause() != null) 800 { 801 throw (Exception) e.getCause(); 802 } 803 throw e; 804 } 805 finally 806 { 807 abort(txn); 808 } 809 } 810 811 private Transaction beginTransaction() 812 { 813 if (envConfig.getTransactional()) 814 { 815 final Transaction txn = env.beginTransaction(null, TXN_READ_COMMITTED); 816 logger.trace("beginTransaction txnid=%d", txn.getId()); 817 return txn; 818 } 819 return null; 820 } 821 822 private void commit(final Transaction txn) 823 { 824 if (txn != null) 825 { 826 txn.commit(); 827 logger.trace("commit txnid=%d", txn.getId()); 828 } 829 } 830 831 private void abort(final Transaction txn) 832 { 833 if (txn != null) 834 { 835 txn.abort(); 836 logger.trace("abort txnid=%d", txn.getId()); 837 } 838 } 839 840 @Override 841 public boolean supportsBackupAndRestore() 842 { 843 return true; 844 } 845 846 @Override 847 public File getDirectory() 848 { 849 return getBackendDirectory(config); 850 } 851 852 private static File getBackendDirectory(JEBackendCfg cfg) 853 { 854 return getDBDirectory(cfg.getDBDirectory(), cfg.getBackendId()); 855 } 856 857 @Override 858 public ListIterator<Path> getFilesToBackup() throws DirectoryException 859 { 860 return new JELogFilesIterator(getDirectory(), config.getBackendId()); 861 } 862 863 /** 864 * Iterator on JE log files to backup. 865 * <p> 866 * The cleaner thread may delete some log files and add new ones during the backup. 867 * The iterator is automatically renewed if at least one file is added. 868 */ 869 static class JELogFilesIterator implements ListIterator<Path> 870 { 871 private final File rootDirectory; 872 private final String backendID; 873 private final SortedSet<Path> rememberedFiles = new TreeSet<>(); 874 private ListIterator<Path> listIterator; 875 private boolean endOfRescan = false; 876 877 JELogFilesIterator(File rootDirectory, String backendID) throws DirectoryException 878 { 879 this.rootDirectory = rootDirectory; 880 this.backendID = backendID; 881 rescanFiles(0); 882 } 883 884 private void rescanFiles(final int startIndex) throws DirectoryException 885 { 886 final int initialSize = rememberedFiles.size(); 887 rememberedFiles.addAll(getFiles(rootDirectory, new JELogFileFilter(), backendID)); 888 listIterator = new ArrayList<>(rememberedFiles).listIterator(startIndex); 889 // if there is no new files, then trigger he end of rescanning files 890 endOfRescan = initialSize == rememberedFiles.size(); 891 } 892 893 @Override 894 public boolean hasNext() 895 { 896 boolean hasNext = listIterator.hasNext(); 897 if (hasNext) { 898 return true; 899 } 900 if (endOfRescan) { 901 return false; 902 } 903 // rescan files to check if there are new ones 904 final int nextIndex = listIterator.nextIndex(); 905 try { 906 rescanFiles(nextIndex); 907 } catch (final DirectoryException e) { 908 logger.error(ERR_BACKEND_LIST_FILES_TO_BACKUP.get(backendID, stackTraceToSingleLineString(e))); 909 } 910 return listIterator.hasNext(); 911 } 912 913 @Override 914 public Path next() 915 { 916 if (hasNext()) 917 { 918 return listIterator.next(); 919 } 920 throw new NoSuchElementException(); 921 } 922 923 @Override 924 public boolean hasPrevious() 925 { 926 return listIterator.hasPrevious(); 927 } 928 929 @Override 930 public Path previous() 931 { 932 return listIterator.previous(); 933 } 934 935 @Override 936 public int nextIndex() 937 { 938 return listIterator.nextIndex(); 939 } 940 941 @Override 942 public int previousIndex() 943 { 944 return listIterator.previousIndex(); 945 } 946 947 @Override 948 public void remove() 949 { 950 throw new UnsupportedOperationException("remove() is not implemented"); 951 } 952 953 @Override 954 public void set(Path e) 955 { 956 throw new UnsupportedOperationException("set() is not implemented"); 957 } 958 959 @Override 960 public void add(Path e) 961 { 962 throw new UnsupportedOperationException("add() is not implemented"); 963 } 964 } 965 966 /** 967 * This class implements a FilenameFilter to detect a JE log file, possibly with a constraint on the file name. 968 */ 969 private static class JELogFileFilter implements FileFilter 970 { 971 private final String latestFilename; 972 973 /** 974 * Creates the filter for log files that are newer than provided file name. 975 * 976 * @param latestFilename the latest file name 977 */ 978 JELogFileFilter(String latestFilename) 979 { 980 this.latestFilename = latestFilename; 981 } 982 983 /** Creates the filter for any JE log file. */ 984 JELogFileFilter() 985 { 986 this(""); 987 } 988 989 @Override 990 public boolean accept(File file) 991 { 992 String name = file.getName(); 993 return name.endsWith(".jdb") && (name.compareTo(latestFilename) > 0); 994 } 995 } 996 997 @Override 998 public Path beforeRestore() throws DirectoryException 999 { 1000 return null; 1001 } 1002 1003 @Override 1004 public boolean isDirectRestore() 1005 { 1006 // restore is done in an intermediate directory 1007 return false; 1008 } 1009 1010 @Override 1011 public void afterRestore(Path restoreDirectory, Path saveDirectory) throws DirectoryException 1012 { 1013 // intermediate directory content is moved to database directory 1014 File targetDirectory = getDirectory(); 1015 recursiveDelete(targetDirectory); 1016 try 1017 { 1018 Files.move(restoreDirectory, targetDirectory.toPath()); 1019 } 1020 catch(IOException e) 1021 { 1022 LocalizableMessage msg = ERR_CANNOT_RENAME_RESTORE_DIRECTORY.get(restoreDirectory, targetDirectory.getPath()); 1023 throw new DirectoryException(DirectoryServer.getServerErrorResultCode(), msg); 1024 } 1025 } 1026 1027 @Override 1028 public void createBackup(BackupConfig backupConfig) throws DirectoryException 1029 { 1030 new BackupManager(config.getBackendId()).createBackup(this, backupConfig); 1031 } 1032 1033 @Override 1034 public void removeBackup(BackupDirectory backupDirectory, String backupID) throws DirectoryException 1035 { 1036 new BackupManager(config.getBackendId()).removeBackup(backupDirectory, backupID); 1037 } 1038 1039 @Override 1040 public void restoreBackup(RestoreConfig restoreConfig) throws DirectoryException 1041 { 1042 new BackupManager(config.getBackendId()).restoreBackup(this, restoreConfig); 1043 } 1044 1045 @Override 1046 public Set<TreeName> listTrees() 1047 { 1048 try 1049 { 1050 List<String> treeNames = env.getDatabaseNames(); 1051 final Set<TreeName> results = new HashSet<>(treeNames.size()); 1052 for (String treeName : treeNames) 1053 { 1054 results.add(TreeName.valueOf(treeName)); 1055 } 1056 return results; 1057 } 1058 catch (DatabaseException e) 1059 { 1060 throw new StorageRuntimeException(e); 1061 } 1062 } 1063 1064 @Override 1065 public boolean isConfigurationChangeAcceptable(JEBackendCfg newCfg, 1066 List<LocalizableMessage> unacceptableReasons) 1067 { 1068 long newSize = computeSize(newCfg); 1069 long oldSize = computeSize(config); 1070 return (newSize <= oldSize || memQuota.isMemoryAvailable(newSize - oldSize)) 1071 && checkConfigurationDirectories(newCfg, unacceptableReasons); 1072 } 1073 1074 private long computeSize(JEBackendCfg cfg) 1075 { 1076 return cfg.getDBCacheSize() > 0 ? cfg.getDBCacheSize() : memQuota.memPercentToBytes(cfg.getDBCachePercent()); 1077 } 1078 1079 /** 1080 * Checks newly created backend has a valid configuration. 1081 * @param cfg the new configuration 1082 * @param unacceptableReasons the list of accumulated errors and their messages 1083 * @param context the server context 1084 * @return true if newly created backend has a valid configuration 1085 */ 1086 static boolean isConfigurationAcceptable(JEBackendCfg cfg, List<LocalizableMessage> unacceptableReasons, 1087 ServerContext context) 1088 { 1089 if (context != null) 1090 { 1091 MemoryQuota memQuota = context.getMemoryQuota(); 1092 if (cfg.getDBCacheSize() > 0 && !memQuota.isMemoryAvailable(cfg.getDBCacheSize())) 1093 { 1094 unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_SIZE_GREATER_THAN_JVM_HEAP.get( 1095 cfg.getDBCacheSize(), memQuota.getAvailableMemory())); 1096 return false; 1097 } 1098 else if (!memQuota.isMemoryAvailable(memQuota.memPercentToBytes(cfg.getDBCachePercent()))) 1099 { 1100 unacceptableReasons.add(ERR_BACKEND_CONFIG_CACHE_PERCENT_GREATER_THAN_JVM_HEAP.get( 1101 cfg.getDBCachePercent(), memQuota.memBytesToPercent(memQuota.getAvailableMemory()))); 1102 return false; 1103 } 1104 } 1105 return checkConfigurationDirectories(cfg, unacceptableReasons); 1106 } 1107 1108 private static boolean checkConfigurationDirectories(JEBackendCfg cfg, 1109 List<LocalizableMessage> unacceptableReasons) 1110 { 1111 final ConfigChangeResult ccr = new ConfigChangeResult(); 1112 File newBackendDirectory = getBackendDirectory(cfg); 1113 1114 checkDBDirExistsOrCanCreate(newBackendDirectory, ccr, true); 1115 checkDBDirPermissions(cfg.getDBDirectoryPermissions(), cfg.dn(), ccr); 1116 if (!ccr.getMessages().isEmpty()) 1117 { 1118 unacceptableReasons.addAll(ccr.getMessages()); 1119 return false; 1120 } 1121 return true; 1122 } 1123 1124 @Override 1125 public ConfigChangeResult applyConfigurationChange(JEBackendCfg cfg) 1126 { 1127 final ConfigChangeResult ccr = new ConfigChangeResult(); 1128 1129 try 1130 { 1131 File newBackendDirectory = getBackendDirectory(cfg); 1132 1133 // Create the directory if it doesn't exist. 1134 if (!cfg.getDBDirectory().equals(config.getDBDirectory())) 1135 { 1136 checkDBDirExistsOrCanCreate(newBackendDirectory, ccr, false); 1137 if (!ccr.getMessages().isEmpty()) 1138 { 1139 return ccr; 1140 } 1141 1142 ccr.setAdminActionRequired(true); 1143 ccr.addMessage(NOTE_CONFIG_DB_DIR_REQUIRES_RESTART.get(config.getDBDirectory(), cfg.getDBDirectory())); 1144 } 1145 1146 if (!cfg.getDBDirectoryPermissions().equalsIgnoreCase(config.getDBDirectoryPermissions()) 1147 || !cfg.getDBDirectory().equals(config.getDBDirectory())) 1148 { 1149 checkDBDirPermissions(cfg.getDBDirectoryPermissions(), cfg.dn(), ccr); 1150 if (!ccr.getMessages().isEmpty()) 1151 { 1152 return ccr; 1153 } 1154 1155 setDBDirPermissions(newBackendDirectory, cfg.getDBDirectoryPermissions(), cfg.dn(), ccr); 1156 if (!ccr.getMessages().isEmpty()) 1157 { 1158 return ccr; 1159 } 1160 } 1161 registerMonitoredDirectory(cfg); 1162 config = cfg; 1163 } 1164 catch (Exception e) 1165 { 1166 addErrorMessage(ccr, LocalizableMessage.raw(stackTraceToSingleLineString(e))); 1167 } 1168 return ccr; 1169 } 1170 1171 private void registerMonitoredDirectory(JEBackendCfg cfg) 1172 { 1173 diskMonitor.registerMonitoredDirectory( 1174 cfg.getBackendId() + " backend", 1175 getDirectory(), 1176 cfg.getDiskLowThreshold(), 1177 cfg.getDiskFullThreshold(), 1178 this); 1179 } 1180 1181 @Override 1182 public void removeStorageFiles() throws StorageRuntimeException 1183 { 1184 StorageUtils.removeStorageFiles(backendDirectory); 1185 } 1186 1187 @Override 1188 public StorageStatus getStorageStatus() 1189 { 1190 return storageStatus; 1191 } 1192 1193 @Override 1194 public void diskFullThresholdReached(File directory, long thresholdInBytes) { 1195 storageStatus = statusWhenDiskSpaceFull(directory, thresholdInBytes, config.getBackendId()); 1196 } 1197 1198 @Override 1199 public void diskLowThresholdReached(File directory, long thresholdInBytes) { 1200 storageStatus = statusWhenDiskSpaceLow(directory, thresholdInBytes, config.getBackendId()); 1201 } 1202 1203 @Override 1204 public void diskSpaceRestored(File directory, long lowThresholdInBytes, long fullThresholdInBytes) { 1205 storageStatus = StorageStatus.working(); 1206 } 1207 1208 private static void setData(final DatabaseEntry dbEntry, final ByteSequence bs) 1209 { 1210 dbEntry.setData(bs != null ? bs.toByteArray() : null); 1211 } 1212 1213 private static DatabaseEntry db(final ByteSequence bs) 1214 { 1215 return new DatabaseEntry(bs != null ? bs.toByteArray() : null); 1216 } 1217 1218 private static ByteString valueToBytes(final DatabaseEntry dbValue, boolean isDefined) 1219 { 1220 if (isDefined) 1221 { 1222 return ByteString.wrap(dbValue.getData()); 1223 } 1224 return null; 1225 } 1226}