001/* 002 * The contents of this file are subject to the terms of the Common Development and 003 * Distribution License (the License). You may not use this file except in compliance with the 004 * License. 005 * 006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the 007 * specific language governing permission and limitations under the License. 008 * 009 * When distributing Covered Software, include this CDDL Header Notice in each file and include 010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL 011 * Header, with the fields enclosed by brackets [] replaced by your own identifying 012 * information: "Portions Copyright [year] [name of copyright owner]". 013 * 014 * Copyright 2006-2010 Sun Microsystems, Inc. 015 * Portions Copyright 2011-2016 ForgeRock AS. 016 */ 017package org.opends.server.tools; 018 019import static org.opends.messages.ToolMessages.*; 020import static org.opends.server.config.ConfigConstants.*; 021import static org.opends.server.util.StaticUtils.*; 022import static com.forgerock.opendj.cli.ArgumentConstants.*; 023import static com.forgerock.opendj.cli.CommonArguments.*; 024import static com.forgerock.opendj.cli.Utils.*; 025 026import java.io.File; 027import java.io.OutputStream; 028import java.io.PrintStream; 029import java.util.ArrayList; 030import java.util.HashSet; 031import java.util.List; 032import java.util.Random; 033import java.util.Set; 034 035import org.forgerock.i18n.LocalizableMessage; 036import org.forgerock.i18n.slf4j.LocalizedLogger; 037import org.forgerock.opendj.ldap.DN; 038import org.forgerock.opendj.ldap.schema.AttributeType; 039import org.forgerock.opendj.server.config.server.BackendCfg; 040import org.opends.server.api.Backend; 041import org.opends.server.api.Backend.BackendOperation; 042import org.opends.server.api.plugin.PluginType; 043import org.opends.server.core.DirectoryServer; 044import org.opends.server.core.DirectoryServer.InitializationBuilder; 045import org.opends.server.core.LockFileManager; 046import org.opends.server.loggers.JDKLogging; 047import org.opends.server.protocols.ldap.LDAPAttribute; 048import org.opends.server.tasks.ImportTask; 049import org.opends.server.tools.makeldif.TemplateFile; 050import org.opends.server.tools.tasks.TaskTool; 051import org.opends.server.types.DirectoryException; 052import org.opends.server.types.ExistingFileBehavior; 053import org.opends.server.types.InitializationException; 054import org.opends.server.types.LDIFImportConfig; 055import org.opends.server.types.LDIFImportResult; 056import org.opends.server.types.NullOutputStream; 057import org.opends.server.types.RawAttribute; 058import org.opends.server.types.SearchFilter; 059import org.opends.server.util.cli.LDAPConnectionArgumentParser; 060 061import com.forgerock.opendj.cli.Argument; 062import com.forgerock.opendj.cli.ArgumentException; 063import com.forgerock.opendj.cli.BooleanArgument; 064import com.forgerock.opendj.cli.ClientException; 065import com.forgerock.opendj.cli.IntegerArgument; 066import com.forgerock.opendj.cli.StringArgument; 067 068/** 069 * This program provides a utility that may be used to import the contents of an 070 * LDIF file into a Directory Server backend. This will be a process that is 071 * intended to run separate from Directory Server and not internally within the 072 * server process (e.g., via the tasks interface). 073 */ 074public class ImportLDIF extends TaskTool { 075 private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass(); 076 077 /** The buffer size that should be used when reading data from LDIF. */ 078 private static final int LDIF_BUFFER_SIZE = 1048576; 079 080 /** 081 * The main method for ImportLDIF tool. 082 * 083 * @param args The command-line arguments provided to this program. 084 */ 085 public static void main(String[] args) 086 { 087 int retCode = mainImportLDIF(args, true, System.out, System.err); 088 if(retCode != 0) 089 { 090 System.exit(filterExitCode(retCode)); 091 } 092 } 093 094 /** 095 * Processes the command-line arguments and invokes the import process. 096 * 097 * @param args The command-line arguments provided to this 098 * program. 099 * @param initializeServer Indicates whether to initialize the server. 100 * @param outStream The output stream to use for standard output, or 101 * {@code null} if standard output is not needed. 102 * @param errStream The output stream to use for standard error, or 103 * {@code null} if standard error is not needed. 104 * 105 * @return The error code. 106 */ 107 public static int mainImportLDIF(String[] args, boolean initializeServer, 108 OutputStream outStream, 109 OutputStream errStream) 110 { 111 ImportLDIF tool = new ImportLDIF(); 112 return tool.process(args, initializeServer, outStream, errStream); 113 } 114 115 /** Define the command-line arguments that may be used with this program. */ 116 private BooleanArgument countRejects; 117 private BooleanArgument isCompressed; 118 private BooleanArgument isEncrypted; 119 private BooleanArgument overwrite; 120 private BooleanArgument quietMode; 121 private BooleanArgument skipSchemaValidation; 122 private BooleanArgument clearBackend; 123 private IntegerArgument randomSeed; 124 private StringArgument backendID; 125 private StringArgument configFile; 126 private StringArgument excludeAttributeStrings; 127 private StringArgument excludeBranchStrings; 128 private StringArgument excludeFilterStrings; 129 private StringArgument includeAttributeStrings; 130 private StringArgument includeBranchStrings; 131 private StringArgument includeFilterStrings; 132 private StringArgument ldifFiles; 133 private StringArgument rejectFile; 134 private StringArgument skipFile; 135 private StringArgument templateFile; 136 private BooleanArgument skipDNValidation; 137 private IntegerArgument threadCount; 138 private StringArgument tmpDirectory; 139 140 private int process(String[] args, boolean initializeServer, 141 OutputStream outStream, OutputStream errStream) { 142 PrintStream out = NullOutputStream.wrapOrNullStream(outStream); 143 PrintStream err = NullOutputStream.wrapOrNullStream(errStream); 144 JDKLogging.disableLogging(); 145 146 // FIXME -- Need to add a mechanism for verifying the file signature. 147 148 // Create the command-line argument parser for use with this program. 149 LDAPConnectionArgumentParser argParser = 150 createArgParser("org.opends.server.tools.ImportLDIF", INFO_LDIFIMPORT_TOOL_DESCRIPTION.get()); 151 argParser.setShortToolDescription(REF_SHORT_DESC_IMPORT_LDIF.get()); 152 153 // Initialize all the command-line argument types and register them with the 154 // parser. 155 try 156 { 157 createArguments(argParser); 158 } 159 catch (ArgumentException ae) 160 { 161 printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage())); 162 return 1; 163 } 164 165 // Init the default values so that they can appear also on the usage. 166 argParser.getArguments().initArgumentsWithConfiguration(argParser); 167 168 // Parse the command-line arguments provided to this program. 169 try 170 { 171 argParser.parseArguments(args); 172 validateTaskArgs(); 173 } 174 catch (ArgumentException ae) 175 { 176 argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage())); 177 return 1; 178 } 179 catch (ClientException ce) 180 { 181 // No need to display the usage since the problem comes with a provided value. 182 printWrappedText(err, ce.getMessageObject()); 183 return 1; 184 } 185 186 if (argParser.usageOrVersionDisplayed()) 187 { 188 return 0; 189 } 190 191 if (skipDNValidation.isPresent()) 192 { 193 printWrappedText(err, ERR_DEPRECATED_SKIP_DN_VALIDATION.get()); 194 // continue 195 } 196 197 // Make sure that either the "ldifFile" argument or the "templateFile" 198 // argument was provided, but not both. 199 if (ldifFiles.isPresent()) 200 { 201 if (templateFile.isPresent()) 202 { 203 printWrappedText(err, conflictingArgsErrorMessage(ldifFiles, templateFile)); 204 return 1; 205 } 206 } 207 else if (! templateFile.isPresent()) 208 { 209 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_REQUIRED_ARGUMENT.get( 210 ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier())); 211 return 1; 212 } 213 214 // Make sure that either the "includeBranchStrings" argument or the 215 // "backendID" argument was provided. 216 if(!includeBranchStrings.isPresent() && !backendID.isPresent()) 217 { 218 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get( 219 includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier())); 220 return 1; 221 } 222 223 // Count rejects option requires the ability to return result codes 224 // which are potentially greater than 1. This is not supported by 225 // the task framework. 226 if (countRejects.isPresent() && argParser.connectionArgumentsPresent()) 227 { 228 argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_COUNT_REJECTS_REQUIRES_OFFLINE.get( 229 countRejects.getLongIdentifier())); 230 return 1; 231 } 232 233 // Don't write non-error messages to console if quite 234 if (quietMode.isPresent()) { 235 out = new PrintStream(NullOutputStream.instance()); 236 } 237 238 // Checks the version - if upgrade required, the tool is unusable 239 try 240 { 241 checkVersion(); 242 } 243 catch (InitializationException e) 244 { 245 printWrappedText(err, e.getMessage()); 246 return 1; 247 } 248 249 return process(argParser, initializeServer, out, err); 250 } 251 252 private void createArguments(LDAPConnectionArgumentParser argParser) throws ArgumentException 253 { 254 configFile = 255 StringArgument.builder("configFile") 256 .shortIdentifier('f') 257 .description(INFO_DESCRIPTION_CONFIG_FILE.get()) 258 .hidden() 259 .required() 260 .valuePlaceholder(INFO_CONFIGFILE_PLACEHOLDER.get()) 261 .buildAndAddToParser(argParser); 262 ldifFiles = 263 StringArgument.builder(OPTION_LONG_LDIF_FILE) 264 .shortIdentifier(OPTION_SHORT_LDIF_FILE) 265 .description(INFO_LDIFIMPORT_DESCRIPTION_LDIF_FILE.get()) 266 .multiValued() 267 .valuePlaceholder(INFO_LDIFFILE_PLACEHOLDER.get()) 268 .buildAndAddToParser(argParser); 269 templateFile = 270 StringArgument.builder("templateFile") 271 .shortIdentifier('A') 272 .description(INFO_LDIFIMPORT_DESCRIPTION_TEMPLATE_FILE.get()) 273 .valuePlaceholder(INFO_TEMPLATE_FILE_PLACEHOLDER.get()) 274 .buildAndAddToParser(argParser); 275 backendID = 276 StringArgument.builder("backendID") 277 .shortIdentifier('n') 278 .description(INFO_LDIFIMPORT_DESCRIPTION_BACKEND_ID.get()) 279 .valuePlaceholder(INFO_BACKENDNAME_PLACEHOLDER.get()) 280 .buildAndAddToParser(argParser); 281 clearBackend = 282 BooleanArgument.builder("clearBackend") 283 .shortIdentifier('F') 284 .description(INFO_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND.get()) 285 .buildAndAddToParser(argParser); 286 includeBranchStrings = 287 StringArgument.builder("includeBranch") 288 .shortIdentifier('b') 289 .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_BRANCH.get()) 290 .multiValued() 291 .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get()) 292 .buildAndAddToParser(argParser); 293 excludeBranchStrings = 294 StringArgument.builder("excludeBranch") 295 .shortIdentifier('B') 296 .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_BRANCH.get()) 297 .multiValued() 298 .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get()) 299 .buildAndAddToParser(argParser); 300 includeAttributeStrings = 301 StringArgument.builder("includeAttribute") 302 .shortIdentifier('i') 303 .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_ATTRIBUTE.get()) 304 .multiValued() 305 .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get()) 306 .buildAndAddToParser(argParser); 307 excludeAttributeStrings = 308 StringArgument.builder("excludeAttribute") 309 .shortIdentifier('e') 310 .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_ATTRIBUTE.get()) 311 .multiValued() 312 .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get()) 313 .buildAndAddToParser(argParser); 314 includeFilterStrings = 315 StringArgument.builder("includeFilter") 316 .shortIdentifier('I') 317 .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_FILTER.get()) 318 .multiValued() 319 .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get()) 320 .buildAndAddToParser(argParser); 321 excludeFilterStrings = 322 StringArgument.builder("excludeFilter") 323 .shortIdentifier('E') 324 .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_FILTER.get()) 325 .multiValued() 326 .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get()) 327 .buildAndAddToParser(argParser); 328 rejectFile = 329 StringArgument.builder("rejectFile") 330 .shortIdentifier('R') 331 .description(INFO_LDIFIMPORT_DESCRIPTION_REJECT_FILE.get()) 332 .valuePlaceholder(INFO_REJECT_FILE_PLACEHOLDER.get()) 333 .buildAndAddToParser(argParser); 334 skipFile = 335 StringArgument.builder("skipFile") 336 .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_FILE.get()) 337 .valuePlaceholder(INFO_SKIP_FILE_PLACEHOLDER.get()) 338 .buildAndAddToParser(argParser); 339 overwrite = 340 BooleanArgument.builder("overwrite") 341 .shortIdentifier('O') 342 .description(INFO_LDIFIMPORT_DESCRIPTION_OVERWRITE.get()) 343 .buildAndAddToParser(argParser); 344 randomSeed = 345 IntegerArgument.builder(OPTION_LONG_RANDOM_SEED) 346 .shortIdentifier(OPTION_SHORT_RANDOM_SEED) 347 .description(INFO_LDIFIMPORT_DESCRIPTION_RANDOM_SEED.get()) 348 .defaultValue(0) 349 .valuePlaceholder(INFO_SEED_PLACEHOLDER.get()) 350 .buildAndAddToParser(argParser); 351 skipSchemaValidation = 352 BooleanArgument.builder("skipSchemaValidation") 353 .shortIdentifier('S') 354 .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get()) 355 .buildAndAddToParser(argParser); 356 skipDNValidation = 357 BooleanArgument.builder("skipDNValidation") 358 .description(INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get()) 359 .hidden() 360 .buildAndAddToParser(argParser); 361 threadCount = 362 IntegerArgument.builder("threadCount") 363 .description(INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get()) 364 .lowerBound(1) 365 .defaultValue(0) 366 .valuePlaceholder(INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER.get()) 367 .buildAndAddToParser(argParser); 368 tmpDirectory = 369 StringArgument.builder("tmpdirectory") 370 .description(INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY.get()) 371 .defaultValue("import-tmp") 372 .valuePlaceholder(INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER.get()) 373 .buildAndAddToParser(argParser); 374 countRejects = 375 BooleanArgument.builder("countRejects") 376 .description(INFO_LDIFIMPORT_DESCRIPTION_COUNT_REJECTS.get()) 377 .buildAndAddToParser(argParser); 378 isCompressed = 379 BooleanArgument.builder("isCompressed") 380 .shortIdentifier('c') 381 .description(INFO_LDIFIMPORT_DESCRIPTION_IS_COMPRESSED.get()) 382 .buildAndAddToParser(argParser); 383 isEncrypted = 384 BooleanArgument.builder("isEncrypted") 385 .shortIdentifier('y') 386 .description(INFO_LDIFIMPORT_DESCRIPTION_IS_ENCRYPTED.get()) 387 .hidden() //See issue #27 388 .buildAndAddToParser(argParser); 389 quietMode = 390 BooleanArgument.builder(OPTION_LONG_QUIET) 391 .shortIdentifier(OPTION_SHORT_QUIET) 392 .description(INFO_LDIFIMPORT_DESCRIPTION_QUIET.get()) 393 .buildAndAddToParser(argParser); 394 395 final BooleanArgument displayUsage = showUsageArgument(); 396 argParser.addArgument(displayUsage); 397 argParser.setUsageArgument(displayUsage); 398 } 399 400 @Override 401 public void addTaskAttributes(List<RawAttribute> attributes) 402 { 403 // Required attributes 404 addAttribute(attributes, ATTR_IMPORT_LDIF_FILE, ldifFiles.getValues()); 405 addAttribute(attributes, ATTR_IMPORT_TEMPLATE_FILE, templateFile.getValue()); 406 addAttribute(attributes, ATTR_IMPORT_RANDOM_SEED, randomSeed.getValue()); 407 addAttribute(attributes, ATTR_IMPORT_THREAD_COUNT, threadCount.getValue()); 408 409 // Optional attributes 410 addAttribute2(attributes, ATTR_IMPORT_BACKEND_ID, backendID); 411 addAttribute(attributes, ATTR_IMPORT_INCLUDE_ATTRIBUTE, includeAttributeStrings.getValues()); 412 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_ATTRIBUTE, excludeAttributeStrings.getValues()); 413 addAttribute(attributes, ATTR_IMPORT_INCLUDE_FILTER, includeFilterStrings.getValues()); 414 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_FILTER, excludeFilterStrings.getValues()); 415 addAttribute(attributes, ATTR_IMPORT_INCLUDE_BRANCH, includeBranchStrings.getValues()); 416 addAttribute(attributes, ATTR_IMPORT_EXCLUDE_BRANCH, excludeBranchStrings.getValues()); 417 addAttribute2(attributes, ATTR_IMPORT_REJECT_FILE, rejectFile); 418 addAttribute2(attributes, ATTR_IMPORT_SKIP_FILE, skipFile); 419 addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite); 420 addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation); 421 addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory); 422 addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed); 423 addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted); 424 addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend); 425 } 426 427 private void addAttribute(List<RawAttribute> attributes, String attrName, String value) 428 { 429 if (value != null) 430 { 431 attributes.add(new LDAPAttribute(attrName, value)); 432 } 433 } 434 435 private void addAttribute2(List<RawAttribute> attributes, String attrName, Argument arg) 436 { 437 final String value = arg.getValue(); 438 if (value != null && !value.equals(arg.getDefaultValue())) 439 { 440 attributes.add(new LDAPAttribute(attrName, value)); 441 } 442 } 443 444 private void addAttribute(List<RawAttribute> attributes, String attrName, List<String> attrValues) 445 { 446 if (attrValues != null && !attrValues.isEmpty()) 447 { 448 attributes.add(new LDAPAttribute(attrName, attrValues)); 449 } 450 } 451 452 @Override 453 public String getTaskObjectclass() { 454 return "ds-task-import"; 455 } 456 457 @Override 458 public Class<?> getTaskClass() { 459 return ImportTask.class; 460 } 461 462 @Override 463 protected int processLocal(boolean initializeServer, PrintStream out, PrintStream err) { 464 if (initializeServer) 465 { 466 try 467 { 468 InitializationBuilder initBuilder = new DirectoryServer.InitializationBuilder(configFile.getValue()) 469 .requireCryptoServices() 470 .requireUserPlugins(PluginType.LDIF_IMPORT); 471 if (!quietMode.isPresent()) 472 { 473 initBuilder.requireErrorAndDebugLogPublisher(out, err); 474 } 475 initBuilder.initialize(); 476 } 477 catch (InitializationException e) 478 { 479 printWrappedText(err, ERR_CANNOT_INITIALIZE_SERVER_COMPONENTS.get(e.getLocalizedMessage())); 480 return 1; 481 } 482 } 483 484 // See if there were any user-defined sets of include/exclude attributes or 485 // filters. If so, then process them. 486 HashSet<AttributeType> excludeAttributes; 487 boolean excludeAllUserAttributes = false; 488 boolean excludeAllOperationalAttributes = false; 489 if (excludeAttributeStrings == null) 490 { 491 excludeAttributes = null; 492 } 493 else 494 { 495 excludeAttributes = new HashSet<>(); 496 for (String attrName : excludeAttributeStrings.getValues()) 497 { 498 String lowerName = attrName.toLowerCase(); 499 if (lowerName.equals("*")) 500 { 501 excludeAllUserAttributes = true; 502 } 503 else if (lowerName.equals("+")) 504 { 505 excludeAllOperationalAttributes = true; 506 } 507 else 508 { 509 excludeAttributes.add(DirectoryServer.getSchema().getAttributeType(attrName)); 510 } 511 } 512 } 513 514 HashSet<AttributeType> includeAttributes; 515 boolean includeAllUserAttributes = false; 516 boolean includeAllOperationalAttributes = false; 517 if (includeAttributeStrings == null) 518 { 519 includeAttributes = null; 520 } 521 else 522 { 523 includeAttributes = new HashSet<>(); 524 for (String attrName : includeAttributeStrings.getValues()) 525 { 526 String lowerName = attrName.toLowerCase(); 527 if (lowerName.equals("*")) 528 { 529 includeAllUserAttributes = true; 530 } 531 else if (lowerName.equals("+")) 532 { 533 includeAllOperationalAttributes = true; 534 } 535 else 536 { 537 includeAttributes.add(DirectoryServer.getSchema().getAttributeType(attrName)); 538 } 539 } 540 } 541 542 ArrayList<SearchFilter> excludeFilters; 543 if (excludeFilterStrings == null) 544 { 545 excludeFilters = null; 546 } 547 else 548 { 549 excludeFilters = new ArrayList<>(); 550 for (String filterString : excludeFilterStrings.getValues()) 551 { 552 try 553 { 554 excludeFilters.add(SearchFilter.createFilterFromString(filterString)); 555 } 556 catch (DirectoryException de) 557 { 558 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject()); 559 return 1; 560 } 561 catch (Exception e) 562 { 563 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, getExceptionMessage(e)); 564 return 1; 565 } 566 } 567 } 568 569 ArrayList<SearchFilter> includeFilters; 570 if (includeFilterStrings == null) 571 { 572 includeFilters = null; 573 } 574 else 575 { 576 includeFilters = new ArrayList<>(); 577 for (String filterString : includeFilterStrings.getValues()) 578 { 579 try 580 { 581 includeFilters.add(SearchFilter.createFilterFromString(filterString)); 582 } 583 catch (DirectoryException de) 584 { 585 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject()); 586 return 1; 587 } 588 catch (Exception e) 589 { 590 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, getExceptionMessage(e)); 591 return 1; 592 } 593 } 594 } 595 596 // Get information about the backends defined in the server. Iterate 597 // through them, finding the one backend into which the LDIF should be 598 // imported and finding backends with subordinate base DNs that should be 599 // excluded from the import. 600 Backend<?> backend = null; 601 Set<DN> defaultIncludeBranches = null; 602 Set<DN> excludeBranches = new HashSet<>(); 603 Set<DN> includeBranches = new HashSet<>(); 604 605 if (includeBranchStrings.isPresent()) 606 { 607 for (String s : includeBranchStrings.getValues()) 608 { 609 DN includeBranch; 610 try 611 { 612 includeBranch = DN.valueOf(s); 613 } 614 catch (Exception e) 615 { 616 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e)); 617 return 1; 618 } 619 620 includeBranches.add(includeBranch); 621 } 622 } 623 624 ArrayList<Backend<?>> backendList = new ArrayList<>(); 625 ArrayList<BackendCfg> entryList = new ArrayList<>(); 626 ArrayList<List<DN>> dnList = new ArrayList<>(); 627 int code = BackendToolUtils.getBackends(backendList, entryList, dnList); 628 if (code != 0) 629 { 630 return code; 631 } 632 633 int numBackends = backendList.size(); 634 for (int i=0; i < numBackends; i++) 635 { 636 Backend<?> b = backendList.get(i); 637 638 if(backendID.isPresent()) 639 { 640 if (! backendID.getValue().equals(b.getBackendID())) 641 { 642 continue; 643 } 644 } 645 else 646 { 647 if (!useBackend(includeBranches, dnList.get(i))) 648 { 649 continue; 650 } 651 } 652 653 if (backend == null) 654 { 655 backend = b; 656 defaultIncludeBranches = new HashSet<>(dnList.get(i)); 657 } 658 else 659 { 660 logger.error(ERR_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID); 661 return 1; 662 } 663 } 664 665 if (backend == null) 666 { 667 logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID); 668 return 1; 669 } 670 else if (!backend.supports(BackendOperation.LDIF_IMPORT)) 671 { 672 logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID.getValue()); 673 return 1; 674 } 675 676 for (List<DN> baseList : dnList) 677 { 678 for (DN baseDN : baseList) 679 { 680 for (DN importBase : defaultIncludeBranches) 681 { 682 if (!baseDN.equals(importBase) && baseDN.isSubordinateOrEqualTo(importBase)) 683 { 684 excludeBranches.add(baseDN); 685 break; 686 } 687 } 688 } 689 } 690 691 for (String s : excludeBranchStrings.getValues()) 692 { 693 DN excludeBranch; 694 try 695 { 696 excludeBranch = DN.valueOf(s); 697 } 698 catch (Exception e) 699 { 700 logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e)); 701 return 1; 702 } 703 704 excludeBranches.add(excludeBranch); 705 } 706 707 if (! includeBranchStrings.isPresent()) 708 { 709 includeBranches = defaultIncludeBranches; 710 } 711 else 712 { 713 // Make sure the selected backend will handle all the include branches 714 for(DN includeBranch : includeBranches) 715 { 716 if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches, 717 excludeBranches)) 718 { 719 logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backendID.getValue()); 720 return 1; 721 } 722 } 723 } 724 725 // See if the data should be read from LDIF files or generated via MakeLDIF. 726 LDIFImportConfig importConfig; 727 if (ldifFiles.isPresent()) 728 { 729 ArrayList<String> fileList = new ArrayList<>(ldifFiles.getValues()); 730 int badFileCount = 0; 731 for (String pathname : fileList) 732 { 733 File f = new File(pathname); 734 if (!f.canRead()) 735 { 736 logger.error(ERR_LDIFIMPORT_CANNOT_READ_FILE, pathname); 737 badFileCount++; 738 } 739 } 740 if (badFileCount > 0) 741 { 742 return 1; 743 } 744 importConfig = new LDIFImportConfig(fileList); 745 } 746 else 747 { 748 Random random = newRandom(); 749 750 String resourcePath = DirectoryServer.getInstanceRoot() + File.separator + 751 PATH_MAKELDIF_RESOURCE_DIR; 752 TemplateFile tf = new TemplateFile(resourcePath, random); 753 754 ArrayList<LocalizableMessage> warnings = new ArrayList<>(); 755 try 756 { 757 tf.parse(templateFile.getValue(), warnings); 758 } 759 catch (Exception e) 760 { 761 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile.getValue(), e.getMessage()); 762 return 1; 763 } 764 765 importConfig = new LDIFImportConfig(tf); 766 } 767 768 // Create the LDIF import configuration to use when reading the LDIF. 769 importConfig.setCompressed(isCompressed.isPresent()); 770 importConfig.setClearBackend(clearBackend.isPresent()); 771 importConfig.setEncrypted(isEncrypted.isPresent()); 772 importConfig.setExcludeAttributes(excludeAttributes); 773 importConfig.setExcludeBranches(excludeBranches); 774 importConfig.setExcludeFilters(excludeFilters); 775 importConfig.setIncludeAttributes(includeAttributes); 776 importConfig.setIncludeBranches(includeBranches); 777 importConfig.setIncludeFilters(includeFilters); 778 importConfig.setValidateSchema(!skipSchemaValidation.isPresent()); 779 importConfig.setTmpDirectory(tmpDirectory.getValue()); 780 781 try 782 { 783 importConfig.setThreadCount(threadCount.getIntValue()); 784 } 785 catch(Exception e) 786 { 787 logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT, 788 threadCount.getValue(), e.getMessage()); 789 return 1; 790 } 791 792 importConfig.setBufferSize(LDIF_BUFFER_SIZE); 793 importConfig.setExcludeAllUserAttributes(excludeAllUserAttributes); 794 importConfig.setExcludeAllOperationalAttributes(excludeAllOperationalAttributes); 795 importConfig.setIncludeAllOpAttributes(includeAllOperationalAttributes); 796 importConfig.setIncludeAllUserAttributes(includeAllUserAttributes); 797 798 // FIXME -- Should this be conditional? 799 importConfig.setInvokeImportPlugins(true); 800 801 if (rejectFile != null) 802 { 803 try 804 { 805 ExistingFileBehavior existingBehavior = overwrite.isPresent() 806 ? ExistingFileBehavior.OVERWRITE 807 : ExistingFileBehavior.APPEND; 808 809 importConfig.writeRejectedEntries(rejectFile.getValue(), 810 existingBehavior); 811 } 812 catch (Exception e) 813 { 814 logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile.getValue(), getExceptionMessage(e)); 815 return 1; 816 } 817 } 818 819 if (skipFile != null) 820 { 821 try 822 { 823 ExistingFileBehavior existingBehavior = overwrite.isPresent() 824 ? ExistingFileBehavior.OVERWRITE 825 : ExistingFileBehavior.APPEND; 826 827 importConfig.writeSkippedEntries(skipFile.getValue(), 828 existingBehavior); 829 } 830 catch (Exception e) 831 { 832 logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile.getValue(), getExceptionMessage(e)); 833 return 1; 834 } 835 } 836 837 // Get the set of base DNs for the backend as an array. 838 DN[] baseDNs = new DN[defaultIncludeBranches.size()]; 839 defaultIncludeBranches.toArray(baseDNs); 840 841 // Acquire an exclusive lock for the backend. 842 try 843 { 844 String lockFile = LockFileManager.getBackendLockFileName(backend); 845 StringBuilder failureReason = new StringBuilder(); 846 if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason)) 847 { 848 logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason); 849 return 1; 850 } 851 } 852 catch (Exception e) 853 { 854 logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e)); 855 return 1; 856 } 857 858 // Launch the import. 859 int retCode = 0; 860 try 861 { 862 LDIFImportResult importResult = 863 backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext()); 864 if (countRejects.isPresent()) 865 { 866 if (importResult.getEntriesRejected() > Integer.MAX_VALUE) 867 { 868 retCode = Integer.MAX_VALUE; 869 } 870 else 871 { 872 retCode = (int) importResult.getEntriesRejected(); 873 } 874 } 875 } 876 catch (DirectoryException de) 877 { 878 LocalizableMessage msg; 879 msg = de.getMessageObject(); 880 logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg)); 881 retCode = 1; 882 } 883 catch (Exception e) 884 { 885 logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e)); 886 retCode = 1; 887 } 888 889 // Release the exclusive lock on the backend. 890 try 891 { 892 String lockFile = LockFileManager.getBackendLockFileName(backend); 893 StringBuilder failureReason = new StringBuilder(); 894 if (! LockFileManager.releaseLock(lockFile, failureReason)) 895 { 896 logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason); 897 retCode = 1; 898 } 899 } 900 catch (Exception e) 901 { 902 logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e)); 903 retCode = 1; 904 } 905 906 // Clean up after the import by closing the import config. 907 importConfig.close(); 908 return retCode; 909 } 910 911 @Override 912 protected void cleanup() 913 { 914 DirectoryServer.shutdownBackends(); 915 } 916 917 private boolean useBackend(Set<DN> includeBranches, List<DN> dnlist) 918 { 919 for (DN baseDN : dnlist) 920 { 921 for (DN includeDN : includeBranches) 922 { 923 if (baseDN.isSuperiorOrEqualTo(includeDN)) 924 { 925 return true; 926 } 927 } 928 } 929 return false; 930 } 931 932 private Random newRandom() 933 { 934 if (randomSeed.isPresent()) 935 { 936 try 937 { 938 return new Random(randomSeed.getIntValue()); 939 } 940 catch (Exception ignored) 941 { 942 // ignore 943 } 944 } 945 return new Random(); 946 } 947 948 @Override 949 public String getTaskId() { 950 // NYI. 951 return null; 952 } 953}