001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2006-2010 Sun Microsystems, Inc.
015 * Portions Copyright 2011-2016 ForgeRock AS.
016 */
017package org.opends.server.tools;
018
019import static org.opends.messages.ToolMessages.*;
020import static org.opends.server.config.ConfigConstants.*;
021import static org.opends.server.util.StaticUtils.*;
022import static com.forgerock.opendj.cli.ArgumentConstants.*;
023import static com.forgerock.opendj.cli.CommonArguments.*;
024import static com.forgerock.opendj.cli.Utils.*;
025
026import java.io.File;
027import java.io.OutputStream;
028import java.io.PrintStream;
029import java.util.ArrayList;
030import java.util.HashSet;
031import java.util.List;
032import java.util.Random;
033import java.util.Set;
034
035import org.forgerock.i18n.LocalizableMessage;
036import org.forgerock.i18n.slf4j.LocalizedLogger;
037import org.forgerock.opendj.ldap.DN;
038import org.forgerock.opendj.ldap.schema.AttributeType;
039import org.forgerock.opendj.server.config.server.BackendCfg;
040import org.opends.server.api.Backend;
041import org.opends.server.api.Backend.BackendOperation;
042import org.opends.server.api.plugin.PluginType;
043import org.opends.server.core.DirectoryServer;
044import org.opends.server.core.DirectoryServer.InitializationBuilder;
045import org.opends.server.core.LockFileManager;
046import org.opends.server.loggers.JDKLogging;
047import org.opends.server.protocols.ldap.LDAPAttribute;
048import org.opends.server.tasks.ImportTask;
049import org.opends.server.tools.makeldif.TemplateFile;
050import org.opends.server.tools.tasks.TaskTool;
051import org.opends.server.types.DirectoryException;
052import org.opends.server.types.ExistingFileBehavior;
053import org.opends.server.types.InitializationException;
054import org.opends.server.types.LDIFImportConfig;
055import org.opends.server.types.LDIFImportResult;
056import org.opends.server.types.NullOutputStream;
057import org.opends.server.types.RawAttribute;
058import org.opends.server.types.SearchFilter;
059import org.opends.server.util.cli.LDAPConnectionArgumentParser;
060
061import com.forgerock.opendj.cli.Argument;
062import com.forgerock.opendj.cli.ArgumentException;
063import com.forgerock.opendj.cli.BooleanArgument;
064import com.forgerock.opendj.cli.ClientException;
065import com.forgerock.opendj.cli.IntegerArgument;
066import com.forgerock.opendj.cli.StringArgument;
067
068/**
069 * This program provides a utility that may be used to import the contents of an
070 * LDIF file into a Directory Server backend.  This will be a process that is
071 * intended to run separate from Directory Server and not internally within the
072 * server process (e.g., via the tasks interface).
073 */
074public class ImportLDIF extends TaskTool {
075  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
076
077  /** The buffer size that should be used when reading data from LDIF. */
078  private static final int LDIF_BUFFER_SIZE = 1048576;
079
080  /**
081   * The main method for ImportLDIF tool.
082   *
083   * @param  args  The command-line arguments provided to this program.
084   */
085  public static void main(String[] args)
086  {
087    int retCode = mainImportLDIF(args, true, System.out, System.err);
088    if(retCode != 0)
089    {
090      System.exit(filterExitCode(retCode));
091    }
092  }
093
094  /**
095   * Processes the command-line arguments and invokes the import process.
096   *
097   * @param  args              The command-line arguments provided to this
098   *                           program.
099   * @param  initializeServer  Indicates whether to initialize the server.
100   * @param  outStream         The output stream to use for standard output, or
101   *                           {@code null} if standard output is not needed.
102   * @param  errStream         The output stream to use for standard error, or
103   *                           {@code null} if standard error is not needed.
104   *
105   * @return The error code.
106   */
107  public static int mainImportLDIF(String[] args, boolean initializeServer,
108                                   OutputStream outStream,
109                                   OutputStream errStream)
110  {
111    ImportLDIF tool = new ImportLDIF();
112    return tool.process(args, initializeServer, outStream, errStream);
113  }
114
115  /** Define the command-line arguments that may be used with this program. */
116  private BooleanArgument countRejects;
117  private BooleanArgument isCompressed;
118  private BooleanArgument isEncrypted;
119  private BooleanArgument overwrite;
120  private BooleanArgument quietMode;
121  private BooleanArgument skipSchemaValidation;
122  private BooleanArgument clearBackend;
123  private IntegerArgument randomSeed;
124  private StringArgument  backendID;
125  private StringArgument  configFile;
126  private StringArgument  excludeAttributeStrings;
127  private StringArgument  excludeBranchStrings;
128  private StringArgument  excludeFilterStrings;
129  private StringArgument  includeAttributeStrings;
130  private StringArgument  includeBranchStrings;
131  private StringArgument  includeFilterStrings;
132  private StringArgument  ldifFiles;
133  private StringArgument  rejectFile;
134  private StringArgument  skipFile;
135  private StringArgument  templateFile;
136  private BooleanArgument skipDNValidation;
137  private IntegerArgument threadCount;
138  private StringArgument  tmpDirectory;
139
140  private int process(String[] args, boolean initializeServer,
141                      OutputStream outStream, OutputStream errStream) {
142    PrintStream out = NullOutputStream.wrapOrNullStream(outStream);
143    PrintStream err = NullOutputStream.wrapOrNullStream(errStream);
144    JDKLogging.disableLogging();
145
146    // FIXME -- Need to add a mechanism for verifying the file signature.
147
148    // Create the command-line argument parser for use with this program.
149    LDAPConnectionArgumentParser argParser =
150            createArgParser("org.opends.server.tools.ImportLDIF", INFO_LDIFIMPORT_TOOL_DESCRIPTION.get());
151    argParser.setShortToolDescription(REF_SHORT_DESC_IMPORT_LDIF.get());
152
153    // Initialize all the command-line argument types and register them with the
154    // parser.
155    try
156    {
157      createArguments(argParser);
158    }
159    catch (ArgumentException ae)
160    {
161      printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
162      return 1;
163    }
164
165    // Init the default values so that they can appear also on the usage.
166    argParser.getArguments().initArgumentsWithConfiguration(argParser);
167
168    // Parse the command-line arguments provided to this program.
169    try
170    {
171      argParser.parseArguments(args);
172      validateTaskArgs();
173    }
174    catch (ArgumentException ae)
175    {
176      argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
177      return 1;
178    }
179    catch (ClientException ce)
180    {
181      // No need to display the usage since the problem comes with a provided value.
182      printWrappedText(err, ce.getMessageObject());
183      return 1;
184    }
185
186    if (argParser.usageOrVersionDisplayed())
187    {
188      return 0;
189    }
190
191    if (skipDNValidation.isPresent())
192    {
193      printWrappedText(err, ERR_DEPRECATED_SKIP_DN_VALIDATION.get());
194      // continue
195    }
196
197    // Make sure that either the "ldifFile" argument or the "templateFile"
198    // argument was provided, but not both.
199    if (ldifFiles.isPresent())
200    {
201      if (templateFile.isPresent())
202      {
203        printWrappedText(err, conflictingArgsErrorMessage(ldifFiles, templateFile));
204        return 1;
205      }
206    }
207    else if (! templateFile.isPresent())
208    {
209      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_REQUIRED_ARGUMENT.get(
210          ldifFiles.getLongIdentifier(), templateFile.getLongIdentifier()));
211      return 1;
212    }
213
214    // Make sure that either the "includeBranchStrings" argument or the
215    // "backendID" argument was provided.
216    if(!includeBranchStrings.isPresent() && !backendID.isPresent())
217    {
218      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_MISSING_BACKEND_ARGUMENT.get(
219          includeBranchStrings.getLongIdentifier(), backendID.getLongIdentifier()));
220      return 1;
221    }
222
223    // Count rejects option requires the ability to return result codes
224    // which are potentially greater than 1. This is not supported by
225    // the task framework.
226    if (countRejects.isPresent() && argParser.connectionArgumentsPresent())
227    {
228      argParser.displayMessageAndUsageReference(err, ERR_LDIFIMPORT_COUNT_REJECTS_REQUIRES_OFFLINE.get(
229          countRejects.getLongIdentifier()));
230      return 1;
231    }
232
233    // Don't write non-error messages to console if quite
234    if (quietMode.isPresent()) {
235      out = new PrintStream(NullOutputStream.instance());
236    }
237
238    // Checks the version - if upgrade required, the tool is unusable
239    try
240    {
241      checkVersion();
242    }
243    catch (InitializationException e)
244    {
245      printWrappedText(err, e.getMessage());
246      return 1;
247    }
248
249    return process(argParser, initializeServer, out, err);
250  }
251
252  private void createArguments(LDAPConnectionArgumentParser argParser) throws ArgumentException
253  {
254      configFile =
255              StringArgument.builder("configFile")
256                      .shortIdentifier('f')
257                      .description(INFO_DESCRIPTION_CONFIG_FILE.get())
258                      .hidden()
259                      .required()
260                      .valuePlaceholder(INFO_CONFIGFILE_PLACEHOLDER.get())
261                      .buildAndAddToParser(argParser);
262      ldifFiles =
263              StringArgument.builder(OPTION_LONG_LDIF_FILE)
264                      .shortIdentifier(OPTION_SHORT_LDIF_FILE)
265                      .description(INFO_LDIFIMPORT_DESCRIPTION_LDIF_FILE.get())
266                      .multiValued()
267                      .valuePlaceholder(INFO_LDIFFILE_PLACEHOLDER.get())
268                      .buildAndAddToParser(argParser);
269      templateFile =
270              StringArgument.builder("templateFile")
271                      .shortIdentifier('A')
272                      .description(INFO_LDIFIMPORT_DESCRIPTION_TEMPLATE_FILE.get())
273                      .valuePlaceholder(INFO_TEMPLATE_FILE_PLACEHOLDER.get())
274                      .buildAndAddToParser(argParser);
275      backendID =
276              StringArgument.builder("backendID")
277                      .shortIdentifier('n')
278                      .description(INFO_LDIFIMPORT_DESCRIPTION_BACKEND_ID.get())
279                      .valuePlaceholder(INFO_BACKENDNAME_PLACEHOLDER.get())
280                      .buildAndAddToParser(argParser);
281      clearBackend =
282              BooleanArgument.builder("clearBackend")
283                      .shortIdentifier('F')
284                      .description(INFO_LDIFIMPORT_DESCRIPTION_CLEAR_BACKEND.get())
285                      .buildAndAddToParser(argParser);
286      includeBranchStrings =
287              StringArgument.builder("includeBranch")
288                      .shortIdentifier('b')
289                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_BRANCH.get())
290                      .multiValued()
291                      .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get())
292                      .buildAndAddToParser(argParser);
293      excludeBranchStrings =
294              StringArgument.builder("excludeBranch")
295                      .shortIdentifier('B')
296                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_BRANCH.get())
297                      .multiValued()
298                      .valuePlaceholder(INFO_BRANCH_DN_PLACEHOLDER.get())
299                      .buildAndAddToParser(argParser);
300      includeAttributeStrings =
301              StringArgument.builder("includeAttribute")
302                      .shortIdentifier('i')
303                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_ATTRIBUTE.get())
304                      .multiValued()
305                      .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get())
306                      .buildAndAddToParser(argParser);
307      excludeAttributeStrings =
308              StringArgument.builder("excludeAttribute")
309                      .shortIdentifier('e')
310                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_ATTRIBUTE.get())
311                      .multiValued()
312                      .valuePlaceholder(INFO_ATTRIBUTE_PLACEHOLDER.get())
313                      .buildAndAddToParser(argParser);
314      includeFilterStrings =
315              StringArgument.builder("includeFilter")
316                      .shortIdentifier('I')
317                      .description(INFO_LDIFIMPORT_DESCRIPTION_INCLUDE_FILTER.get())
318                      .multiValued()
319                      .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get())
320                      .buildAndAddToParser(argParser);
321      excludeFilterStrings =
322              StringArgument.builder("excludeFilter")
323                      .shortIdentifier('E')
324                      .description(INFO_LDIFIMPORT_DESCRIPTION_EXCLUDE_FILTER.get())
325                      .multiValued()
326                      .valuePlaceholder(INFO_FILTER_PLACEHOLDER.get())
327                      .buildAndAddToParser(argParser);
328      rejectFile =
329              StringArgument.builder("rejectFile")
330                      .shortIdentifier('R')
331                      .description(INFO_LDIFIMPORT_DESCRIPTION_REJECT_FILE.get())
332                      .valuePlaceholder(INFO_REJECT_FILE_PLACEHOLDER.get())
333                      .buildAndAddToParser(argParser);
334      skipFile =
335              StringArgument.builder("skipFile")
336                      .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_FILE.get())
337                      .valuePlaceholder(INFO_SKIP_FILE_PLACEHOLDER.get())
338                      .buildAndAddToParser(argParser);
339      overwrite =
340              BooleanArgument.builder("overwrite")
341                      .shortIdentifier('O')
342                      .description(INFO_LDIFIMPORT_DESCRIPTION_OVERWRITE.get())
343                      .buildAndAddToParser(argParser);
344      randomSeed =
345              IntegerArgument.builder(OPTION_LONG_RANDOM_SEED)
346                      .shortIdentifier(OPTION_SHORT_RANDOM_SEED)
347                      .description(INFO_LDIFIMPORT_DESCRIPTION_RANDOM_SEED.get())
348                      .defaultValue(0)
349                      .valuePlaceholder(INFO_SEED_PLACEHOLDER.get())
350                      .buildAndAddToParser(argParser);
351      skipSchemaValidation =
352              BooleanArgument.builder("skipSchemaValidation")
353                      .shortIdentifier('S')
354                      .description(INFO_LDIFIMPORT_DESCRIPTION_SKIP_SCHEMA_VALIDATION.get())
355                      .buildAndAddToParser(argParser);
356      skipDNValidation =
357            BooleanArgument.builder("skipDNValidation")
358                    .description(INFO_LDIFIMPORT_DESCRIPTION_DN_VALIDATION.get())
359                    .hidden()
360                    .buildAndAddToParser(argParser);
361      threadCount =
362              IntegerArgument.builder("threadCount")
363                      .description(INFO_LDIFIMPORT_DESCRIPTION_THREAD_COUNT.get())
364                      .lowerBound(1)
365                      .defaultValue(0)
366                      .valuePlaceholder(INFO_LDIFIMPORT_THREAD_COUNT_PLACEHOLDER.get())
367                      .buildAndAddToParser(argParser);
368      tmpDirectory =
369              StringArgument.builder("tmpdirectory")
370                      .description(INFO_LDIFIMPORT_DESCRIPTION_TEMP_DIRECTORY.get())
371                      .defaultValue("import-tmp")
372                      .valuePlaceholder(INFO_LDIFIMPORT_TEMP_DIR_PLACEHOLDER.get())
373                      .buildAndAddToParser(argParser);
374      countRejects =
375              BooleanArgument.builder("countRejects")
376                      .description(INFO_LDIFIMPORT_DESCRIPTION_COUNT_REJECTS.get())
377                      .buildAndAddToParser(argParser);
378      isCompressed =
379              BooleanArgument.builder("isCompressed")
380                      .shortIdentifier('c')
381                      .description(INFO_LDIFIMPORT_DESCRIPTION_IS_COMPRESSED.get())
382                      .buildAndAddToParser(argParser);
383      isEncrypted =
384              BooleanArgument.builder("isEncrypted")
385                      .shortIdentifier('y')
386                      .description(INFO_LDIFIMPORT_DESCRIPTION_IS_ENCRYPTED.get())
387                      .hidden() //See issue #27
388                      .buildAndAddToParser(argParser);
389      quietMode =
390              BooleanArgument.builder(OPTION_LONG_QUIET)
391                      .shortIdentifier(OPTION_SHORT_QUIET)
392                      .description(INFO_LDIFIMPORT_DESCRIPTION_QUIET.get())
393                      .buildAndAddToParser(argParser);
394
395    final BooleanArgument displayUsage = showUsageArgument();
396      argParser.addArgument(displayUsage);
397      argParser.setUsageArgument(displayUsage);
398  }
399
400  @Override
401  public void addTaskAttributes(List<RawAttribute> attributes)
402  {
403    // Required attributes
404    addAttribute(attributes, ATTR_IMPORT_LDIF_FILE, ldifFiles.getValues());
405    addAttribute(attributes, ATTR_IMPORT_TEMPLATE_FILE, templateFile.getValue());
406    addAttribute(attributes, ATTR_IMPORT_RANDOM_SEED, randomSeed.getValue());
407    addAttribute(attributes, ATTR_IMPORT_THREAD_COUNT, threadCount.getValue());
408
409    // Optional attributes
410    addAttribute2(attributes, ATTR_IMPORT_BACKEND_ID, backendID);
411    addAttribute(attributes, ATTR_IMPORT_INCLUDE_ATTRIBUTE, includeAttributeStrings.getValues());
412    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_ATTRIBUTE, excludeAttributeStrings.getValues());
413    addAttribute(attributes, ATTR_IMPORT_INCLUDE_FILTER, includeFilterStrings.getValues());
414    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_FILTER, excludeFilterStrings.getValues());
415    addAttribute(attributes, ATTR_IMPORT_INCLUDE_BRANCH, includeBranchStrings.getValues());
416    addAttribute(attributes, ATTR_IMPORT_EXCLUDE_BRANCH, excludeBranchStrings.getValues());
417    addAttribute2(attributes, ATTR_IMPORT_REJECT_FILE, rejectFile);
418    addAttribute2(attributes, ATTR_IMPORT_SKIP_FILE, skipFile);
419    addAttribute2(attributes, ATTR_IMPORT_OVERWRITE, overwrite);
420    addAttribute2(attributes, ATTR_IMPORT_SKIP_SCHEMA_VALIDATION, skipSchemaValidation);
421    addAttribute2(attributes, ATTR_IMPORT_TMP_DIRECTORY, tmpDirectory);
422    addAttribute2(attributes, ATTR_IMPORT_IS_COMPRESSED, isCompressed);
423    addAttribute2(attributes, ATTR_IMPORT_IS_ENCRYPTED, isEncrypted);
424    addAttribute2(attributes, ATTR_IMPORT_CLEAR_BACKEND, clearBackend);
425  }
426
427  private void addAttribute(List<RawAttribute> attributes, String attrName, String value)
428  {
429    if (value != null)
430    {
431      attributes.add(new LDAPAttribute(attrName, value));
432    }
433  }
434
435  private void addAttribute2(List<RawAttribute> attributes, String attrName, Argument arg)
436  {
437    final String value = arg.getValue();
438    if (value != null && !value.equals(arg.getDefaultValue()))
439    {
440      attributes.add(new LDAPAttribute(attrName, value));
441    }
442  }
443
444  private void addAttribute(List<RawAttribute> attributes, String attrName, List<String> attrValues)
445  {
446    if (attrValues != null && !attrValues.isEmpty())
447    {
448      attributes.add(new LDAPAttribute(attrName, attrValues));
449    }
450  }
451
452  @Override
453  public String getTaskObjectclass() {
454    return "ds-task-import";
455  }
456
457  @Override
458  public Class<?> getTaskClass() {
459    return ImportTask.class;
460  }
461
462  @Override
463  protected int processLocal(boolean initializeServer, PrintStream out, PrintStream err) {
464    if (initializeServer)
465    {
466      try
467      {
468        InitializationBuilder initBuilder = new DirectoryServer.InitializationBuilder(configFile.getValue())
469            .requireCryptoServices()
470            .requireUserPlugins(PluginType.LDIF_IMPORT);
471        if (!quietMode.isPresent())
472        {
473          initBuilder.requireErrorAndDebugLogPublisher(out, err);
474        }
475        initBuilder.initialize();
476      }
477      catch (InitializationException e)
478      {
479        printWrappedText(err, ERR_CANNOT_INITIALIZE_SERVER_COMPONENTS.get(e.getLocalizedMessage()));
480        return 1;
481      }
482    }
483
484    // See if there were any user-defined sets of include/exclude attributes or
485    // filters.  If so, then process them.
486    HashSet<AttributeType> excludeAttributes;
487    boolean excludeAllUserAttributes = false;
488    boolean excludeAllOperationalAttributes = false;
489    if (excludeAttributeStrings == null)
490    {
491      excludeAttributes = null;
492    }
493    else
494    {
495      excludeAttributes = new HashSet<>();
496      for (String attrName : excludeAttributeStrings.getValues())
497      {
498        String lowerName = attrName.toLowerCase();
499        if (lowerName.equals("*"))
500        {
501          excludeAllUserAttributes = true;
502        }
503        else if (lowerName.equals("+"))
504        {
505          excludeAllOperationalAttributes = true;
506        }
507        else
508        {
509          excludeAttributes.add(DirectoryServer.getSchema().getAttributeType(attrName));
510        }
511      }
512    }
513
514    HashSet<AttributeType> includeAttributes;
515    boolean includeAllUserAttributes = false;
516    boolean includeAllOperationalAttributes = false;
517    if (includeAttributeStrings == null)
518    {
519      includeAttributes = null;
520    }
521    else
522    {
523      includeAttributes = new HashSet<>();
524      for (String attrName : includeAttributeStrings.getValues())
525      {
526        String lowerName = attrName.toLowerCase();
527        if (lowerName.equals("*"))
528        {
529          includeAllUserAttributes = true;
530        }
531        else if (lowerName.equals("+"))
532        {
533          includeAllOperationalAttributes = true;
534        }
535        else
536        {
537          includeAttributes.add(DirectoryServer.getSchema().getAttributeType(attrName));
538        }
539      }
540    }
541
542    ArrayList<SearchFilter> excludeFilters;
543    if (excludeFilterStrings == null)
544    {
545      excludeFilters = null;
546    }
547    else
548    {
549      excludeFilters = new ArrayList<>();
550      for (String filterString : excludeFilterStrings.getValues())
551      {
552        try
553        {
554          excludeFilters.add(SearchFilter.createFilterFromString(filterString));
555        }
556        catch (DirectoryException de)
557        {
558          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, de.getMessageObject());
559          return 1;
560        }
561        catch (Exception e)
562        {
563          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_EXCLUDE_FILTER, filterString, getExceptionMessage(e));
564          return 1;
565        }
566      }
567    }
568
569    ArrayList<SearchFilter> includeFilters;
570    if (includeFilterStrings == null)
571    {
572      includeFilters = null;
573    }
574    else
575    {
576      includeFilters = new ArrayList<>();
577      for (String filterString : includeFilterStrings.getValues())
578      {
579        try
580        {
581          includeFilters.add(SearchFilter.createFilterFromString(filterString));
582        }
583        catch (DirectoryException de)
584        {
585          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, de.getMessageObject());
586          return 1;
587        }
588        catch (Exception e)
589        {
590          logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_INCLUDE_FILTER, filterString, getExceptionMessage(e));
591          return 1;
592        }
593      }
594    }
595
596    // Get information about the backends defined in the server.  Iterate
597    // through them, finding the one backend into which the LDIF should be
598    // imported and finding backends with subordinate base DNs that should be
599    // excluded from the import.
600    Backend<?> backend = null;
601    Set<DN> defaultIncludeBranches = null;
602    Set<DN> excludeBranches = new HashSet<>();
603    Set<DN> includeBranches = new HashSet<>();
604
605    if (includeBranchStrings.isPresent())
606    {
607      for (String s : includeBranchStrings.getValues())
608      {
609        DN includeBranch;
610        try
611        {
612          includeBranch = DN.valueOf(s);
613        }
614        catch (Exception e)
615        {
616          logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_INCLUDE_BASE, s, getExceptionMessage(e));
617          return 1;
618        }
619
620        includeBranches.add(includeBranch);
621      }
622    }
623
624    ArrayList<Backend<?>> backendList = new ArrayList<>();
625    ArrayList<BackendCfg> entryList = new ArrayList<>();
626    ArrayList<List<DN>> dnList = new ArrayList<>();
627    int code = BackendToolUtils.getBackends(backendList, entryList, dnList);
628    if (code != 0)
629    {
630      return code;
631    }
632
633    int numBackends = backendList.size();
634    for (int i=0; i < numBackends; i++)
635    {
636      Backend<?> b = backendList.get(i);
637
638      if(backendID.isPresent())
639      {
640        if (! backendID.getValue().equals(b.getBackendID()))
641        {
642          continue;
643        }
644      }
645      else
646      {
647        if (!useBackend(includeBranches, dnList.get(i)))
648        {
649          continue;
650        }
651      }
652
653      if (backend == null)
654      {
655        backend                = b;
656        defaultIncludeBranches = new HashSet<>(dnList.get(i));
657      }
658      else
659      {
660        logger.error(ERR_LDIFIMPORT_MULTIPLE_BACKENDS_FOR_ID);
661        return 1;
662      }
663    }
664
665    if (backend == null)
666    {
667      logger.error(ERR_LDIFIMPORT_NO_BACKENDS_FOR_ID);
668      return 1;
669    }
670    else if (!backend.supports(BackendOperation.LDIF_IMPORT))
671    {
672      logger.error(ERR_LDIFIMPORT_CANNOT_IMPORT, backendID.getValue());
673      return 1;
674    }
675
676    for (List<DN> baseList : dnList)
677    {
678      for (DN baseDN : baseList)
679      {
680        for (DN importBase : defaultIncludeBranches)
681        {
682          if (!baseDN.equals(importBase) && baseDN.isSubordinateOrEqualTo(importBase))
683          {
684            excludeBranches.add(baseDN);
685            break;
686          }
687        }
688      }
689    }
690
691    for (String s : excludeBranchStrings.getValues())
692    {
693      DN excludeBranch;
694      try
695      {
696        excludeBranch = DN.valueOf(s);
697      }
698      catch (Exception e)
699      {
700        logger.error(ERR_LDIFIMPORT_CANNOT_DECODE_EXCLUDE_BASE, s, getExceptionMessage(e));
701        return 1;
702      }
703
704      excludeBranches.add(excludeBranch);
705    }
706
707    if (! includeBranchStrings.isPresent())
708    {
709      includeBranches = defaultIncludeBranches;
710    }
711    else
712    {
713      // Make sure the selected backend will handle all the include branches
714      for(DN includeBranch : includeBranches)
715      {
716        if (! Backend.handlesEntry(includeBranch, defaultIncludeBranches,
717                                   excludeBranches))
718        {
719          logger.error(ERR_LDIFIMPORT_INVALID_INCLUDE_BASE, includeBranch, backendID.getValue());
720          return 1;
721        }
722      }
723    }
724
725    // See if the data should be read from LDIF files or generated via MakeLDIF.
726    LDIFImportConfig importConfig;
727    if (ldifFiles.isPresent())
728    {
729      ArrayList<String> fileList = new ArrayList<>(ldifFiles.getValues());
730      int badFileCount = 0;
731      for (String pathname : fileList)
732      {
733        File f = new File(pathname);
734        if (!f.canRead())
735        {
736          logger.error(ERR_LDIFIMPORT_CANNOT_READ_FILE, pathname);
737          badFileCount++;
738        }
739      }
740      if (badFileCount > 0)
741      {
742        return 1;
743      }
744      importConfig = new LDIFImportConfig(fileList);
745    }
746    else
747    {
748      Random random = newRandom();
749
750      String resourcePath = DirectoryServer.getInstanceRoot() + File.separator +
751                            PATH_MAKELDIF_RESOURCE_DIR;
752      TemplateFile tf = new TemplateFile(resourcePath, random);
753
754      ArrayList<LocalizableMessage> warnings = new ArrayList<>();
755      try
756      {
757        tf.parse(templateFile.getValue(), warnings);
758      }
759      catch (Exception e)
760      {
761        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_TEMPLATE_FILE, templateFile.getValue(), e.getMessage());
762        return 1;
763      }
764
765      importConfig = new LDIFImportConfig(tf);
766    }
767
768    // Create the LDIF import configuration to use when reading the LDIF.
769    importConfig.setCompressed(isCompressed.isPresent());
770    importConfig.setClearBackend(clearBackend.isPresent());
771    importConfig.setEncrypted(isEncrypted.isPresent());
772    importConfig.setExcludeAttributes(excludeAttributes);
773    importConfig.setExcludeBranches(excludeBranches);
774    importConfig.setExcludeFilters(excludeFilters);
775    importConfig.setIncludeAttributes(includeAttributes);
776    importConfig.setIncludeBranches(includeBranches);
777    importConfig.setIncludeFilters(includeFilters);
778    importConfig.setValidateSchema(!skipSchemaValidation.isPresent());
779    importConfig.setTmpDirectory(tmpDirectory.getValue());
780
781    try
782    {
783        importConfig.setThreadCount(threadCount.getIntValue());
784    }
785    catch(Exception e)
786    {
787        logger.error(ERR_LDIFIMPORT_CANNOT_PARSE_THREAD_COUNT,
788                threadCount.getValue(), e.getMessage());
789        return 1;
790    }
791
792    importConfig.setBufferSize(LDIF_BUFFER_SIZE);
793    importConfig.setExcludeAllUserAttributes(excludeAllUserAttributes);
794    importConfig.setExcludeAllOperationalAttributes(excludeAllOperationalAttributes);
795    importConfig.setIncludeAllOpAttributes(includeAllOperationalAttributes);
796    importConfig.setIncludeAllUserAttributes(includeAllUserAttributes);
797
798    // FIXME -- Should this be conditional?
799    importConfig.setInvokeImportPlugins(true);
800
801    if (rejectFile != null)
802    {
803      try
804      {
805        ExistingFileBehavior existingBehavior = overwrite.isPresent()
806            ? ExistingFileBehavior.OVERWRITE
807            : ExistingFileBehavior.APPEND;
808
809        importConfig.writeRejectedEntries(rejectFile.getValue(),
810                                          existingBehavior);
811      }
812      catch (Exception e)
813      {
814        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_REJECTS_FILE, rejectFile.getValue(), getExceptionMessage(e));
815        return 1;
816      }
817    }
818
819    if (skipFile != null)
820    {
821      try
822      {
823        ExistingFileBehavior existingBehavior = overwrite.isPresent()
824            ? ExistingFileBehavior.OVERWRITE
825            : ExistingFileBehavior.APPEND;
826
827        importConfig.writeSkippedEntries(skipFile.getValue(),
828                                          existingBehavior);
829      }
830      catch (Exception e)
831      {
832        logger.error(ERR_LDIFIMPORT_CANNOT_OPEN_SKIP_FILE, skipFile.getValue(), getExceptionMessage(e));
833        return 1;
834      }
835    }
836
837    // Get the set of base DNs for the backend as an array.
838    DN[] baseDNs = new DN[defaultIncludeBranches.size()];
839    defaultIncludeBranches.toArray(baseDNs);
840
841    // Acquire an exclusive lock for the backend.
842    try
843    {
844      String lockFile = LockFileManager.getBackendLockFileName(backend);
845      StringBuilder failureReason = new StringBuilder();
846      if (! LockFileManager.acquireExclusiveLock(lockFile, failureReason))
847      {
848        logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), failureReason);
849        return 1;
850      }
851    }
852    catch (Exception e)
853    {
854      logger.error(ERR_LDIFIMPORT_CANNOT_LOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
855      return 1;
856    }
857
858    // Launch the import.
859    int retCode = 0;
860    try
861    {
862      LDIFImportResult importResult =
863          backend.importLDIF(importConfig, DirectoryServer.getInstance().getServerContext());
864      if (countRejects.isPresent())
865      {
866        if (importResult.getEntriesRejected() > Integer.MAX_VALUE)
867        {
868          retCode = Integer.MAX_VALUE;
869        }
870        else
871        {
872          retCode = (int) importResult.getEntriesRejected();
873        }
874      }
875    }
876    catch (DirectoryException de)
877    {
878      LocalizableMessage msg;
879      msg = de.getMessageObject();
880      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT.get(msg));
881      retCode = 1;
882    }
883    catch (Exception e)
884    {
885      logger.error(ERR_LDIFIMPORT_ERROR_DURING_IMPORT, getExceptionMessage(e));
886      retCode = 1;
887    }
888
889    // Release the exclusive lock on the backend.
890    try
891    {
892      String lockFile = LockFileManager.getBackendLockFileName(backend);
893      StringBuilder failureReason = new StringBuilder();
894      if (! LockFileManager.releaseLock(lockFile, failureReason))
895      {
896        logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason);
897        retCode = 1;
898      }
899    }
900    catch (Exception e)
901    {
902      logger.warn(WARN_LDIFIMPORT_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
903      retCode = 1;
904    }
905
906    // Clean up after the import by closing the import config.
907    importConfig.close();
908    return retCode;
909  }
910
911  @Override
912  protected void cleanup()
913  {
914    DirectoryServer.shutdownBackends();
915  }
916
917  private boolean useBackend(Set<DN> includeBranches, List<DN> dnlist)
918  {
919    for (DN baseDN : dnlist)
920    {
921      for (DN includeDN : includeBranches)
922      {
923        if (baseDN.isSuperiorOrEqualTo(includeDN))
924        {
925          return true;
926        }
927      }
928    }
929    return false;
930  }
931
932  private Random newRandom()
933  {
934    if (randomSeed.isPresent())
935    {
936      try
937      {
938        return new Random(randomSeed.getIntValue());
939      }
940      catch (Exception ignored)
941      {
942        // ignore
943      }
944    }
945    return new Random();
946  }
947
948  @Override
949  public String getTaskId() {
950    // NYI.
951    return null;
952  }
953}