001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2006-2009 Sun Microsystems, Inc.
015 * Portions Copyright 2011-2016 ForgeRock AS.
016 */
017package org.opends.server.tools;
018
019import static org.opends.messages.ToolMessages.*;
020import static org.opends.server.config.ConfigConstants.*;
021import static org.opends.server.util.StaticUtils.*;
022import static com.forgerock.opendj.cli.CommonArguments.*;
023import static com.forgerock.opendj.cli.Utils.*;
024
025import java.io.OutputStream;
026import java.io.PrintStream;
027import java.util.ArrayList;
028import java.util.Collection;
029import java.util.List;
030
031import org.forgerock.i18n.LocalizableMessage;
032import org.forgerock.i18n.slf4j.LocalizedLogger;
033import org.forgerock.opendj.config.server.ConfigException;
034import org.forgerock.opendj.ldap.DN;
035import org.forgerock.opendj.server.config.server.BackendCfg;
036import org.opends.server.api.Backend;
037import org.opends.server.api.Backend.BackendOperation;
038import org.opends.server.backends.RebuildConfig;
039import org.opends.server.backends.RebuildConfig.RebuildMode;
040import org.opends.server.core.DirectoryServer;
041import org.opends.server.core.LockFileManager;
042import org.opends.server.loggers.JDKLogging;
043import org.opends.server.protocols.ldap.LDAPAttribute;
044import org.opends.server.tasks.RebuildTask;
045import org.opends.server.tools.tasks.TaskTool;
046import org.opends.server.types.InitializationException;
047import org.opends.server.types.NullOutputStream;
048import org.opends.server.types.RawAttribute;
049import org.opends.server.util.StaticUtils;
050import org.opends.server.util.cli.LDAPConnectionArgumentParser;
051
052import com.forgerock.opendj.cli.ArgumentException;
053import com.forgerock.opendj.cli.BooleanArgument;
054import com.forgerock.opendj.cli.StringArgument;
055
056/**
057 * This program provides a utility to rebuild the contents of the indexes of a
058 * Directory Server backend. This will be a process that is intended to run
059 * separate from Directory Server and not internally within the server process
060 * (e.g., via the tasks interface).
061 */
062public class RebuildIndex extends TaskTool
063{
064
065  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
066
067  private StringArgument configFile;
068  private StringArgument baseDNString;
069  private StringArgument indexList;
070  private StringArgument tmpDirectory;
071  private BooleanArgument rebuildAll;
072  private BooleanArgument rebuildDegraded;
073  private BooleanArgument clearDegradedState;
074
075  private final LDAPConnectionArgumentParser argParser = createArgParser(
076      "org.opends.server.tools.RebuildIndex",
077      INFO_REBUILDINDEX_TOOL_DESCRIPTION.get());
078
079  private RebuildConfig rebuildConfig = new RebuildConfig();
080  private Backend<?> currentBackend;
081
082  /**
083   * Processes the command-line arguments and invokes the rebuild process.
084   *
085   * @param args
086   *          The command-line arguments provided to this program.
087   */
088  public static void main(final String[] args)
089  {
090    final int retCode =
091        mainRebuildIndex(args, true, System.out, System.err);
092    if (retCode != 0)
093    {
094      System.exit(filterExitCode(retCode));
095    }
096  }
097
098  /**
099   * Processes the command-line arguments and invokes the rebuild process.
100   *
101   * @param args
102   *          The command-line arguments provided to this program.
103   * @param initializeServer
104   *          Indicates whether to initialize the server.
105   * @param outStream
106   *          The output stream to use for standard output, or {@code null} if
107   *          standard output is not needed.
108   * @param errStream
109   *          The output stream to use for standard error, or {@code null} if
110   *          standard error is not needed.
111   * @return The error code.
112   */
113  public static int mainRebuildIndex(final String[] args,
114      final boolean initializeServer, final OutputStream outStream,
115      final OutputStream errStream)
116  {
117    final RebuildIndex tool = new RebuildIndex();
118    return tool.process(args, initializeServer, outStream, errStream);
119  }
120
121  private int process(final String[] args, final boolean initializeServer,
122      final OutputStream outStream, final OutputStream errStream )
123  {
124    final PrintStream out = NullOutputStream.wrapOrNullStream(outStream);
125    final PrintStream err = NullOutputStream.wrapOrNullStream(errStream);
126    JDKLogging.enableConsoleLoggingForOpenDJTool();
127
128    // Initialize all the command-line argument types and register them with the
129    // parser.
130    try
131    {
132      initializeArguments(false);
133    }
134    catch (ArgumentException ae)
135    {
136      printWrappedText(err, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
137      return 1;
138    }
139
140    // Parse the command-line arguments provided to this program.
141    try
142    {
143      argParser.parseArguments(args);
144    }
145    catch (ArgumentException ae)
146    {
147      argParser.displayMessageAndUsageReference(err, ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
148      return 1;
149    }
150
151    // If we should just display usage or version information,
152    // then print it and exit.
153    if (argParser.usageOrVersionDisplayed())
154    {
155      return 0;
156    }
157
158    if (indexList.getValues().isEmpty()
159        && !rebuildAll.isPresent()
160        && !rebuildDegraded.isPresent())
161    {
162      argParser.displayMessageAndUsageReference(err, ERR_REBUILDINDEX_REQUIRES_AT_LEAST_ONE_INDEX.get());
163      return 1;
164    }
165
166    if (rebuildAll.isPresent() && indexList.isPresent())
167    {
168      argParser.displayMessageAndUsageReference(err, ERR_REBUILDINDEX_REBUILD_ALL_ERROR.get());
169      return 1;
170    }
171
172    if (rebuildDegraded.isPresent() && indexList.isPresent())
173    {
174      argParser.displayMessageAndUsageReference(err, ERR_REBUILDINDEX_REBUILD_DEGRADED_ERROR.get("index"));
175      return 1;
176    }
177
178    if (rebuildDegraded.isPresent() && clearDegradedState.isPresent())
179    {
180      argParser.displayMessageAndUsageReference(err, ERR_REBUILDINDEX_REBUILD_DEGRADED_ERROR.get("clearDegradedState"));
181      return 1;
182    }
183
184    if (rebuildAll.isPresent() && rebuildDegraded.isPresent())
185    {
186      argParser.displayMessageAndUsageReference(err,
187          ERR_REBUILDINDEX_REBUILD_ALL_DEGRADED_ERROR.get("rebuildDegraded"));
188      return 1;
189    }
190
191    if (rebuildAll.isPresent() && clearDegradedState.isPresent())
192    {
193      argParser.displayMessageAndUsageReference(err,
194          ERR_REBUILDINDEX_REBUILD_ALL_DEGRADED_ERROR.get("clearDegradedState"));
195      return 1;
196    }
197
198    // Checks the version - if upgrade required, the tool is unusable
199    try
200    {
201      checkVersion();
202    }
203    catch (InitializationException e)
204    {
205      printWrappedText(err, e.getMessage());
206      return 1;
207    }
208    return process(argParser, initializeServer, out, err);
209  }
210
211  /**
212   * Initializes the arguments for the rebuild index tool.
213   *
214   * @param isMultipleBackends
215   *          {@code true} if the tool is used as internal.
216   * @throws ArgumentException
217   *           If the initialization fails.
218   */
219  private void initializeArguments(final boolean isMultipleBackends)
220      throws ArgumentException
221  {
222    argParser.setShortToolDescription(REF_SHORT_DESC_REBUILD_INDEX.get());
223
224    configFile =
225            StringArgument.builder("configFile")
226                    .shortIdentifier('f')
227                    .description(INFO_DESCRIPTION_CONFIG_FILE.get())
228                    .hidden()
229                    .required()
230                    .valuePlaceholder(INFO_CONFIGFILE_PLACEHOLDER.get())
231                    .buildAndAddToParser(argParser);
232
233    final StringArgument.Builder builder =
234            StringArgument.builder("baseDN")
235                    .shortIdentifier('b')
236                    .description(INFO_REBUILDINDEX_DESCRIPTION_BASE_DN.get())
237                    .required()
238                    .valuePlaceholder(INFO_BASEDN_PLACEHOLDER.get());
239    if (isMultipleBackends) {
240      builder.multiValued();
241    }
242    baseDNString = builder.buildAndAddToParser(argParser);
243
244    indexList =
245            StringArgument.builder("index")
246                    .shortIdentifier('i')
247                    .description(INFO_REBUILDINDEX_DESCRIPTION_INDEX_NAME.get())
248                    .multiValued()
249                    .valuePlaceholder(INFO_INDEX_PLACEHOLDER.get())
250                    .buildAndAddToParser(argParser);
251    rebuildAll =
252            BooleanArgument.builder("rebuildAll")
253                    .description(INFO_REBUILDINDEX_DESCRIPTION_REBUILD_ALL.get())
254                    .buildAndAddToParser(argParser);
255    rebuildDegraded =
256            BooleanArgument.builder("rebuildDegraded")
257                    .description(INFO_REBUILDINDEX_DESCRIPTION_REBUILD_DEGRADED.get())
258                    .buildAndAddToParser(argParser);
259    clearDegradedState =
260            BooleanArgument.builder("clearDegradedState")
261                    .description(INFO_REBUILDINDEX_DESCRIPTION_CLEAR_DEGRADED_STATE.get())
262                    .buildAndAddToParser(argParser);
263    tmpDirectory =
264            StringArgument.builder("tmpdirectory")
265                    .description(INFO_REBUILDINDEX_DESCRIPTION_TEMP_DIRECTORY.get())
266                    .defaultValue("import-tmp")
267                    .valuePlaceholder(INFO_REBUILDINDEX_TEMP_DIR_PLACEHOLDER.get())
268                    .buildAndAddToParser(argParser);
269
270    final BooleanArgument displayUsage = showUsageArgument();
271    argParser.addArgument(displayUsage);
272    argParser.setUsageArgument(displayUsage);
273  }
274
275  @Override
276  protected int processLocal(final boolean initializeServer, final PrintStream out, final PrintStream err)
277  {
278    if (initializeServer)
279    {
280      final int init = initializeServer(out, err);
281      if (init != 0)
282      {
283        return init;
284      }
285    }
286
287    if (!configureRebuildProcess(baseDNString.getValue()))
288    {
289      return 1;
290    }
291
292    return rebuildIndex(currentBackend, rebuildConfig);
293  }
294
295  @Override
296  protected void cleanup()
297  {
298    DirectoryServer.shutdownBackends();
299  }
300
301  /**
302   * Configures the rebuild index process. i.e.: decodes the selected DN and
303   * retrieves the backend which holds it. Finally, initializes and sets the
304   * rebuild configuration.
305   *
306   * @param dn
307   *          User selected base DN.
308   * @return A boolean representing the result of the process.
309   */
310  private boolean configureRebuildProcess(final String dn) {
311    // Decodes the base DN provided by the user.
312    DN rebuildBaseDN = null;
313    try
314    {
315      rebuildBaseDN = DN.valueOf(dn);
316    }
317    catch (Exception e)
318    {
319      logger.error(ERR_CANNOT_DECODE_BASE_DN, dn,
320              getExceptionMessage(e));
321      return false;
322    }
323
324    // Retrieves the backend which holds the selected base DN.
325    try
326    {
327      setCurrentBackend(retrieveBackend(rebuildBaseDN));
328    }
329    catch (Exception e)
330    {
331      logger.error(LocalizableMessage.raw(e.getMessage()));
332      return false;
333    }
334
335    setRebuildConfig(initializeRebuildIndexConfiguration(rebuildBaseDN));
336    return true;
337  }
338
339  /**
340   * Initializes the directory server.
341   *
342   * @param out stream to write messages; may be null
343   * @param err
344   *          The output stream to use for standard error, or {@code null} if
345   *          standard error is not needed.
346   * @return The result code.
347   */
348  private int initializeServer(final PrintStream out, final PrintStream err)
349  {
350    try
351    {
352      new DirectoryServer.InitializationBuilder(configFile.getValue())
353          .requireCryptoServices()
354          .requireErrorAndDebugLogPublisher(out, err)
355          .initialize();
356      return 0;
357    }
358    catch (InitializationException ie)
359    {
360      printWrappedText(err, ERR_CANNOT_INITIALIZE_SERVER_COMPONENTS.get(ie.getLocalizedMessage()));
361      return 1;
362    }
363  }
364
365  /**
366   * Initializes and sets the rebuild index configuration.
367   *
368   * @param rebuildBaseDN
369   *          The selected base DN.
370   * @return A rebuild configuration.
371   */
372  private RebuildConfig initializeRebuildIndexConfiguration(
373      final DN rebuildBaseDN)
374  {
375    final RebuildConfig config = new RebuildConfig();
376    config.setBaseDN(rebuildBaseDN);
377    for (final String s : indexList.getValues())
378    {
379      config.addRebuildIndex(s);
380    }
381
382    if (rebuildAll.isPresent())
383    {
384      config.setRebuildMode(RebuildMode.ALL);
385    }
386    else if (rebuildDegraded.isPresent())
387    {
388      config.setRebuildMode(RebuildMode.DEGRADED);
389    }
390    else
391    {
392      if (clearDegradedState.isPresent())
393      {
394        config.isClearDegradedState(true);
395      }
396      config.setRebuildMode(RebuildMode.USER_DEFINED);
397    }
398
399    config.setTmpDirectory(tmpDirectory.getValue());
400    return config;
401  }
402
403  /**
404   * Launches the rebuild index process.
405   *
406   * @param backend
407   *          The directory server backend.
408   * @param rebuildConfig
409   *          The configuration which is going to be used by the rebuild index
410   *          process.
411   * @return An integer representing the result of the process.
412   */
413  private int rebuildIndex(final Backend<?> backend, final RebuildConfig rebuildConfig)
414  {
415    // Acquire an exclusive lock for the backend.
416    //TODO: Find a way to do this with the server online.
417    try
418    {
419      final String lockFile = LockFileManager.getBackendLockFileName(backend);
420      final StringBuilder failureReason = new StringBuilder();
421      if (!LockFileManager.acquireExclusiveLock(lockFile, failureReason))
422      {
423        logger.error(ERR_REBUILDINDEX_CANNOT_EXCLUSIVE_LOCK_BACKEND, backend.getBackendID(), failureReason);
424        return 1;
425      }
426    }
427    catch (Exception e)
428    {
429      logger.error(ERR_REBUILDINDEX_CANNOT_EXCLUSIVE_LOCK_BACKEND, backend
430              .getBackendID(), getExceptionMessage(e));
431      return 1;
432    }
433
434    int returnCode = 0;
435    try
436    {
437      backend.rebuildBackend(rebuildConfig, DirectoryServer.getInstance().getServerContext());
438    }
439    catch (InitializationException e)
440    {
441      logger.error(ERR_REBUILDINDEX_ERROR_DURING_REBUILD, e.getMessage());
442      returnCode = 1;
443    }
444    catch (Exception e)
445    {
446      logger.error(ERR_REBUILDINDEX_ERROR_DURING_REBUILD, getExceptionMessage(e));
447      returnCode = 1;
448    }
449    finally
450    {
451      // Release the shared lock on the backend.
452      try
453      {
454        final String lockFile = LockFileManager.getBackendLockFileName(backend);
455        final StringBuilder failureReason = new StringBuilder();
456        if (!LockFileManager.releaseLock(lockFile, failureReason))
457        {
458          logger.warn(WARN_REBUILDINDEX_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), failureReason);
459        }
460      }
461      catch (Exception e)
462      {
463        logger.error(WARN_REBUILDINDEX_CANNOT_UNLOCK_BACKEND, backend.getBackendID(), getExceptionMessage(e));
464      }
465    }
466
467    return returnCode;
468  }
469
470  /**
471   * Gets information about the backends defined in the server. Iterates through
472   * them, finding the one that holds the base DN.
473   *
474   * @param selectedDN
475   *          The user selected DN.
476   * @return The backend which holds the selected base DN.
477   * @throws ConfigException
478   *           If the backend is poorly configured.
479   * @throws Exception
480   *           If an exception occurred during the backend search.
481   */
482  private Backend<?> retrieveBackend(final DN selectedDN) throws ConfigException, Exception
483  {
484    final List<Backend<?>> backendList = new ArrayList<>();
485    final List<BackendCfg> entryList = new ArrayList<>();
486    final List<List<DN>> dnList = new ArrayList<>();
487    BackendToolUtils.getBackends(backendList, entryList, dnList);
488
489    Backend<?> backend = null;
490    final int numBackends = backendList.size();
491    for (int i = 0; i < numBackends; i++)
492    {
493      final Backend<?> b = backendList.get(i);
494      final List<DN> baseDNs = dnList.get(i);
495      if (baseDNs.contains(selectedDN))
496      {
497        if (backend != null)
498        {
499          throw new ConfigException(ERR_MULTIPLE_BACKENDS_FOR_BASE.get(baseDNString.getValue()));
500        }
501        backend = b;
502      }
503    }
504
505    if (backend == null)
506    {
507      throw new ConfigException(ERR_NO_BACKENDS_FOR_BASE.get(baseDNString.getValue()));
508    }
509    if (!backend.supports(BackendOperation.INDEXING))
510    {
511      throw new ConfigException(ERR_BACKEND_NO_INDEXING_SUPPORT.get());
512    }
513    return backend;
514  }
515
516  /**
517   * This function allow internal use of the rebuild index tools. This function
518   * rebuilds indexes shared by multiple backends.
519   *
520   * @param initializeServer
521   *          Indicates whether to initialize the server.
522   * @param out
523   *          The print stream which is used to display errors/debug lines.
524   *          Usually redirected into a logger if the tool is used as external.
525   * @param args
526   *          The arguments used to launch the rebuild index process.
527   * @return An integer indicating the result of this action.
528   */
529  public int rebuildIndexesWithinMultipleBackends(
530      final boolean initializeServer, final PrintStream out, final Collection<String> args)
531  {
532    try
533    {
534      try
535      {
536        initializeArguments(true);
537      }
538      catch (ArgumentException ae)
539      {
540        printWrappedText(out, ERR_CANNOT_INITIALIZE_ARGS.get(ae.getMessage()));
541        return 1;
542      }
543
544      try
545      {
546        argParser.parseArguments(args.toArray(new String[args.size()]));
547      }
548      catch (ArgumentException ae)
549      {
550        argParser.displayMessageAndUsageReference(out, ERR_ERROR_PARSING_ARGS.get(ae.getMessage()));
551        return 1;
552      }
553
554      if (initializeServer)
555      {
556        final int init = initializeServer(out, out);
557        if (init != 0)
558        {
559          return init;
560        }
561      }
562
563      for (final String dn : baseDNString.getValues())
564      {
565        if (!configureRebuildProcess(dn))
566        {
567          return 1;
568        }
569
570        final int result =
571            rebuildIndex(getCurrentBackend(), getRebuildConfig());
572        // If the rebuild index is going bad, process is stopped.
573        if (result != 0)
574        {
575          out.println(String.format(
576                  "An error occurs during the rebuild index process" +
577                  " in %s, rebuild index(es) aborted.",
578                  dn));
579          return 1;
580        }
581      }
582    }
583    finally
584    {
585      StaticUtils.close(out);
586    }
587    return 0;
588  }
589
590  /** {@inheritDoc} */
591  @Override
592  public String getTaskId()
593  {
594    // NYI.
595    return null;
596  }
597
598  /** {@inheritDoc} */
599  @Override
600  public void addTaskAttributes(List<RawAttribute> attributes)
601  {
602    // Required attributes
603    addLdapAttribute(attributes, ATTR_REBUILD_BASE_DN, baseDNString.getValue());
604
605    attributes.add(new LDAPAttribute(ATTR_REBUILD_INDEX, indexList.getValues()));
606
607    if (hasNonDefaultValue(tmpDirectory))
608    {
609      addLdapAttribute(attributes, ATTR_REBUILD_TMP_DIRECTORY, tmpDirectory.getValue());
610    }
611
612    if (hasNonDefaultValue(rebuildAll))
613    {
614      addLdapAttribute(attributes, ATTR_REBUILD_INDEX, REBUILD_ALL);
615    }
616
617    if (hasNonDefaultValue(rebuildDegraded))
618    {
619      addLdapAttribute(attributes, ATTR_REBUILD_INDEX, REBUILD_DEGRADED);
620    }
621
622    if (hasNonDefaultValue(clearDegradedState))
623    {
624      addLdapAttribute(attributes, ATTR_REBUILD_INDEX_CLEARDEGRADEDSTATE, "true");
625    }
626  }
627
628  private void addLdapAttribute(List<RawAttribute> attributes, String attrType, String attrValue)
629  {
630    attributes.add(new LDAPAttribute(attrType, attrValue));
631  }
632
633  private boolean hasNonDefaultValue(BooleanArgument arg)
634  {
635    return arg.getValue() != null
636        && !arg.getValue().equals(arg.getDefaultValue());
637  }
638
639  private boolean hasNonDefaultValue(StringArgument arg)
640  {
641    return arg.getValue() != null
642        && !arg.getValue().equals(arg.getDefaultValue());
643  }
644
645  /** {@inheritDoc} */
646  @Override
647  public String getTaskObjectclass()
648  {
649    return "ds-task-rebuild";
650  }
651
652  /** {@inheritDoc} */
653  @Override
654  public Class<?> getTaskClass()
655  {
656    return RebuildTask.class;
657  }
658
659  /**
660   * Returns the rebuild configuration.
661   *
662   * @return The rebuild configuration.
663   */
664  public RebuildConfig getRebuildConfig()
665  {
666    return rebuildConfig;
667  }
668
669  /**
670   * Sets the rebuild configuration.
671   *
672   * @param rebuildConfig
673   *          The rebuild configuration to set.
674   */
675  public void setRebuildConfig(RebuildConfig rebuildConfig)
676  {
677    this.rebuildConfig = rebuildConfig;
678  }
679
680  /**
681   * Returns the current backend.
682   *
683   * @return The current backend.
684   */
685  public Backend<?> getCurrentBackend()
686  {
687    return currentBackend;
688  }
689
690  /**
691   * Sets the current backend.
692   *
693   * @param currentBackend
694   *          The current backend to set.
695   */
696  public void setCurrentBackend(Backend<?> currentBackend)
697  {
698    this.currentBackend = currentBackend;
699  }
700}