001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2006-2010 Sun Microsystems, Inc.
015 * Portions Copyright 2012-2016 ForgeRock AS.
016 */
017package org.opends.server.util;
018
019import static org.forgerock.util.Reject.*;
020import static org.opends.messages.UtilityMessages.*;
021import static org.opends.server.util.CollectionUtils.*;
022import static org.opends.server.util.StaticUtils.*;
023
024import java.io.BufferedReader;
025import java.io.BufferedWriter;
026import java.io.Closeable;
027import java.io.IOException;
028import java.io.InputStream;
029import java.net.URL;
030import java.util.ArrayList;
031import java.util.HashMap;
032import java.util.LinkedList;
033import java.util.List;
034import java.util.Map;
035import java.util.concurrent.atomic.AtomicLong;
036
037import org.forgerock.i18n.LocalizableMessage;
038import org.forgerock.i18n.LocalizableMessageBuilder;
039import org.forgerock.i18n.slf4j.LocalizedLogger;
040import org.forgerock.opendj.ldap.AVA;
041import org.forgerock.opendj.ldap.AttributeDescription;
042import org.forgerock.opendj.ldap.ByteString;
043import org.forgerock.opendj.ldap.ByteStringBuilder;
044import org.forgerock.opendj.ldap.DN;
045import org.forgerock.opendj.ldap.ModificationType;
046import org.forgerock.opendj.ldap.RDN;
047import org.forgerock.opendj.ldap.schema.AttributeType;
048import org.forgerock.opendj.ldap.schema.CoreSchema;
049import org.opends.server.api.plugin.PluginResult;
050import org.opends.server.core.DirectoryServer;
051import org.opends.server.core.PluginConfigManager;
052import org.opends.server.protocols.ldap.LDAPAttribute;
053import org.opends.server.protocols.ldap.LDAPModification;
054import org.opends.server.types.AcceptRejectWarn;
055import org.opends.server.types.Attribute;
056import org.opends.server.types.AttributeBuilder;
057import org.opends.server.types.Attributes;
058import org.opends.server.types.Entry;
059import org.opends.server.types.LDIFImportConfig;
060import org.forgerock.opendj.ldap.schema.ObjectClass;
061import org.opends.server.types.RawModification;
062
063/**
064 * This class provides the ability to read information from an LDIF file.  It
065 * provides support for both standard entries and change entries (as would be
066 * used with a tool like ldapmodify).
067 */
068@org.opends.server.types.PublicAPI(
069     stability=org.opends.server.types.StabilityLevel.UNCOMMITTED,
070     mayInstantiate=true,
071     mayExtend=false,
072     mayInvoke=true)
073public class LDIFReader implements Closeable
074{
075  private static final LocalizedLogger logger = LocalizedLogger.getLoggerForThisClass();
076
077  /** The reader that will be used to read the data. */
078  private BufferedReader reader;
079  /** The import configuration that specifies what should be imported. */
080  protected final LDIFImportConfig importConfig;
081
082  /** The lines that comprise the body of the last entry read. */
083  protected List<StringBuilder> lastEntryBodyLines;
084  /** The lines that comprise the header (DN and any comments) for the last entry read. */
085  protected List<StringBuilder> lastEntryHeaderLines;
086
087  /**
088   * The number of entries that have been ignored by this LDIF reader because
089   * they didn't match the criteria.
090   */
091  private final AtomicLong entriesIgnored = new AtomicLong();
092  /**
093   * The number of entries that have been read by this LDIF reader, including
094   * those that were ignored because they didn't match the criteria, and
095   * including those that were rejected because they were invalid in some way.
096   */
097  protected final AtomicLong entriesRead = new AtomicLong();
098  /** The number of entries that have been rejected by this LDIF reader. */
099  private final AtomicLong entriesRejected = new AtomicLong();
100
101  /** The line number on which the last entry started. */
102  protected long lastEntryLineNumber = -1;
103  /** The line number of the last line read from the LDIF file, starting with 1. */
104  private long lineNumber;
105
106  /**
107   * The plugin config manager that will be used if we are to invoke plugins on
108   * the entries as they are read.
109   */
110  protected final PluginConfigManager pluginConfigManager;
111
112  /**
113   * Creates a new LDIF reader that will read information from the specified
114   * file.
115   *
116   * @param  importConfig  The import configuration for this LDIF reader.  It
117   *                       must not be <CODE>null</CODE>.
118   *
119   * @throws  IOException  If a problem occurs while opening the LDIF file for
120   *                       reading.
121   */
122  public LDIFReader(LDIFImportConfig importConfig)
123         throws IOException
124  {
125    ifNull(importConfig);
126    this.importConfig = importConfig;
127
128    reader               = importConfig.getReader();
129    lastEntryBodyLines   = new LinkedList<>();
130    lastEntryHeaderLines = new LinkedList<>();
131    pluginConfigManager  = DirectoryServer.getPluginConfigManager();
132    // If we should invoke import plugins, then do so.
133    if (importConfig.invokeImportPlugins())
134    {
135      // Inform LDIF import plugins that an import session is ending
136      pluginConfigManager.invokeLDIFImportBeginPlugins(importConfig);
137    }
138  }
139
140
141  /**
142   * Reads the next entry from the LDIF source.
143   *
144   * @return  The next entry read from the LDIF source, or <CODE>null</CODE> if
145   *          the end of the LDIF data is reached.
146   *
147   * @throws  IOException  If an I/O problem occurs while reading from the file.
148   *
149   * @throws  LDIFException  If the information read cannot be parsed as an LDIF
150   *                         entry.
151   */
152  public Entry readEntry()
153         throws IOException, LDIFException
154  {
155    return readEntry(importConfig.validateSchema());
156  }
157
158
159
160  /**
161   * Reads the next entry from the LDIF source.
162   *
163   * @param  checkSchema  Indicates whether this reader should perform schema
164   *                      checking on the entry before returning it to the
165   *                      caller.  Note that some basic schema checking (like
166   *                      refusing multiple values for a single-valued
167   *                      attribute) may always be performed.
168   *
169   *
170   * @return  The next entry read from the LDIF source, or <CODE>null</CODE> if
171   *          the end of the LDIF data is reached.
172   *
173   * @throws  IOException  If an I/O problem occurs while reading from the file.
174   *
175   * @throws  LDIFException  If the information read cannot be parsed as an LDIF
176   *                         entry.
177   */
178  public Entry readEntry(boolean checkSchema)
179         throws IOException, LDIFException
180  {
181    while (true)
182    {
183      // Read the set of lines that make up the next entry.
184      LinkedList<StringBuilder> lines = readEntryLines();
185      if (lines == null)
186      {
187        return null;
188      }
189      lastEntryBodyLines   = lines;
190      lastEntryHeaderLines = new LinkedList<>();
191
192
193      // Read the DN of the entry and see if it is one that should be included
194      // in the import.
195      DN entryDN = readDN(lines);
196      if (entryDN == null)
197      {
198        // This should only happen if the LDIF starts with the "version:" line
199        // and has a blank line immediately after that.  In that case, simply
200        // read and return the next entry.
201        continue;
202      }
203      else if (!importConfig.includeEntry(entryDN))
204      {
205        logger.trace("Skipping entry %s because the DN is not one that "
206            + "should be included based on the include and exclude branches.", entryDN);
207        entriesRead.incrementAndGet();
208        logToSkipWriter(lines, ERR_LDIF_SKIP.get(entryDN));
209        continue;
210      }
211      else
212      {
213        entriesRead.incrementAndGet();
214      }
215
216      // Create the entry and see if it is one that should be included in the import.
217      final Entry entry = createEntry(entryDN, lines, checkSchema);
218      if (!isIncludedInImport(entry,lines)
219          || !invokeImportPlugins(entry, lines))
220      {
221        continue;
222      }
223      validateAgainstSchemaIfNeeded(checkSchema, entry, lines);
224
225      // The entry should be included in the import, so return it.
226      return entry;
227    }
228  }
229
230  private Entry createEntry(DN entryDN, List<StringBuilder> lines, boolean checkSchema) throws LDIFException
231  {
232    Map<ObjectClass, String> objectClasses = new HashMap<>();
233    Map<AttributeType, List<AttributeBuilder>> userAttrBuilders = new HashMap<>();
234    Map<AttributeType, List<AttributeBuilder>> operationalAttrBuilders = new HashMap<>();
235    for (StringBuilder line : lines)
236    {
237      readAttribute(lines, line, entryDN, objectClasses, userAttrBuilders, operationalAttrBuilders, checkSchema);
238    }
239
240    final Entry entry = new Entry(entryDN, objectClasses,
241        toAttributesMap(userAttrBuilders), toAttributesMap(operationalAttrBuilders));
242    logger.trace("readEntry(), created entry: %s", entry);
243    return entry;
244  }
245
246  private boolean isIncludedInImport(Entry entry, LinkedList<StringBuilder> lines) throws LDIFException
247  {
248    try
249    {
250      if (!importConfig.includeEntry(entry))
251      {
252        final DN entryDN = entry.getName();
253        logger.trace("Skipping entry %s because the DN is not one that "
254            + "should be included based on the include and exclude filters.", entryDN);
255        logToSkipWriter(lines, ERR_LDIF_SKIP.get(entryDN));
256        return false;
257      }
258      return true;
259    }
260    catch (Exception e)
261    {
262      logger.traceException(e);
263
264      LocalizableMessage message =
265          ERR_LDIF_COULD_NOT_EVALUATE_FILTERS_FOR_IMPORT.get(entry.getName(), lastEntryLineNumber, e);
266      throw new LDIFException(message, lastEntryLineNumber, true, e);
267    }
268  }
269
270  private boolean invokeImportPlugins(Entry entry, LinkedList<StringBuilder> lines)
271  {
272    if (importConfig.invokeImportPlugins())
273    {
274      PluginResult.ImportLDIF pluginResult =
275          pluginConfigManager.invokeLDIFImportPlugins(importConfig, entry);
276      if (!pluginResult.continueProcessing())
277      {
278        final DN entryDN = entry.getName();
279        LocalizableMessage rejectMessage = pluginResult.getErrorMessage();
280        LocalizableMessage m = rejectMessage != null
281            ? ERR_LDIF_REJECTED_BY_PLUGIN.get(entryDN, rejectMessage)
282            : ERR_LDIF_REJECTED_BY_PLUGIN_NOMESSAGE.get(entryDN);
283
284        logToRejectWriter(lines, m);
285        return false;
286      }
287    }
288    return true;
289  }
290
291  private void validateAgainstSchemaIfNeeded(boolean checkSchema, final Entry entry, LinkedList<StringBuilder> lines)
292      throws LDIFException
293  {
294    if (checkSchema)
295    {
296      LocalizableMessageBuilder invalidReason = new LocalizableMessageBuilder();
297      if (!entry.conformsToSchema(null, false, true, false, invalidReason))
298      {
299        final DN entryDN = entry.getName();
300        LocalizableMessage message = ERR_LDIF_SCHEMA_VIOLATION.get(entryDN, lastEntryLineNumber, invalidReason);
301        logToRejectWriter(lines, message);
302        throw new LDIFException(message, lastEntryLineNumber, true);
303      }
304      // Add any superior objectclass(s) missing in an entries objectclass map.
305      addSuperiorObjectClasses(entry.getObjectClasses());
306    }
307  }
308
309  /**
310   * Returns a new Map where the provided Map with AttributeBuilders is converted to another Map
311   * with Attributes.
312   *
313   * @param attrBuilders
314   *          the provided Map containing AttributeBuilders
315   * @return a new Map containing Attributes
316   */
317  protected Map<AttributeType, List<Attribute>> toAttributesMap(Map<AttributeType, List<AttributeBuilder>> attrBuilders)
318  {
319    Map<AttributeType, List<Attribute>> attributes = new HashMap<>(attrBuilders.size());
320    for (Map.Entry<AttributeType, List<AttributeBuilder>> attrTypeEntry : attrBuilders.entrySet())
321    {
322      AttributeType attrType = attrTypeEntry.getKey();
323      List<Attribute> attrList = toAttributesList(attrTypeEntry.getValue());
324      attributes.put(attrType, attrList);
325    }
326    return attributes;
327  }
328
329  /**
330   * Converts the provided List of AttributeBuilders to a new list of Attributes.
331   *
332   * @param builders the list of AttributeBuilders
333   * @return a new list of Attributes
334   */
335  private List<Attribute> toAttributesList(List<AttributeBuilder> builders)
336  {
337    List<Attribute> results = new ArrayList<>(builders.size());
338    for (AttributeBuilder builder : builders)
339    {
340      results.add(builder.toAttribute());
341    }
342    return results;
343  }
344
345  /**
346   * Reads the next change record from the LDIF source.
347   *
348   * @param  defaultAdd  Indicates whether the change type should default to
349   *                     "add" if none is explicitly provided.
350   *
351   * @return  The next change record from the LDIF source, or <CODE>null</CODE>
352   *          if the end of the LDIF data is reached.
353   *
354   * @throws  IOException  If an I/O problem occurs while reading from the file.
355   *
356   * @throws  LDIFException  If the information read cannot be parsed as an LDIF
357   *                         entry.
358   */
359  public ChangeRecordEntry readChangeRecord(boolean defaultAdd)
360         throws IOException, LDIFException
361  {
362    while (true)
363    {
364      // Read the set of lines that make up the next entry.
365      LinkedList<StringBuilder> lines = readEntryLines();
366      if (lines == null)
367      {
368        return null;
369      }
370
371
372      // Read the DN of the entry and see if it is one that should be included
373      // in the import.
374      DN entryDN = readDN(lines);
375      if (entryDN == null)
376      {
377        // This should only happen if the LDIF starts with the "version:" line
378        // and has a blank line immediately after that.  In that case, simply
379        // read and return the next entry.
380        continue;
381      }
382
383      String changeType = readChangeType(lines);
384      if(changeType != null)
385      {
386        switch (changeType)
387        {
388        case "add":
389          return parseAddChangeRecordEntry(entryDN, lines);
390        case "delete":
391          return parseDeleteChangeRecordEntry(entryDN, lines);
392        case "modify":
393          return parseModifyChangeRecordEntry(entryDN, lines);
394        case "modrdn":
395          return parseModifyDNChangeRecordEntry(entryDN, lines);
396        case "moddn":
397          return parseModifyDNChangeRecordEntry(entryDN, lines);
398        default:
399          LocalizableMessage message = ERR_LDIF_INVALID_CHANGETYPE_ATTRIBUTE.get(
400              changeType, "add, delete, modify, moddn, modrdn");
401          throw new LDIFException(message, lastEntryLineNumber, false);
402        }
403      }
404      else if (defaultAdd)
405      {
406        // default to "add"
407        return parseAddChangeRecordEntry(entryDN, lines);
408      }
409      else
410      {
411        LocalizableMessage message =
412            ERR_LDIF_INVALID_CHANGETYPE_ATTRIBUTE.get(null, "add, delete, modify, moddn, modrdn");
413        throw new LDIFException(message, lastEntryLineNumber, false);
414      }
415    }
416  }
417
418
419
420  /**
421   * Reads a set of lines from the next entry in the LDIF source.
422   *
423   * @return  A set of lines from the next entry in the LDIF source.
424   *
425   * @throws  IOException  If a problem occurs while reading from the LDIF
426   *                       source.
427   *
428   * @throws  LDIFException  If the information read is not valid LDIF.
429   */
430  protected LinkedList<StringBuilder> readEntryLines() throws IOException, LDIFException
431  {
432    if(reader == null)
433    {
434      return null;
435    }
436
437    // Read the entry lines into a buffer.
438    LinkedList<StringBuilder> lines = new LinkedList<>();
439    int lastLine = -1;
440    while (true)
441    {
442      String line = reader.readLine();
443      lineNumber++;
444
445      if (line == null)
446      {
447        // This must mean that we have reached the end of the LDIF source.
448        // If the set of lines read so far is empty, then move onto the next
449        // file or return null.  Otherwise, break out of this loop.
450        if (!lines.isEmpty())
451        {
452          break;
453        }
454        reader = importConfig.nextReader();
455        return reader != null ? readEntryLines() : null;
456      }
457      else if (line.length() == 0)
458      {
459        // This is a blank line.  If the set of lines read so far is empty,
460        // then just skip over it.  Otherwise, break out of this loop.
461        if (!lines.isEmpty())
462        {
463          break;
464        }
465        continue;
466      }
467      else if (line.charAt(0) == '#')
468      {
469        // This is a comment.  Ignore it.
470        continue;
471      }
472      else if (line.charAt(0) == ' ' || line.charAt(0) == '\t')
473      {
474        // This is a continuation of the previous line.  If there is no
475        // previous line, then that's a problem.  Note that while RFC 2849
476        // technically only allows a space in this position, both OpenLDAP and
477        // the Sun Java System Directory Server allow a tab as well, so we will
478        // too for compatibility reasons.  See issue #852 for details.
479        if (lastLine >= 0)
480        {
481          lines.get(lastLine).append(line.substring(1));
482        }
483        else
484        {
485          LocalizableMessage message =
486                  ERR_LDIF_INVALID_LEADING_SPACE.get(lineNumber, line);
487          logToRejectWriter(lines, message);
488          throw new LDIFException(message, lineNumber, false);
489        }
490      }
491      else
492      {
493        // This is a new line.
494        if (lines.isEmpty())
495        {
496          lastEntryLineNumber = lineNumber;
497        }
498        if(((byte)line.charAt(0) == (byte)0xEF) &&
499          ((byte)line.charAt(1) == (byte)0xBB) &&
500          ((byte)line.charAt(2) == (byte)0xBF))
501        {
502          // This is a UTF-8 BOM that Java doesn't skip. We will skip it here.
503          line = line.substring(3, line.length());
504        }
505        lines.add(new StringBuilder(line));
506        lastLine++;
507      }
508    }
509
510
511    return lines;
512  }
513
514
515
516  /**
517   * Reads the DN of the entry from the provided list of lines.  The DN must be
518   * the first line in the list, unless the first line starts with "version",
519   * in which case the DN should be the second line.
520   *
521   * @param  lines  The set of lines from which the DN should be read.
522   *
523   * @return  The decoded entry DN.
524   *
525   * @throws  LDIFException  If DN is not the first element in the list (or the
526   *                         second after the LDIF version), or if a problem
527   *                         occurs while trying to parse it.
528   */
529  protected DN readDN(LinkedList<StringBuilder> lines) throws LDIFException
530  {
531    if (lines.isEmpty())
532    {
533      // This is possible if the contents of the first "entry" were just
534      // the version identifier.  If that is the case, then return null and
535      // use that as a signal to the caller to go ahead and read the next entry.
536      return null;
537    }
538
539    StringBuilder line = lines.remove();
540    lastEntryHeaderLines.add(line);
541    int colonPos = line.indexOf(":");
542    if (colonPos <= 0)
543    {
544      LocalizableMessage message =
545              ERR_LDIF_NO_ATTR_NAME.get(lastEntryLineNumber, line);
546
547      logToRejectWriter(lines, message);
548      throw new LDIFException(message, lastEntryLineNumber, true);
549    }
550
551    String attrName = toLowerCase(line.substring(0, colonPos));
552    if (attrName.equals("version"))
553    {
554      // This is the version line, and we can skip it.
555      return readDN(lines);
556    }
557    else if (! attrName.equals("dn"))
558    {
559      LocalizableMessage message =
560              ERR_LDIF_NO_DN.get(lastEntryLineNumber, line);
561
562      logToRejectWriter(lines, message);
563      throw new LDIFException(message, lastEntryLineNumber, true);
564    }
565
566
567    // Look at the character immediately after the colon.  If there is none,
568    // then assume the null DN.  If it is another colon, then the DN must be
569    // base64-encoded.  Otherwise, it may be one or more spaces.
570    if (colonPos == line.length() - 1)
571    {
572      return DN.rootDN();
573    }
574
575    if (line.charAt(colonPos+1) == ':')
576    {
577      // The DN is base64-encoded.  Find the first non-blank character and
578      // take the rest of the line, base64-decode it, and parse it as a DN.
579      int pos = findFirstNonSpaceCharPosition(line, colonPos + 2);
580      String dnStr = base64Decode(line.substring(pos), lines, line);
581      return decodeDN(dnStr, lines, line);
582    }
583    else
584    {
585      // The rest of the value should be the DN.  Skip over any spaces and
586      // attempt to decode the rest of the line as the DN.
587      int pos = findFirstNonSpaceCharPosition(line, colonPos + 1);
588      return decodeDN(line.substring(pos), lines, line);
589    }
590  }
591
592  private int findFirstNonSpaceCharPosition(StringBuilder line, int startPos)
593  {
594    final int length = line.length();
595    int pos = startPos;
596    while (pos < length && line.charAt(pos) == ' ')
597    {
598      pos++;
599    }
600    return pos;
601  }
602
603  private String base64Decode(String encodedStr, List<StringBuilder> lines,
604      StringBuilder line) throws LDIFException
605  {
606    try
607    {
608      return new String(Base64.decode(encodedStr), "UTF-8");
609    }
610    catch (Exception e)
611    {
612      // The value did not have a valid base64-encoding.
613      final String stackTrace = StaticUtils.stackTraceToSingleLineString(e);
614      if (logger.isTraceEnabled())
615      {
616        logger.trace(
617            "Base64 decode failed for dn '%s', exception stacktrace: %s",
618            encodedStr, stackTrace);
619      }
620
621      LocalizableMessage message = ERR_LDIF_COULD_NOT_BASE64_DECODE_DN.get(
622          lastEntryLineNumber, line, stackTrace);
623      logToRejectWriter(lines, message);
624      throw new LDIFException(message, lastEntryLineNumber, true, e);
625    }
626  }
627
628  private DN decodeDN(String dnString, List<StringBuilder> lines,
629      StringBuilder line) throws LDIFException
630  {
631    try
632    {
633      return DN.valueOf(dnString);
634    }
635    catch (Exception e)
636    {
637      logger.trace("DN decode failed for: ", dnString, e);
638      LocalizableMessage message = ERR_LDIF_INVALID_DN.get(lastEntryLineNumber, line, getExceptionMessage(e));
639      logToRejectWriter(lines, message);
640      throw new LDIFException(message, lastEntryLineNumber, true, e);
641    }
642  }
643
644  /**
645   * Reads the changetype of the entry from the provided list of lines.  If
646   * there is no changetype attribute then an add is assumed.
647   *
648   * @param  lines  The set of lines from which the DN should be read.
649   *
650   * @return  The decoded entry DN.
651   *
652   * @throws  LDIFException  If DN is not the first element in the list (or the
653   *                         second after the LDIF version), or if a problem
654   *                         occurs while trying to parse it.
655   */
656  private String readChangeType(LinkedList<StringBuilder> lines)
657          throws LDIFException
658  {
659    if (lines.isEmpty())
660    {
661      // Error. There must be other entries.
662      return null;
663    }
664
665    StringBuilder line = lines.get(0);
666    lastEntryHeaderLines.add(line);
667    int colonPos = line.indexOf(":");
668    if (colonPos <= 0)
669    {
670      LocalizableMessage message = ERR_LDIF_NO_ATTR_NAME.get(lastEntryLineNumber, line);
671      logToRejectWriter(lines, message);
672      throw new LDIFException(message, lastEntryLineNumber, true);
673    }
674
675    String attrName = toLowerCase(line.substring(0, colonPos));
676    if (! attrName.equals("changetype"))
677    {
678      // No changetype attribute - return null
679      return null;
680    }
681    // Remove the line
682    lines.remove();
683
684
685    // Look at the character immediately after the colon.  If there is none,
686    // then no value was specified. Throw an exception
687    int length = line.length();
688    if (colonPos == (length-1))
689    {
690      LocalizableMessage message = ERR_LDIF_INVALID_CHANGETYPE_ATTRIBUTE.get(
691          null, "add, delete, modify, moddn, modrdn");
692      throw new LDIFException(message, lastEntryLineNumber, false );
693    }
694
695    if (line.charAt(colonPos+1) == ':')
696    {
697      // The change type is base64-encoded.  Find the first non-blank character
698      // and take the rest of the line, and base64-decode it.
699      int pos = findFirstNonSpaceCharPosition(line, colonPos + 2);
700      return base64Decode(line.substring(pos), lines, line);
701    }
702    else
703    {
704      // The rest of the value should be the changetype. Skip over any spaces
705      // and attempt to decode the rest of the line as the changetype string.
706      int pos = findFirstNonSpaceCharPosition(line, colonPos + 1);
707      return line.substring(pos);
708    }
709  }
710
711
712  /**
713   * Decodes the provided line as an LDIF attribute and adds it to the
714   * appropriate hash.
715   *
716   * @param  lines                  The full set of lines that comprise the
717   *                                entry (used for writing reject information).
718   * @param  line                   The line to decode.
719   * @param  entryDN                The DN of the entry being decoded.
720   * @param  objectClasses          The set of objectclasses decoded so far for
721   *                                the current entry.
722   * @param userAttrBuilders        The map of user attribute builders decoded
723   *                                so far for the current entry.
724   * @param  operationalAttrBuilders  The map of operational attribute builders
725   *                                  decoded so far for the current entry.
726   * @param  checkSchema            Indicates whether to perform schema
727   *                                validation for the attribute.
728   *
729   * @throws  LDIFException  If a problem occurs while trying to decode the
730   *                         attribute contained in the provided entry.
731   */
732  protected void readAttribute(List<StringBuilder> lines,
733       StringBuilder line, DN entryDN,
734       Map<ObjectClass,String> objectClasses,
735       Map<AttributeType,List<AttributeBuilder>> userAttrBuilders,
736       Map<AttributeType,List<AttributeBuilder>> operationalAttrBuilders,
737       boolean checkSchema)
738          throws LDIFException
739  {
740    // Parse the attribute type description.
741    int colonPos = parseColonPosition(lines, line);
742    String attrDescStr = line.substring(0, colonPos);
743    final AttributeDescription attrDesc = parseAttrDescription(attrDescStr);
744    final AttributeType attrType = attrDesc.getAttributeType();
745
746    // Now parse the attribute value.
747    ByteString value = parseSingleValue(lines, line, entryDN, colonPos, attrDescStr);
748
749    // See if this is an objectclass or an attribute.  Then get the
750    // corresponding definition and add the value to the appropriate hash.
751    if (attrType.isObjectClass())
752    {
753      if (! importConfig.includeObjectClasses())
754      {
755        logger.trace("Skipping objectclass %s for entry %s due to the import configuration.", value, entryDN);
756        return;
757      }
758
759      String ocName      = value.toString().trim();
760      ObjectClass objectClass = DirectoryServer.getSchema().getObjectClass(ocName);
761      if (objectClasses.containsKey(objectClass))
762      {
763        logger.warn(WARN_LDIF_DUPLICATE_OBJECTCLASS, entryDN, lastEntryLineNumber, ocName);
764      }
765      else
766      {
767        objectClasses.put(objectClass, ocName);
768      }
769    }
770    else
771    {
772      if (! importConfig.includeAttribute(attrType))
773      {
774        logger.trace("Skipping attribute %s for entry %s due to the import configuration.", attrDescStr, entryDN);
775        return;
776      }
777
778       //The attribute is not being ignored so check for binary option.
779      if (checkSchema
780          && !attrType.getSyntax().isBEREncodingRequired()
781          && attrDesc.hasOption("binary"))
782      {
783        LocalizableMessage message = ERR_LDIF_INVALID_ATTR_OPTION.get(entryDN, lastEntryLineNumber, attrDescStr);
784        logToRejectWriter(lines, message);
785        throw new LDIFException(message, lastEntryLineNumber,true);
786      }
787      if (checkSchema &&
788          DirectoryServer.getSyntaxEnforcementPolicy() != AcceptRejectWarn.ACCEPT)
789      {
790        LocalizableMessageBuilder invalidReason = new LocalizableMessageBuilder();
791        if (! attrType.getSyntax().valueIsAcceptable(value, invalidReason))
792        {
793          LocalizableMessage message = WARN_LDIF_VALUE_VIOLATES_SYNTAX.get(
794              entryDN, lastEntryLineNumber, value, attrDescStr, invalidReason);
795          if (DirectoryServer.getSyntaxEnforcementPolicy() == AcceptRejectWarn.WARN)
796          {
797            logger.error(message);
798          }
799          else
800          {
801            logToRejectWriter(lines, message);
802            throw new LDIFException(message, lastEntryLineNumber, true);
803          }
804        }
805      }
806
807      ByteString attributeValue = value;
808      final Map<AttributeType, List<AttributeBuilder>> attrBuilders;
809      if (attrType.isOperational())
810      {
811        attrBuilders = operationalAttrBuilders;
812      }
813      else
814      {
815        attrBuilders = userAttrBuilders;
816      }
817
818      final List<AttributeBuilder> attrList = attrBuilders.get(attrType);
819      if (attrList == null)
820      {
821        AttributeBuilder builder = new AttributeBuilder(attrDesc);
822        builder.add(attributeValue);
823        attrBuilders.put(attrType, newArrayList(builder));
824        return;
825      }
826
827      // Check to see if any of the attributes in the list have the same set of
828      // options.  If so, then try to add a value to that attribute.
829      for (AttributeBuilder a : attrList)
830      {
831        if (a.optionsEqual(attrDesc))
832        {
833          if (!a.add(attributeValue) && checkSchema)
834          {
835              LocalizableMessage message = WARN_LDIF_DUPLICATE_ATTR.get(
836                  entryDN, lastEntryLineNumber, attrDescStr, value);
837              logToRejectWriter(lines, message);
838            throw new LDIFException(message, lastEntryLineNumber, true);
839          }
840          if (attrType.isSingleValue() && a.size() > 1 && checkSchema)
841          {
842            LocalizableMessage message = ERR_LDIF_MULTIPLE_VALUES_FOR_SINGLE_VALUED_ATTR
843                    .get(entryDN, lastEntryLineNumber, attrDescStr);
844            logToRejectWriter(lines, message);
845            throw new LDIFException(message, lastEntryLineNumber, true);
846          }
847
848          return;
849        }
850      }
851
852      // No set of matching options was found, so create a new one and
853      // add it to the list.
854      AttributeBuilder builder = new AttributeBuilder(attrDesc);
855      builder.add(attributeValue);
856      attrList.add(builder);
857    }
858  }
859
860
861
862  /**
863   * Decodes the provided line as an LDIF attribute and returns the
864   * Attribute (name and values) for the specified attribute name.
865   *
866   * @param  lines                  The full set of lines that comprise the
867   *                                entry (used for writing reject information).
868   * @param  line                   The line to decode.
869   * @param  entryDN                The DN of the entry being decoded.
870   * @param  attributeName          The name and options of the attribute to
871   *                                return the values for.
872   *
873   * @return                        The attribute in octet string form.
874   * @throws  LDIFException         If a problem occurs while trying to decode
875   *                                the attribute contained in the provided
876   *                                entry or if the parsed attribute name does
877   *                                not match the specified attribute name.
878   */
879  private Attribute readSingleValueAttribute(
880       List<StringBuilder> lines, StringBuilder line, DN entryDN,
881       String attributeName) throws LDIFException
882  {
883    // Parse the attribute type description.
884    int colonPos = parseColonPosition(lines, line);
885    String attrDescStr = line.substring(0, colonPos);
886    AttributeDescription attrDesc = parseAttrDescription(attrDescStr);
887
888    if (attributeName != null)
889    {
890      AttributeDescription expectedAttrDesc = parseAttrDescription(attributeName);
891      if (!attrDesc.equals(expectedAttrDesc))
892      {
893        LocalizableMessage message = ERR_LDIF_INVALID_CHANGERECORD_ATTRIBUTE.get(attrDescStr, attributeName);
894        throw new LDIFException(message, lastEntryLineNumber, false);
895      }
896    }
897
898    //  Now parse the attribute value.
899    ByteString value = parseSingleValue(lines, line, entryDN, colonPos, attrDescStr);
900
901    AttributeBuilder builder = new AttributeBuilder(attrDesc);
902    builder.add(value);
903    return builder.toAttribute();
904  }
905
906
907  /**
908   * Retrieves the starting line number for the last entry read from the LDIF
909   * source.
910   *
911   * @return  The starting line number for the last entry read from the LDIF
912   *          source.
913   */
914  public long getLastEntryLineNumber()
915  {
916    return lastEntryLineNumber;
917  }
918
919
920
921  /**
922   * Rejects the last entry read from the LDIF.  This method is intended for use
923   * by components that perform their own validation of entries (e.g., backends
924   * during import processing) in which the entry appeared valid to the LDIF
925   * reader but some other problem was encountered.
926   *
927   * @param  message  A human-readable message providing the reason that the
928   *                  last entry read was not acceptable.
929   */
930  public void rejectLastEntry(LocalizableMessage message)
931  {
932    entriesRejected.incrementAndGet();
933
934    BufferedWriter rejectWriter = importConfig.getRejectWriter();
935    if (rejectWriter != null)
936    {
937      try
938      {
939        if (message != null && message.length() > 0)
940        {
941          rejectWriter.write("# ");
942          rejectWriter.write(message.toString());
943          rejectWriter.newLine();
944        }
945
946        for (StringBuilder sb : lastEntryHeaderLines)
947        {
948          rejectWriter.write(sb.toString());
949          rejectWriter.newLine();
950        }
951
952        for (StringBuilder sb : lastEntryBodyLines)
953        {
954          rejectWriter.write(sb.toString());
955          rejectWriter.newLine();
956        }
957
958        rejectWriter.newLine();
959      }
960      catch (Exception e)
961      {
962        logger.traceException(e);
963      }
964    }
965  }
966
967  /**
968   * Log the specified entry and messages in the reject writer. The method is
969   * intended to be used in a threaded environment, where individual import
970   * threads need to log an entry and message to the reject file.
971   *
972   * @param e The entry to log.
973   * @param message The message to log.
974   */
975  public synchronized void rejectEntry(Entry e, LocalizableMessage message) {
976    BufferedWriter rejectWriter = importConfig.getRejectWriter();
977    entriesRejected.incrementAndGet();
978    if (rejectWriter != null) {
979      try {
980        if (message != null && message.length() > 0) {
981          rejectWriter.write("# ");
982          rejectWriter.write(message.toString());
983          rejectWriter.newLine();
984        }
985        rejectWriter.write(e.getName().toString());
986        rejectWriter.newLine();
987        List<StringBuilder> eLDIF = e.toLDIF();
988        for(StringBuilder l : eLDIF) {
989          rejectWriter.write(l.toString());
990          rejectWriter.newLine();
991        }
992        rejectWriter.newLine();
993      } catch (IOException ex) {
994        logger.traceException(ex);
995      }
996    }
997  }
998
999  /** Closes this LDIF reader and the underlying file or input stream. */
1000  @Override
1001  public void close()
1002  {
1003    // If we should invoke import plugins, then do so.
1004    if (importConfig.invokeImportPlugins())
1005    {
1006      // Inform LDIF import plugins that an import session is ending
1007      pluginConfigManager.invokeLDIFImportEndPlugins(importConfig);
1008    }
1009    importConfig.close();
1010  }
1011
1012
1013
1014  /**
1015   * Parse an AttributeDescription (an attribute type name and its
1016   * options).
1017   *
1018   * @param attrDescr
1019   *          The attribute description to be parsed.
1020   * @return A new attribute with no values, representing the
1021   *         attribute type and its options.
1022   */
1023  public static AttributeDescription parseAttrDescription(String attrDescr)
1024  {
1025    AttributeDescription result = AttributeDescription.valueOf(attrDescr);
1026    if (result.getAttributeType().getSyntax().isBEREncodingRequired())
1027    {
1028      result = result.withOption("binary");
1029    }
1030    return result;
1031  }
1032
1033
1034
1035  /**
1036   * Retrieves the total number of entries read so far by this LDIF reader,
1037   * including those that have been ignored or rejected.
1038   *
1039   * @return  The total number of entries read so far by this LDIF reader.
1040   */
1041  public long getEntriesRead()
1042  {
1043    return entriesRead.get();
1044  }
1045
1046
1047
1048  /**
1049   * Retrieves the total number of entries that have been ignored so far by this
1050   * LDIF reader because they did not match the import criteria.
1051   *
1052   * @return  The total number of entries ignored so far by this LDIF reader.
1053   */
1054  public long getEntriesIgnored()
1055  {
1056    return entriesIgnored.get();
1057  }
1058
1059
1060
1061  /**
1062   * Retrieves the total number of entries rejected so far by this LDIF reader.
1063   * This  includes both entries that were rejected because  of internal
1064   * validation failure (e.g., they didn't conform to the defined  server
1065   * schema) or an external validation failure (e.g., the component using this
1066   * LDIF reader didn't accept the entry because it didn't have a parent).
1067   *
1068   * @return  The total number of entries rejected so far by this LDIF reader.
1069   */
1070  public long getEntriesRejected()
1071  {
1072    return entriesRejected.get();
1073  }
1074
1075
1076
1077  /**
1078   * Parse a modifyDN change record entry from LDIF.
1079   *
1080   * @param entryDN
1081   *          The name of the entry being modified.
1082   * @param lines
1083   *          The lines to parse.
1084   * @return Returns the parsed modifyDN change record entry.
1085   * @throws LDIFException
1086   *           If there was an error when parsing the change record.
1087   */
1088  private ChangeRecordEntry parseModifyDNChangeRecordEntry(DN entryDN,
1089      LinkedList<StringBuilder> lines) throws LDIFException {
1090
1091    DN newSuperiorDN = null;
1092    RDN newRDN;
1093    boolean deleteOldRDN;
1094
1095    if(lines.isEmpty())
1096    {
1097      LocalizableMessage message = ERR_LDIF_NO_MOD_DN_ATTRIBUTES.get();
1098      throw new LDIFException(message, lineNumber, true);
1099    }
1100
1101    StringBuilder line = lines.remove();
1102    String rdnStr = getModifyDNAttributeValue(lines, line, entryDN, "newrdn");
1103
1104    try
1105    {
1106      newRDN = RDN.valueOf(rdnStr);
1107    }
1108    catch (Exception e)
1109    {
1110      logger.traceException(e);
1111      LocalizableMessage message = ERR_LDIF_INVALID_DN.get(lineNumber, line, getExceptionMessage(e));
1112      throw new LDIFException(message, lineNumber, true);
1113    }
1114
1115    if(lines.isEmpty())
1116    {
1117      LocalizableMessage message = ERR_LDIF_NO_DELETE_OLDRDN_ATTRIBUTE.get();
1118      throw new LDIFException(message, lineNumber, true);
1119    }
1120    lineNumber++;
1121
1122    line = lines.remove();
1123    String delStr = getModifyDNAttributeValue(lines, line,
1124        entryDN, "deleteoldrdn");
1125
1126    if(delStr.equalsIgnoreCase("false") ||
1127        delStr.equalsIgnoreCase("no") ||
1128        delStr.equalsIgnoreCase("0"))
1129    {
1130      deleteOldRDN = false;
1131    } else if(delStr.equalsIgnoreCase("true") ||
1132        delStr.equalsIgnoreCase("yes") ||
1133        delStr.equalsIgnoreCase("1"))
1134    {
1135      deleteOldRDN = true;
1136    } else
1137    {
1138      LocalizableMessage message = ERR_LDIF_INVALID_DELETE_OLDRDN_ATTRIBUTE.get(delStr);
1139      throw new LDIFException(message, lineNumber, true);
1140    }
1141
1142    if(!lines.isEmpty())
1143    {
1144      lineNumber++;
1145
1146      line = lines.remove();
1147
1148      String dnStr = getModifyDNAttributeValue(lines, line,
1149          entryDN, "newsuperior");
1150      try
1151      {
1152        newSuperiorDN = DN.valueOf(dnStr);
1153      }
1154      catch (Exception e)
1155      {
1156        logger.traceException(e);
1157        LocalizableMessage message = ERR_LDIF_INVALID_DN.get(lineNumber, line, getExceptionMessage(e));
1158        throw new LDIFException(message, lineNumber, true);
1159      }
1160    }
1161
1162    return new ModifyDNChangeRecordEntry(entryDN, newRDN, deleteOldRDN,
1163                                         newSuperiorDN);
1164  }
1165
1166
1167
1168  /**
1169   * Return the string value for the specified attribute name which only
1170   * has one value.
1171   *
1172   * @param lines
1173   *          The set of lines for this change record entry.
1174   * @param line
1175   *          The line currently being examined.
1176   * @param entryDN
1177   *          The name of the entry being modified.
1178   * @param attributeName
1179   *          The attribute name
1180   * @return the string value for the attribute name.
1181   * @throws LDIFException
1182   *           If a problem occurs while attempting to determine the
1183   *           attribute value.
1184   */
1185  private String getModifyDNAttributeValue(List<StringBuilder> lines,
1186                                   StringBuilder line,
1187                                   DN entryDN,
1188                                   String attributeName) throws LDIFException
1189  {
1190    Attribute attr =
1191      readSingleValueAttribute(lines, line, entryDN, attributeName);
1192    return attr.iterator().next().toString();
1193  }
1194
1195
1196
1197  /**
1198   * Parse a modify change record entry from LDIF.
1199   *
1200   * @param entryDN
1201   *          The name of the entry being modified.
1202   * @param lines
1203   *          The lines to parse.
1204   * @return Returns the parsed modify change record entry.
1205   * @throws LDIFException
1206   *           If there was an error when parsing the change record.
1207   */
1208  private ChangeRecordEntry parseModifyChangeRecordEntry(DN entryDN,
1209      LinkedList<StringBuilder> lines) throws LDIFException {
1210
1211    List<RawModification> modifications = new ArrayList<>();
1212    while(!lines.isEmpty())
1213    {
1214      StringBuilder line = lines.remove();
1215      Attribute attr = readSingleValueAttribute(lines, line, entryDN, null);
1216
1217      // Get the attribute description
1218      String attrDescStr = attr.iterator().next().toString();
1219
1220      String name = attr.getAttributeDescription().getAttributeType().getNameOrOID();
1221      ModificationType modType = toModType(name);
1222
1223      // Now go through the rest of the attributes till the "-" line is reached.
1224      AttributeDescription modAttrDesc = LDIFReader.parseAttrDescription(attrDescStr);
1225      AttributeBuilder builder = new AttributeBuilder(modAttrDesc);
1226      while (! lines.isEmpty())
1227      {
1228        line = lines.remove();
1229        if(line.toString().equals("-"))
1230        {
1231          break;
1232        }
1233        builder.addAll(readSingleValueAttribute(lines, line, entryDN, attrDescStr));
1234      }
1235
1236      LDAPAttribute ldapAttr = new LDAPAttribute(builder.toAttribute());
1237      modifications.add(new LDAPModification(modType, ldapAttr));
1238    }
1239
1240    return new ModifyChangeRecordEntry(entryDN, modifications);
1241  }
1242
1243
1244  private ModificationType toModType(String name) throws LDIFException
1245  {
1246    if ("add".equalsIgnoreCase(name))
1247    {
1248      return ModificationType.ADD;
1249    }
1250    else if ("delete".equalsIgnoreCase(name))
1251    {
1252      return ModificationType.DELETE;
1253    }
1254    else if ("replace".equalsIgnoreCase(name))
1255    {
1256      return ModificationType.REPLACE;
1257    }
1258    else if ("increment".equalsIgnoreCase(name))
1259    {
1260      return ModificationType.INCREMENT;
1261    }
1262    else
1263    {
1264      // Invalid attribute name.
1265      LocalizableMessage message = ERR_LDIF_INVALID_MODIFY_ATTRIBUTE.get(name, "add, delete, replace, increment");
1266      throw new LDIFException(message, lineNumber, true);
1267    }
1268  }
1269
1270  /**
1271   * Parse a delete change record entry from LDIF.
1272   *
1273   * @param entryDN
1274   *          The name of the entry being deleted.
1275   * @param lines
1276   *          The lines to parse.
1277   * @return Returns the parsed delete change record entry.
1278   * @throws LDIFException
1279   *           If there was an error when parsing the change record.
1280   */
1281  private ChangeRecordEntry parseDeleteChangeRecordEntry(DN entryDN,
1282      List<StringBuilder> lines) throws LDIFException
1283  {
1284    if (!lines.isEmpty())
1285    {
1286      LocalizableMessage message = ERR_LDIF_INVALID_DELETE_ATTRIBUTES.get();
1287      throw new LDIFException(message, lineNumber, true);
1288    }
1289    return new DeleteChangeRecordEntry(entryDN);
1290  }
1291
1292
1293
1294  /**
1295   * Parse an add change record entry from LDIF.
1296   *
1297   * @param entryDN
1298   *          The name of the entry being added.
1299   * @param lines
1300   *          The lines to parse.
1301   * @return Returns the parsed add change record entry.
1302   * @throws LDIFException
1303   *           If there was an error when parsing the change record.
1304   */
1305  private ChangeRecordEntry parseAddChangeRecordEntry(DN entryDN,
1306      List<StringBuilder> lines) throws LDIFException
1307  {
1308    Map<ObjectClass, String> objectClasses = new HashMap<>();
1309    Map<AttributeType, List<AttributeBuilder>> attrBuilders = new HashMap<>();
1310    for(StringBuilder line : lines)
1311    {
1312      readAttribute(lines, line, entryDN, objectClasses,
1313          attrBuilders, attrBuilders, importConfig.validateSchema());
1314    }
1315
1316    // Reconstruct the object class attribute.
1317    AttributeType ocType = CoreSchema.getObjectClassAttributeType();
1318    AttributeBuilder builder = new AttributeBuilder(ocType);
1319    builder.addAllStrings(objectClasses.values());
1320    Map<AttributeType, List<Attribute>> attributes = toAttributesMap(attrBuilders);
1321    if (attributes.get(ocType) == null)
1322    {
1323      attributes.put(ocType, builder.toAttributeList());
1324    }
1325
1326    return new AddChangeRecordEntry(entryDN, attributes);
1327  }
1328
1329
1330
1331  /**
1332   * Parse colon position in an attribute description.
1333   *
1334   * @param lines
1335   *          The current set of lines.
1336   * @param line
1337   *          The current line.
1338   * @return The colon position.
1339   * @throws LDIFException
1340   *           If the colon was badly placed or not found.
1341   */
1342  private int parseColonPosition(List<StringBuilder> lines,
1343      StringBuilder line) throws LDIFException {
1344    int colonPos = line.indexOf(":");
1345    if (colonPos <= 0)
1346    {
1347      LocalizableMessage message = ERR_LDIF_NO_ATTR_NAME.get(
1348              lastEntryLineNumber, line);
1349      logToRejectWriter(lines, message);
1350      throw new LDIFException(message, lastEntryLineNumber, true);
1351    }
1352    return colonPos;
1353  }
1354
1355
1356
1357  /**
1358   * Parse a single attribute value from a line of LDIF.
1359   *
1360   * @param lines
1361   *          The current set of lines.
1362   * @param line
1363   *          The current line.
1364   * @param entryDN
1365   *          The DN of the entry being parsed.
1366   * @param colonPos
1367   *          The position of the separator colon in the line.
1368   * @param attrName
1369   *          The name of the attribute being parsed.
1370   * @return The parsed attribute value.
1371   * @throws LDIFException
1372   *           If an error occurred when parsing the attribute value.
1373   */
1374  private ByteString parseSingleValue(
1375      List<StringBuilder> lines,
1376      StringBuilder line,
1377      DN entryDN,
1378      int colonPos,
1379      String attrName) throws LDIFException {
1380
1381    // Look at the character immediately after the colon. If there is
1382    // none, then assume an attribute with an empty value. If it is another
1383    // colon, then the value must be base64-encoded. If it is a less-than
1384    // sign, then assume that it is a URL. Otherwise, it is a regular value.
1385    int length = line.length();
1386    ByteString value;
1387    if (colonPos == (length-1))
1388    {
1389      value = ByteString.empty();
1390    }
1391    else
1392    {
1393      char c = line.charAt(colonPos+1);
1394      if (c == ':')
1395      {
1396        // The value is base64-encoded. Find the first non-blank
1397        // character, take the rest of the line, and base64-decode it.
1398        int pos = findFirstNonSpaceCharPosition(line, colonPos + 2);
1399
1400        try
1401        {
1402          value = ByteString.wrap(Base64.decode(line.substring(pos)));
1403        }
1404        catch (Exception e)
1405        {
1406          // The value did not have a valid base64-encoding.
1407          logger.traceException(e);
1408
1409          LocalizableMessage message = ERR_LDIF_COULD_NOT_BASE64_DECODE_ATTR.get(
1410              entryDN, lastEntryLineNumber, line, e);
1411          logToRejectWriter(lines, message);
1412          throw new LDIFException(message, lastEntryLineNumber, true, e);
1413        }
1414      }
1415      else if (c == '<')
1416      {
1417        // Find the first non-blank character, decode the rest of the
1418        // line as a URL, and read its contents.
1419        int pos = findFirstNonSpaceCharPosition(line, colonPos + 2);
1420
1421        URL contentURL;
1422        try
1423        {
1424          contentURL = new URL(line.substring(pos));
1425        }
1426        catch (Exception e)
1427        {
1428          // The URL was malformed or had an invalid protocol.
1429          logger.traceException(e);
1430
1431          LocalizableMessage message = ERR_LDIF_INVALID_URL.get(
1432              entryDN, lastEntryLineNumber, attrName, e);
1433          logToRejectWriter(lines, message);
1434          throw new LDIFException(message, lastEntryLineNumber, true, e);
1435        }
1436
1437
1438        InputStream inputStream = null;
1439        try
1440        {
1441          ByteStringBuilder builder = new ByteStringBuilder(4096);
1442          inputStream  = contentURL.openConnection().getInputStream();
1443
1444          while (builder.appendBytes(inputStream, 4096) != -1) { /* Do nothing */ }
1445
1446          value = builder.toByteString();
1447        }
1448        catch (Exception e)
1449        {
1450          // We were unable to read the contents of that URL for some reason.
1451          logger.traceException(e);
1452
1453          LocalizableMessage message = ERR_LDIF_URL_IO_ERROR.get(
1454              entryDN, lastEntryLineNumber, attrName, contentURL, e);
1455          logToRejectWriter(lines, message);
1456          throw new LDIFException(message, lastEntryLineNumber, true, e);
1457        }
1458        finally
1459        {
1460          StaticUtils.close(inputStream);
1461        }
1462      }
1463      else
1464      {
1465        // The rest of the line should be the value. Skip over any
1466        // spaces and take the rest of the line as the value.
1467        int pos = findFirstNonSpaceCharPosition(line, colonPos + 1);
1468        value = ByteString.valueOfUtf8(line.substring(pos));
1469      }
1470    }
1471    return value;
1472  }
1473
1474  /**
1475   * Log a message to the reject writer if one is configured.
1476   *
1477   * @param lines
1478   *          The set of rejected lines.
1479   * @param message
1480   *          The associated error message.
1481   */
1482  protected void logToRejectWriter(List<StringBuilder> lines, LocalizableMessage message)
1483  {
1484    entriesRejected.incrementAndGet();
1485    BufferedWriter rejectWriter = importConfig.getRejectWriter();
1486    if (rejectWriter != null)
1487    {
1488      logToWriter(rejectWriter, lines, message);
1489    }
1490  }
1491
1492  /**
1493   * Log a message to the reject writer if one is configured.
1494   *
1495   * @param lines
1496   *          The set of rejected lines.
1497   * @param message
1498   *          The associated error message.
1499   */
1500  protected void logToSkipWriter(List<StringBuilder> lines, LocalizableMessage message)
1501  {
1502    entriesIgnored.incrementAndGet();
1503    BufferedWriter skipWriter = importConfig.getSkipWriter();
1504    if (skipWriter != null)
1505    {
1506      logToWriter(skipWriter, lines, message);
1507    }
1508  }
1509
1510  /**
1511   * Log a message to the given writer.
1512   *
1513   * @param writer
1514   *          The writer to write to.
1515   * @param lines
1516   *          The set of rejected lines.
1517   * @param message
1518   *          The associated error message.
1519   */
1520  private void logToWriter(BufferedWriter writer, List<StringBuilder> lines,
1521      LocalizableMessage message)
1522  {
1523    if (writer != null)
1524    {
1525      try
1526      {
1527        writer.write("# ");
1528        writer.write(String.valueOf(message));
1529        writer.newLine();
1530        for (StringBuilder sb : lines)
1531        {
1532          writer.write(sb.toString());
1533          writer.newLine();
1534        }
1535
1536        writer.newLine();
1537      }
1538      catch (Exception e)
1539      {
1540        logger.traceException(e);
1541      }
1542    }
1543  }
1544
1545
1546  /**
1547   * Adds any missing RDN attributes to the entry that is being imported.
1548   * @param entryDN the entry DN
1549   * @param userAttributes the user attributes
1550   * @param operationalAttributes the operational attributes
1551   */
1552  protected void addRDNAttributesIfNecessary(DN entryDN,
1553          Map<AttributeType,List<Attribute>>userAttributes,
1554          Map<AttributeType,List<Attribute>> operationalAttributes)
1555  {
1556    for (AVA ava : entryDN.rdn())
1557    {
1558      AttributeType t = ava.getAttributeType();
1559      addRDNAttributesIfNecessary(t.isOperational() ? operationalAttributes : userAttributes, ava);
1560    }
1561  }
1562
1563
1564  private void addRDNAttributesIfNecessary(Map<AttributeType, List<Attribute>> attributes, AVA ava)
1565  {
1566    AttributeType t = ava.getAttributeType();
1567    String n = ava.getAttributeName();
1568    ByteString v = ava.getAttributeValue();
1569    final List<Attribute> attrList = attributes.get(t);
1570    if (attrList == null)
1571    {
1572      attributes.put(t, newArrayList(Attributes.create(t, n, v)));
1573      return;
1574    }
1575
1576    for (int j = 0; j < attrList.size(); j++)
1577    {
1578      Attribute a = attrList.get(j);
1579      if (a.getAttributeDescription().hasOptions())
1580      {
1581        continue;
1582      }
1583
1584      if (!a.contains(v))
1585      {
1586        AttributeBuilder builder = new AttributeBuilder(a);
1587        builder.add(v);
1588        attrList.set(j, builder.toAttribute());
1589      }
1590
1591      return;
1592    }
1593
1594    // not found
1595    attrList.add(Attributes.create(t, n, v));
1596  }
1597}