001/*
002 * The contents of this file are subject to the terms of the Common Development and
003 * Distribution License (the License). You may not use this file except in compliance with the
004 * License.
005 *
006 * You can obtain a copy of the License at legal/CDDLv1.0.txt. See the License for the
007 * specific language governing permission and limitations under the License.
008 *
009 * When distributing Covered Software, include this CDDL Header Notice in each file and include
010 * the License file at legal/CDDLv1.0.txt. If applicable, add the following below the CDDL
011 * Header, with the fields enclosed by brackets [] replaced by your own identifying
012 * information: "Portions Copyright [year] [name of copyright owner]".
013 *
014 * Copyright 2006-2009 Sun Microsystems, Inc.
015 * Portions Copyright 2013-2016 ForgeRock AS.
016 */
017package org.opends.server.tools.makeldif;
018
019import static org.opends.messages.ToolMessages.*;
020import static org.opends.server.util.StaticUtils.*;
021
022import java.io.BufferedReader;
023import java.io.File;
024import java.io.FileReader;
025import java.io.IOException;
026import java.util.ArrayList;
027import java.util.HashMap;
028import java.util.LinkedHashMap;
029import java.util.List;
030import java.util.Map;
031import java.util.Random;
032import java.util.StringTokenizer;
033
034import org.forgerock.i18n.LocalizableMessage;
035import org.forgerock.opendj.ldap.DN;
036import org.forgerock.opendj.ldap.schema.AttributeType;
037import org.opends.server.core.DirectoryServer;
038import org.opends.server.types.InitializationException;
039
040/**
041 * This class defines a template file, which is a collection of constant
042 * definitions, branches, and templates.
043 */
044public class TemplateFile
045{
046  /** The name of the file holding the list of first names. */
047  private static final String FIRST_NAME_FILE = "first.names";
048  /** The name of the file holding the list of last names. */
049  private static final String LAST_NAME_FILE = "last.names";
050
051  /**
052   * A map of the contents of various text files used during the parsing
053   * process, mapped from absolute path to the array of lines in the file.
054   */
055  private final Map<String, String[]> fileLines = new HashMap<>();
056
057  /** The index of the next first name value that should be used. */
058  private int firstNameIndex;
059  /** The index of the next last name value that should be used. */
060  private int lastNameIndex;
061
062  /**
063   * A counter used to keep track of the number of times that the larger of the
064   * first/last name list has been completed.
065   */
066  private int nameLoopCounter;
067  /**
068   * A counter that will be used in case we have exhausted all possible first
069   * and last name combinations.
070   */
071  private int nameUniquenessCounter;
072
073  /** The set of branch definitions for this template file. */
074  private final LinkedHashMap<DN, Branch> branches = new LinkedHashMap<>();
075  /** The set of constant definitions for this template file. */
076  private final LinkedHashMap<String, String> constants = new LinkedHashMap<>();
077  /** The set of registered tags for this template file. */
078  private final LinkedHashMap<String, Tag> registeredTags = new LinkedHashMap<>();
079  /** The set of template definitions for this template file. */
080  private final LinkedHashMap<String, Template> templates = new LinkedHashMap<>();
081
082  /** The random number generator for this template file. */
083  private final Random random;
084
085  /** The next first name that should be used. */
086  private String firstName;
087  /** The next last name that should be used. */
088  private String lastName;
089
090  /** The resource path to use for filesystem elements that cannot be found anywhere else. */
091  private final String resourcePath;
092  /** The path to the directory containing the template file, if available. */
093  private String templatePath;
094
095  /** The set of first names to use when generating the LDIF. */
096  private String[] firstNames;
097  /** The set of last names to use when generating the LDIF. */
098  private String[] lastNames;
099
100  /**
101   * Creates a new, empty template file structure.
102   *
103   *
104   * @param  resourcePath  The path to the directory that may contain additional
105   *                       resource files needed during the LDIF generation
106   *                       process.
107   * @param  random        The random number generator for this template file.
108   */
109  public TemplateFile(String resourcePath, Random random)
110  {
111    this.resourcePath = resourcePath;
112    this.random       = random;
113
114    firstNames            = new String[0];
115    lastNames             = new String[0];
116    nameUniquenessCounter = 1;
117
118    registerDefaultTags();
119
120    try
121    {
122      readNameFiles();
123    }
124    catch (IOException ioe)
125    {
126      // FIXME -- What to do here?
127      ioe.printStackTrace();
128      firstNames = new String[] { "John" };
129      lastNames  = new String[] { "Doe" };
130    }
131  }
132
133  /**
134   * Retrieves the set of tags that have been registered.  They will be in the
135   * form of a mapping between the name of the tag (in all lowercase characters)
136   * and the corresponding tag implementation.
137   *
138   * @return  The set of tags that have been registered.
139   */
140  public Map<String,Tag> getTags()
141  {
142    return registeredTags;
143  }
144
145  /**
146   * Retrieves the tag with the specified name.
147   *
148   * @param  lowerName  The name of the tag to retrieve, in all lowercase
149   *                    characters.
150   *
151   * @return  The requested tag, or <CODE>null</CODE> if no such tag has been
152   *          registered.
153   */
154  private Tag getTag(String lowerName)
155  {
156    return registeredTags.get(lowerName);
157  }
158
159  /** Registers the set of tags that will always be available for use in templates. */
160  private void registerDefaultTags()
161  {
162    Class<?>[] defaultTagClasses =
163    {
164      AttributeValueTag.class,
165      DNTag.class,
166      FileTag.class,
167      FirstNameTag.class,
168      GUIDTag.class,
169      IfAbsentTag.class,
170      IfPresentTag.class,
171      LastNameTag.class,
172      ListTag.class,
173      ParentDNTag.class,
174      PresenceTag.class,
175      RandomTag.class,
176      RDNTag.class,
177      SequentialTag.class,
178      StaticTextTag.class,
179      UnderscoreDNTag.class,
180      UnderscoreParentDNTag.class
181    };
182
183    for (Class<?> c : defaultTagClasses)
184    {
185      try
186      {
187        Tag t = (Tag) c.newInstance();
188        registeredTags.put(toLowerCase(t.getName()), t);
189      }
190      catch (Exception e)
191      {
192        // This should never happen.
193        e.printStackTrace();
194      }
195    }
196  }
197
198  /**
199   * Retrieves the set of constants defined for this template file.
200   *
201   * @return  The set of constants defined for this template file.
202   */
203  public Map<String,String> getConstants()
204  {
205    return constants;
206  }
207
208  /**
209   * Retrieves the set of branches defined in this template file.
210   *
211   * @return  The set of branches defined in this template file.
212   */
213  public Map<DN,Branch> getBranches()
214  {
215    return branches;
216  }
217
218  /**
219   * Retrieves the set of templates defined in this template file.
220   *
221   * @return  The set of templates defined in this template file.
222   */
223  public Map<String,Template> getTemplates()
224  {
225    return templates;
226  }
227
228  /**
229   * Retrieves the random number generator for this template file.
230   *
231   * @return  The random number generator for this template file.
232   */
233  public Random getRandom()
234  {
235    return random;
236  }
237
238  /**
239   * Reads the contents of the first and last name files into the appropriate
240   * arrays and sets up the associated index pointers.
241   *
242   * @throws  IOException  If a problem occurs while reading either of the
243   *                       files.
244   */
245  private void readNameFiles()
246          throws IOException
247  {
248    File f = getFile(FIRST_NAME_FILE);
249    List<String> nameList = readLines(f);
250    firstNames = new String[nameList.size()];
251    nameList.toArray(firstNames);
252
253    f = getFile(LAST_NAME_FILE);
254    nameList = readLines(f);
255    lastNames = new String[nameList.size()];
256    nameList.toArray(lastNames);
257  }
258
259  private List<String> readLines(File f) throws IOException
260  {
261    try (BufferedReader reader = new BufferedReader(new FileReader(f)))
262    {
263      ArrayList<String> lines = new ArrayList<>();
264      while (true)
265      {
266        String line = reader.readLine();
267        if (line == null)
268        {
269          break;
270        }
271        lines.add(line);
272      }
273      return lines;
274    }
275  }
276
277  /**
278   * Updates the first and last name indexes to choose new values.  The
279   * algorithm used is designed to ensure that the combination of first and last
280   * names will never be repeated.  It depends on the number of first names and
281   * the number of last names being relatively prime.  This method should be
282   * called before beginning generation of each template entry.
283   */
284  public void nextFirstAndLastNames()
285  {
286    firstName = firstNames[firstNameIndex++];
287    lastName  = lastNames[lastNameIndex++];
288
289    // If we've already exhausted every possible combination, then append an
290    // integer to the last name.
291    if (nameUniquenessCounter > 1)
292    {
293      lastName += nameUniquenessCounter;
294    }
295
296    if (firstNameIndex >= firstNames.length)
297    {
298      // We're at the end of the first name list, so start over.  If the first
299      // name list is larger than the last name list, then we'll also need to
300      // set the last name index to the next loop counter position.
301      firstNameIndex = 0;
302      if (firstNames.length > lastNames.length)
303      {
304        lastNameIndex = ++nameLoopCounter;
305        if (lastNameIndex >= lastNames.length)
306        {
307          lastNameIndex = 0;
308          nameUniquenessCounter++;
309        }
310      }
311    }
312
313    if (lastNameIndex >= lastNames.length)
314    {
315      // We're at the end of the last name list, so start over.  If the last
316      // name list is larger than the first name list, then we'll also need to
317      // set the first name index to the next loop counter position.
318      lastNameIndex = 0;
319      if (lastNames.length > firstNames.length)
320      {
321        firstNameIndex = ++nameLoopCounter;
322        if (firstNameIndex >= firstNames.length)
323        {
324          firstNameIndex = 0;
325          nameUniquenessCounter++;
326        }
327      }
328    }
329  }
330
331  /**
332   * Retrieves the first name value that should be used for the current entry.
333   *
334   * @return  The first name value that should be used for the current entry.
335   */
336  public String getFirstName()
337  {
338    return firstName;
339  }
340
341  /**
342   * Retrieves the last name value that should be used for the current entry.
343   *
344   * @return  The last name value that should be used for the current entry.
345   */
346  public String getLastName()
347  {
348    return lastName;
349  }
350
351  /**
352   * Parses the contents of the specified file as a MakeLDIF template file
353   * definition.
354   *
355   * @param  filename  The name of the file containing the template data.
356   * @param  warnings  A list into which any warnings identified may be placed.
357   *
358   * @throws  IOException  If a problem occurs while attempting to read data
359   *                       from the specified file.
360   *
361   * @throws  InitializationException  If a problem occurs while initializing
362   *                                   any of the MakeLDIF components.
363   *
364   * @throws  MakeLDIFException  If any other problem occurs while parsing the
365   *                             template file.
366   */
367  public void parse(String filename, List<LocalizableMessage> warnings)
368         throws IOException, InitializationException, MakeLDIFException
369  {
370    templatePath = null;
371    File f = getFile(filename);
372    if (f == null || !f.exists())
373    {
374      LocalizableMessage message = ERR_MAKELDIF_COULD_NOT_FIND_TEMPLATE_FILE.get(filename);
375      throw new IOException(message.toString());
376    }
377    templatePath = f.getParentFile().getAbsolutePath();
378
379    List<String> fileLines = readLines(f);
380    String[] lines = fileLines.toArray(new String[fileLines.size()]);
381    parse(lines, warnings);
382  }
383
384  /**
385   * Parses the provided data as a MakeLDIF template file definition.
386   *
387   * @param  lines  The lines that make up the template file.
388   * @param  warnings  A list into which any warnings identified may be placed.
389   *
390   * @throws  InitializationException  If a problem occurs while initializing
391   *                                   any of the MakeLDIF components.
392   *
393   * @throws  MakeLDIFException  If any other problem occurs while parsing the
394   *                             template file.
395   */
396  public void parse(String[] lines, List<LocalizableMessage> warnings)
397         throws InitializationException, MakeLDIFException
398  {
399    // Create temporary variables that will be used to hold the data read.
400    LinkedHashMap<String,Tag> templateFileIncludeTags = new LinkedHashMap<>();
401    LinkedHashMap<String,String> templateFileConstants = new LinkedHashMap<>();
402    LinkedHashMap<DN,Branch> templateFileBranches = new LinkedHashMap<>();
403    LinkedHashMap<String,Template> templateFileTemplates = new LinkedHashMap<>();
404
405    for (int lineNumber=0; lineNumber < lines.length; lineNumber++)
406    {
407      String line = lines[lineNumber];
408
409      line = replaceConstants(line, lineNumber,
410                              templateFileConstants, warnings);
411
412      String lowerLine = toLowerCase(line);
413      if (line.length() == 0 || line.startsWith("#"))
414      {
415        // This is a comment or a blank line, so we'll ignore it.
416        continue;
417      }
418      else if (lowerLine.startsWith("include "))
419      {
420        // This should be an include definition.  The next element should be the
421        // name of the class.  Load and instantiate it and make sure there are
422        // no conflicts.
423        String className = line.substring(8).trim();
424
425        Class<?> tagClass;
426        try
427        {
428          tagClass = Class.forName(className);
429        }
430        catch (Exception e)
431        {
432          LocalizableMessage message = ERR_MAKELDIF_CANNOT_LOAD_TAG_CLASS.get(className);
433          throw new MakeLDIFException(message, e);
434        }
435
436        Tag tag;
437        try
438        {
439          tag = (Tag) tagClass.newInstance();
440        }
441        catch (Exception e)
442        {
443          LocalizableMessage message = ERR_MAKELDIF_CANNOT_INSTANTIATE_TAG.get(className);
444          throw new MakeLDIFException(message, e);
445        }
446
447        String lowerName = toLowerCase(tag.getName());
448        if (registeredTags.containsKey(lowerName) ||
449            templateFileIncludeTags.containsKey(lowerName))
450        {
451          LocalizableMessage message =
452              ERR_MAKELDIF_CONFLICTING_TAG_NAME.get(className, tag.getName());
453          throw new MakeLDIFException(message);
454        }
455
456        templateFileIncludeTags.put(lowerName, tag);
457      }
458      else if (lowerLine.startsWith("define "))
459      {
460        // This should be a constant definition.  The rest of the line should
461        // contain the constant name, an equal sign, and the constant value.
462        int equalPos = line.indexOf('=', 7);
463        if (equalPos < 0)
464        {
465          LocalizableMessage message = ERR_MAKELDIF_DEFINE_MISSING_EQUALS.get(lineNumber);
466          throw new MakeLDIFException(message);
467        }
468
469        String name  = line.substring(7, equalPos).trim();
470        if (name.length() == 0)
471        {
472          LocalizableMessage message = ERR_MAKELDIF_DEFINE_NAME_EMPTY.get(lineNumber);
473          throw new MakeLDIFException(message);
474        }
475
476        String lowerName = toLowerCase(name);
477        if (templateFileConstants.containsKey(lowerName))
478        {
479          LocalizableMessage message =
480              ERR_MAKELDIF_CONFLICTING_CONSTANT_NAME.get(name, lineNumber);
481          throw new MakeLDIFException(message);
482        }
483
484        String value = line.substring(equalPos+1);
485        if (value.length() == 0)
486        {
487          LocalizableMessage message = ERR_MAKELDIF_WARNING_DEFINE_VALUE_EMPTY.get(
488                  name, lineNumber);
489          warnings.add(message);
490        }
491
492        templateFileConstants.put(lowerName, value);
493      }
494      else if (lowerLine.startsWith("branch: "))
495      {
496        int startLineNumber = lineNumber;
497        ArrayList<String> lineList = new ArrayList<>();
498        lineList.add(line);
499        while (true)
500        {
501          lineNumber++;
502          if (lineNumber >= lines.length)
503          {
504            break;
505          }
506
507          line = lines[lineNumber];
508          if (line.length() == 0)
509          {
510            break;
511          }
512          line = replaceConstants(line, lineNumber, templateFileConstants, warnings);
513          lineList.add(line);
514        }
515
516        String[] branchLines = new String[lineList.size()];
517        lineList.toArray(branchLines);
518
519        Branch b = parseBranchDefinition(branchLines, lineNumber,
520                                         templateFileIncludeTags,
521            warnings);
522        DN branchDN = b.getBranchDN();
523        if (templateFileBranches.containsKey(branchDN))
524        {
525          LocalizableMessage message = ERR_MAKELDIF_CONFLICTING_BRANCH_DN.get(branchDN, startLineNumber);
526          throw new MakeLDIFException(message);
527        }
528        templateFileBranches.put(branchDN, b);
529      }
530      else if (lowerLine.startsWith("template: "))
531      {
532        int startLineNumber = lineNumber;
533        ArrayList<String> lineList = new ArrayList<>();
534        lineList.add(line);
535        while (true)
536        {
537          lineNumber++;
538          if (lineNumber >= lines.length)
539          {
540            break;
541          }
542
543          line = lines[lineNumber];
544          if (line.length() == 0)
545          {
546            break;
547          }
548          line = replaceConstants(line, lineNumber, templateFileConstants, warnings);
549          lineList.add(line);
550        }
551
552        String[] templateLines = new String[lineList.size()];
553        lineList.toArray(templateLines);
554
555        Template t = parseTemplateDefinition(templateLines, startLineNumber,
556                                             templateFileIncludeTags,
557                                             templateFileTemplates, warnings);
558        String lowerName = toLowerCase(t.getName());
559        if (templateFileTemplates.containsKey(lowerName))
560        {
561          LocalizableMessage message = ERR_MAKELDIF_CONFLICTING_TEMPLATE_NAME.get(t.getName(), startLineNumber);
562          throw new MakeLDIFException(message);
563        }
564        templateFileTemplates.put(lowerName, t);
565      }
566      else
567      {
568        LocalizableMessage message =
569            ERR_MAKELDIF_UNEXPECTED_TEMPLATE_FILE_LINE.get(line, lineNumber);
570        throw new MakeLDIFException(message);
571      }
572    }
573
574    // If we've gotten here, then we're almost done.  We just need to finalize
575    // the branch and template definitions and then update the template file
576    // variables.
577    for (Branch b : templateFileBranches.values())
578    {
579      b.completeBranchInitialization(templateFileTemplates);
580    }
581
582    for (Template t : templateFileTemplates.values())
583    {
584      t.completeTemplateInitialization(templateFileTemplates);
585    }
586
587    registeredTags.putAll(templateFileIncludeTags);
588    constants.putAll(templateFileConstants);
589    branches.putAll(templateFileBranches);
590    templates.putAll(templateFileTemplates);
591  }
592
593  /**
594   * Parse a line and replace all constants within [ ] with their
595   * values.
596   *
597   * @param line        The line to parse.
598   * @param lineNumber  The line number in the template file.
599   * @param constants   The set of constants defined in the template file.
600   * @param warnings    A list into which any warnings identified may be
601   *                    placed.
602   * @return The line in which all constant variables have been replaced
603   *         with their value
604   */
605  private String replaceConstants(String line, int lineNumber,
606                                  Map<String,String> constants,
607                                  List<LocalizableMessage> warnings)
608  {
609    int closePos = line.lastIndexOf(']');
610    // Loop until we've scanned all closing brackets
611    do
612    {
613      // Skip escaped closing brackets
614      while (closePos > 0 &&
615          line.charAt(closePos - 1) == '\\')
616      {
617        closePos = line.lastIndexOf(']', closePos - 1);
618      }
619      if (closePos > 0)
620      {
621        StringBuilder lineBuffer = new StringBuilder(line);
622        int openPos = line.lastIndexOf('[', closePos);
623        // Find the opening bracket. If it's escaped, then it's not a constant
624        if ((openPos > 0 && line.charAt(openPos - 1) != '\\')
625            || openPos == 0)
626        {
627          String constantName =
628              toLowerCase(line.substring(openPos+1, closePos));
629          String constantValue = constants.get(constantName);
630          if (constantValue == null)
631          {
632            LocalizableMessage message = WARN_MAKELDIF_WARNING_UNDEFINED_CONSTANT.get(
633                constantName, lineNumber);
634            warnings.add(message);
635          }
636          else
637          {
638            lineBuffer.replace(openPos, closePos+1, constantValue);
639          }
640        }
641        if (openPos >= 0)
642        {
643          closePos = openPos;
644        }
645        line = lineBuffer.toString();
646        closePos = line.lastIndexOf(']', closePos);
647      }
648    } while (closePos > 0);
649    return line;
650  }
651
652  /**
653   * Parses the information contained in the provided set of lines as a MakeLDIF
654   * branch definition.
655   *
656   *
657   * @param  branchLines      The set of lines containing the branch definition.
658   * @param  startLineNumber  The line number in the template file on which the
659   *                          first of the branch lines appears.
660   * @param  tags             The set of defined tags from the template file.
661   *                          Note that this does not include the tags that are
662   *                          always registered by default.
663   * @param  warnings         A list into which any warnings identified may be
664   *                          placed.
665   *
666   * @return  The decoded branch definition.
667   *
668   * @throws  InitializationException  If a problem occurs while initializing
669   *                                   any of the branch elements.
670   *
671   * @throws  MakeLDIFException  If some other problem occurs during processing.
672   */
673  private Branch parseBranchDefinition(String[] branchLines,
674                                       int startLineNumber,
675                                       Map<String, Tag> tags,
676                                       List<LocalizableMessage> warnings)
677          throws InitializationException, MakeLDIFException
678  {
679    // The first line must be "branch: " followed by the branch DN.
680    String dnString = branchLines[0].substring(8).trim();
681    DN branchDN;
682    try
683    {
684      branchDN = DN.valueOf(dnString);
685    }
686    catch (Exception e)
687    {
688      LocalizableMessage message =
689          ERR_MAKELDIF_CANNOT_DECODE_BRANCH_DN.get(dnString, startLineNumber);
690      throw new MakeLDIFException(message);
691    }
692
693    // Create a new branch that will be used for the verification process.
694    Branch branch = new Branch(this, branchDN);
695
696    for (int i=1; i < branchLines.length; i++)
697    {
698      String line       = branchLines[i];
699      String lowerLine  = toLowerCase(line);
700      int    lineNumber = startLineNumber + i;
701
702      if (lowerLine.startsWith("#"))
703      {
704        // It's a comment, so we should ignore it.
705        continue;
706      }
707      else if (lowerLine.startsWith("subordinatetemplate: "))
708      {
709        // It's a subordinate template, so we'll want to parse the name and the
710        // number of entries.
711        int colonPos = line.indexOf(':', 21);
712        if (colonPos <= 21)
713        {
714          LocalizableMessage message = ERR_MAKELDIF_BRANCH_SUBORDINATE_TEMPLATE_NO_COLON.
715              get(lineNumber, dnString);
716          throw new MakeLDIFException(message);
717        }
718
719        String templateName = line.substring(21, colonPos).trim();
720
721        int numEntries;
722        try
723        {
724          numEntries = Integer.parseInt(line.substring(colonPos+1).trim());
725          if (numEntries < 0)
726          {
727            LocalizableMessage message =
728              ERR_MAKELDIF_BRANCH_SUBORDINATE_INVALID_NUM_ENTRIES.
729                  get(lineNumber, dnString, numEntries, templateName);
730            throw new MakeLDIFException(message);
731          }
732          else if (numEntries == 0)
733          {
734            LocalizableMessage message = WARN_MAKELDIF_BRANCH_SUBORDINATE_ZERO_ENTRIES.get(
735                    lineNumber, dnString,
736                                        templateName);
737            warnings.add(message);
738          }
739
740          branch.addSubordinateTemplate(templateName, numEntries);
741        }
742        catch (NumberFormatException nfe)
743        {
744          LocalizableMessage message =
745            ERR_MAKELDIF_BRANCH_SUBORDINATE_CANT_PARSE_NUMENTRIES.
746                get(templateName, lineNumber, dnString);
747          throw new MakeLDIFException(message);
748        }
749      }
750      else
751      {
752        TemplateLine templateLine = parseTemplateLine(line, lowerLine,
753                                                      lineNumber, branch, null,
754                                                      tags, warnings);
755        branch.addExtraLine(templateLine);
756      }
757    }
758
759    return branch;
760  }
761
762  /**
763   * Parses the information contained in the provided set of lines as a MakeLDIF
764   * template definition.
765   *
766   *
767   * @param  templateLines     The set of lines containing the template
768   *                           definition.
769   * @param  startLineNumber   The line number in the template file on which the
770   *                           first of the template lines appears.
771   * @param  tags              The set of defined tags from the template file.
772   *                           Note that this does not include the tags that are
773   *                           always registered by default.
774   * @param  definedTemplates  The set of templates already defined in the
775   *                           template file.
776   * @param  warnings          A list into which any warnings identified may be
777   *                           placed.
778   *
779   * @return  The decoded template definition.
780   *
781   * @throws  InitializationException  If a problem occurs while initializing
782   *                                   any of the template elements.
783   *
784   * @throws  MakeLDIFException  If some other problem occurs during processing.
785   */
786  private Template parseTemplateDefinition(String[] templateLines,
787                                           int startLineNumber,
788                                           Map<String, Tag> tags,
789                                           Map<String, Template>
790                                               definedTemplates,
791                                           List<LocalizableMessage> warnings)
792          throws InitializationException, MakeLDIFException
793  {
794    // The first line must be "template: " followed by the template name.
795    String templateName = templateLines[0].substring(10).trim();
796
797    // The next line may start with either "extends: ", "rdnAttr: ", or
798    // "subordinateTemplate: ".  Keep reading until we find something that's
799    // not one of those.
800    int                arrayLineNumber    = 1;
801    Template           parentTemplate     = null;
802    AttributeType[]    rdnAttributes      = null;
803    ArrayList<String>  subTemplateNames   = new ArrayList<>();
804    ArrayList<Integer> entriesPerTemplate = new ArrayList<>();
805    for ( ; arrayLineNumber < templateLines.length; arrayLineNumber++)
806    {
807      int    lineNumber = startLineNumber + arrayLineNumber;
808      String line       = templateLines[arrayLineNumber];
809      String lowerLine  = toLowerCase(line);
810
811      if (lowerLine.startsWith("#"))
812      {
813        // It's a comment.  Ignore it.
814        continue;
815      }
816      else if (lowerLine.startsWith("extends: "))
817      {
818        String parentTemplateName = line.substring(9).trim();
819        parentTemplate = definedTemplates.get(parentTemplateName.toLowerCase());
820        if (parentTemplate == null)
821        {
822          LocalizableMessage message = ERR_MAKELDIF_TEMPLATE_INVALID_PARENT_TEMPLATE.get(
823              parentTemplateName, lineNumber, templateName);
824          throw new MakeLDIFException(message);
825        }
826      }
827      else if (lowerLine.startsWith("rdnattr: "))
828      {
829        // This is the set of RDN attributes.  If there are multiple, they may
830        // be separated by plus signs.
831        ArrayList<AttributeType> attrList = new ArrayList<>();
832        String rdnAttrNames = lowerLine.substring(9).trim();
833        StringTokenizer tokenizer = new StringTokenizer(rdnAttrNames, "+");
834        while (tokenizer.hasMoreTokens())
835        {
836          attrList.add(DirectoryServer.getSchema().getAttributeType(tokenizer.nextToken()));
837        }
838
839        rdnAttributes = new AttributeType[attrList.size()];
840        attrList.toArray(rdnAttributes);
841      }
842      else if (lowerLine.startsWith("subordinatetemplate: "))
843      {
844        // It's a subordinate template, so we'll want to parse the name and the
845        // number of entries.
846        int colonPos = line.indexOf(':', 21);
847        if (colonPos <= 21)
848        {
849          LocalizableMessage message = ERR_MAKELDIF_TEMPLATE_SUBORDINATE_TEMPLATE_NO_COLON.
850              get(lineNumber, templateName);
851          throw new MakeLDIFException(message);
852        }
853
854        String subTemplateName = line.substring(21, colonPos).trim();
855
856        int numEntries;
857        try
858        {
859          numEntries = Integer.parseInt(line.substring(colonPos+1).trim());
860          if (numEntries < 0)
861          {
862            LocalizableMessage message =
863              ERR_MAKELDIF_TEMPLATE_SUBORDINATE_INVALID_NUM_ENTRIES.
864                  get(lineNumber, templateName, numEntries, subTemplateName);
865            throw new MakeLDIFException(message);
866          }
867          else if (numEntries == 0)
868          {
869            LocalizableMessage message = WARN_MAKELDIF_TEMPLATE_SUBORDINATE_ZERO_ENTRIES
870                    .get(lineNumber, templateName, subTemplateName);
871            warnings.add(message);
872          }
873
874          subTemplateNames.add(subTemplateName);
875          entriesPerTemplate.add(numEntries);
876        }
877        catch (NumberFormatException nfe)
878        {
879          LocalizableMessage message =
880            ERR_MAKELDIF_TEMPLATE_SUBORDINATE_CANT_PARSE_NUMENTRIES.
881                get(subTemplateName, lineNumber, templateName);
882          throw new MakeLDIFException(message);
883        }
884      }
885      else
886      {
887        // It's something we don't recognize, so it must be a template line.
888        break;
889      }
890    }
891
892    // Create a new template that will be used for the verification process.
893    String[] subordinateTemplateNames = new String[subTemplateNames.size()];
894    subTemplateNames.toArray(subordinateTemplateNames);
895
896    int[] numEntriesPerTemplate = new int[entriesPerTemplate.size()];
897    for (int i=0; i < numEntriesPerTemplate.length; i++)
898    {
899      numEntriesPerTemplate[i] = entriesPerTemplate.get(i);
900    }
901
902    TemplateLine[] parsedLines;
903    if (parentTemplate == null)
904    {
905      parsedLines = new TemplateLine[0];
906    }
907    else
908    {
909      TemplateLine[] parentLines = parentTemplate.getTemplateLines();
910      parsedLines = new TemplateLine[parentLines.length];
911      System.arraycopy(parentLines, 0, parsedLines, 0, parentLines.length);
912    }
913
914    Template template = new Template(this, templateName, rdnAttributes,
915                                     subordinateTemplateNames,
916                                     numEntriesPerTemplate, parsedLines);
917
918    for ( ; arrayLineNumber < templateLines.length; arrayLineNumber++)
919    {
920      String line       = templateLines[arrayLineNumber];
921      String lowerLine  = toLowerCase(line);
922      int    lineNumber = startLineNumber + arrayLineNumber;
923
924      if (lowerLine.startsWith("#"))
925      {
926        // It's a comment, so we should ignore it.
927        continue;
928      }
929      template.addTemplateLine(parseTemplateLine(line, lowerLine, lineNumber, null, template, tags, warnings));
930    }
931
932    return template;
933  }
934
935  /**
936   * Parses the provided line as a template line.  Note that exactly one of the
937   * branch or template arguments must be non-null and the other must be null.
938   *
939   * @param  line        The text of the template line.
940   * @param  lowerLine   The template line in all lowercase characters.
941   * @param  lineNumber  The line number on which the template line appears.
942   * @param  branch      The branch with which the template line is associated.
943   * @param  template    The template with which the template line is
944   *                     associated.
945   * @param  tags        The set of defined tags from the template file.  Note
946   *                     that this does not include the tags that are always
947   *                     registered by default.
948   * @param  warnings    A list into which any warnings identified may be
949   *                     placed.
950   *
951   * @return  The template line that has been parsed.
952   *
953   * @throws  InitializationException  If a problem occurs while initializing
954   *                                   any of the template elements.
955   *
956   * @throws  MakeLDIFException  If some other problem occurs during processing.
957   */
958  private TemplateLine parseTemplateLine(String line, String lowerLine,
959                                         int lineNumber, Branch branch,
960                                         Template template,
961                                         Map<String,Tag> tags,
962                                         List<LocalizableMessage> warnings)
963          throws InitializationException, MakeLDIFException
964  {
965    // The first component must be the attribute type, followed by a colon.
966    int colonPos = lowerLine.indexOf(':');
967    if (colonPos < 0)
968    {
969      if (branch == null)
970      {
971        LocalizableMessage message = ERR_MAKELDIF_NO_COLON_IN_TEMPLATE_LINE.get(
972            lineNumber, template.getName());
973        throw new MakeLDIFException(message);
974      }
975      else
976      {
977        LocalizableMessage message = ERR_MAKELDIF_NO_COLON_IN_BRANCH_EXTRA_LINE.get(
978            lineNumber, branch.getBranchDN());
979        throw new MakeLDIFException(message);
980      }
981    }
982    else if (colonPos == 0)
983    {
984      if (branch == null)
985      {
986        LocalizableMessage message = ERR_MAKELDIF_NO_ATTR_IN_TEMPLATE_LINE.get(
987            lineNumber, template.getName());
988        throw new MakeLDIFException(message);
989      }
990      else
991      {
992        LocalizableMessage message = ERR_MAKELDIF_NO_ATTR_IN_BRANCH_EXTRA_LINE.get(
993            lineNumber, branch.getBranchDN());
994        throw new MakeLDIFException(message);
995      }
996    }
997
998    AttributeType attributeType = DirectoryServer.getSchema().getAttributeType(lowerLine.substring(0, colonPos));
999
1000    // First, check whether the value is an URL value: <attrName>:< <url>
1001    int length = line.length();
1002    int pos    = colonPos + 1;
1003    boolean valueIsURL = false;
1004    boolean valueIsBase64 = false;
1005    if (pos < length)
1006    {
1007      if (lowerLine.charAt(pos) == '<')
1008      {
1009        valueIsURL = true;
1010        pos ++;
1011      }
1012      else if (lowerLine.charAt(pos) == ':')
1013      {
1014        valueIsBase64 = true;
1015        pos ++;
1016      }
1017    }
1018    //  Then, find the position of the first non-blank character in the line.
1019    while (pos < length && lowerLine.charAt(pos) == ' ')
1020    {
1021      pos++;
1022    }
1023
1024    if (pos >= length)
1025    {
1026      // We've hit the end of the line with no value.  We'll allow it, but add a
1027      // warning.
1028      if (branch == null)
1029      {
1030        LocalizableMessage message = WARN_MAKELDIF_NO_VALUE_IN_TEMPLATE_LINE.get(
1031                lineNumber, template.getName());
1032        warnings.add(message);
1033      }
1034      else
1035      {
1036        LocalizableMessage message = WARN_MAKELDIF_NO_VALUE_IN_BRANCH_EXTRA_LINE.get(
1037                lineNumber, branch.getBranchDN());
1038        warnings.add(message);
1039      }
1040    }
1041
1042    // Define constants that specify what we're currently parsing.
1043    final int PARSING_STATIC_TEXT     = 0;
1044    final int PARSING_REPLACEMENT_TAG = 1;
1045    final int PARSING_ATTRIBUTE_TAG   = 2;
1046    final int PARSING_ESCAPED_CHAR    = 3;
1047
1048    int phase = PARSING_STATIC_TEXT;
1049    int previousPhase = PARSING_STATIC_TEXT;
1050
1051    ArrayList<Tag> tagList = new ArrayList<>();
1052    StringBuilder buffer = new StringBuilder();
1053
1054    for ( ; pos < length; pos++)
1055    {
1056      char c = line.charAt(pos);
1057      switch (phase)
1058      {
1059        case PARSING_STATIC_TEXT:
1060          switch (c)
1061          {
1062            case '\\':
1063              phase = PARSING_ESCAPED_CHAR;
1064              previousPhase = PARSING_STATIC_TEXT;
1065              break;
1066            case '<':
1067              if (buffer.length() > 0)
1068              {
1069                StaticTextTag t = new StaticTextTag();
1070                String[] args = new String[] { buffer.toString() };
1071                t.initializeForBranch(this, branch, args, lineNumber,
1072                    warnings);
1073                tagList.add(t);
1074                buffer = new StringBuilder();
1075              }
1076
1077              phase = PARSING_REPLACEMENT_TAG;
1078              break;
1079            case '{':
1080              if (buffer.length() > 0)
1081              {
1082                StaticTextTag t = new StaticTextTag();
1083                String[] args = new String[] { buffer.toString() };
1084                t.initializeForBranch(this, branch, args, lineNumber,
1085                                      warnings);
1086                tagList.add(t);
1087                buffer = new StringBuilder();
1088              }
1089
1090              phase = PARSING_ATTRIBUTE_TAG;
1091              break;
1092            default:
1093              buffer.append(c);
1094          }
1095          break;
1096
1097        case PARSING_REPLACEMENT_TAG:
1098          switch (c)
1099          {
1100            case '\\':
1101              phase = PARSING_ESCAPED_CHAR;
1102              previousPhase = PARSING_REPLACEMENT_TAG;
1103              break;
1104            case '>':
1105              Tag t = parseReplacementTag(buffer.toString(), branch, template,
1106                                          lineNumber, tags, warnings);
1107              tagList.add(t);
1108              buffer = new StringBuilder();
1109
1110              phase = PARSING_STATIC_TEXT;
1111              break;
1112            default:
1113              buffer.append(c);
1114              break;
1115          }
1116          break;
1117
1118        case PARSING_ATTRIBUTE_TAG:
1119          switch (c)
1120          {
1121            case '\\':
1122              phase = PARSING_ESCAPED_CHAR;
1123              previousPhase = PARSING_ATTRIBUTE_TAG;
1124              break;
1125            case '}':
1126              Tag t = parseAttributeTag(buffer.toString(), branch, template,
1127                                        lineNumber, warnings);
1128              tagList.add(t);
1129              buffer = new StringBuilder();
1130
1131              phase = PARSING_STATIC_TEXT;
1132              break;
1133            default:
1134              buffer.append(c);
1135              break;
1136          }
1137          break;
1138
1139        case PARSING_ESCAPED_CHAR:
1140          buffer.append(c);
1141          phase = previousPhase;
1142          break;
1143      }
1144    }
1145
1146    if (phase == PARSING_STATIC_TEXT)
1147    {
1148      if (buffer.length() > 0)
1149      {
1150        StaticTextTag t = new StaticTextTag();
1151        String[] args = new String[] { buffer.toString() };
1152        t.initializeForBranch(this, branch, args, lineNumber, warnings);
1153        tagList.add(t);
1154      }
1155    }
1156    else
1157    {
1158      LocalizableMessage message = ERR_MAKELDIF_INCOMPLETE_TAG.get(lineNumber);
1159      throw new InitializationException(message);
1160    }
1161
1162    Tag[] tagArray = new Tag[tagList.size()];
1163    tagList.toArray(tagArray);
1164    return new TemplateLine(attributeType, lineNumber, tagArray, valueIsURL,
1165        valueIsBase64);
1166  }
1167
1168  /**
1169   * Parses the provided string as a replacement tag.  Exactly one of the branch
1170   * or template must be null, and the other must be non-null.
1171   *
1172   * @param  tagString   The string containing the encoded tag.
1173   * @param  branch      The branch in which this tag appears.
1174   * @param  template    The template in which this tag appears.
1175   * @param  lineNumber  The line number on which this tag appears in the
1176   *                     template file.
1177   * @param  tags        The set of defined tags from the template file.  Note
1178   *                     that this does not include the tags that are always
1179   *                     registered by default.
1180   * @param  warnings    A list into which any warnings identified may be
1181   *                     placed.
1182   *
1183   * @return  The replacement tag parsed from the provided string.
1184   *
1185   * @throws  InitializationException  If a problem occurs while initializing
1186   *                                   the tag.
1187   *
1188   * @throws  MakeLDIFException  If some other problem occurs during processing.
1189   */
1190  private Tag parseReplacementTag(String tagString, Branch branch,
1191                                  Template template, int lineNumber,
1192                                  Map<String,Tag> tags,
1193                                  List<LocalizableMessage> warnings)
1194          throws InitializationException, MakeLDIFException
1195  {
1196    // The components of the replacement tag will be separated by colons, with
1197    // the first being the tag name and the remainder being arguments.
1198    StringTokenizer tokenizer = new StringTokenizer(tagString, ":");
1199    String          tagName      = tokenizer.nextToken().trim();
1200    String          lowerTagName = toLowerCase(tagName);
1201
1202    Tag t = getTag(lowerTagName);
1203    if (t == null)
1204    {
1205      t = tags.get(lowerTagName);
1206      if (t == null)
1207      {
1208        LocalizableMessage message = ERR_MAKELDIF_NO_SUCH_TAG.get(tagName, lineNumber);
1209        throw new MakeLDIFException(message);
1210      }
1211    }
1212
1213    ArrayList<String> argList = new ArrayList<>();
1214    while (tokenizer.hasMoreTokens())
1215    {
1216      argList.add(tokenizer.nextToken().trim());
1217    }
1218
1219    String[] args = new String[argList.size()];
1220    argList.toArray(args);
1221
1222    Tag newTag;
1223    try
1224    {
1225      newTag = t.getClass().newInstance();
1226    }
1227    catch (Exception e)
1228    {
1229      throw new MakeLDIFException(ERR_MAKELDIF_CANNOT_INSTANTIATE_NEW_TAG.get(tagName, lineNumber, e), e);
1230    }
1231
1232    if (branch == null)
1233    {
1234      newTag.initializeForTemplate(this, template, args, lineNumber, warnings);
1235    }
1236    else if (newTag.allowedInBranch())
1237    {
1238      newTag.initializeForBranch(this, branch, args, lineNumber, warnings);
1239    }
1240    else
1241    {
1242      throw new MakeLDIFException(ERR_MAKELDIF_TAG_NOT_ALLOWED_IN_BRANCH.get(newTag.getName(), lineNumber));
1243    }
1244
1245    return newTag;
1246  }
1247
1248  /**
1249   * Parses the provided string as an attribute tag.  Exactly one of the branch
1250   * or template must be null, and the other must be non-null.
1251   *
1252   * @param  tagString   The string containing the encoded tag.
1253   * @param  branch      The branch in which this tag appears.
1254   * @param  template    The template in which this tag appears.
1255   * @param  lineNumber  The line number on which this tag appears in the
1256   *                     template file.
1257   * @param  warnings    A list into which any warnings identified may be
1258   *                     placed.
1259   *
1260   * @return  The attribute tag parsed from the provided string.
1261   *
1262   * @throws  InitializationException  If a problem occurs while initializing
1263   *                                   the tag.
1264   *
1265   * @throws  MakeLDIFException  If some other problem occurs during processing.
1266   */
1267  private Tag parseAttributeTag(String tagString, Branch branch,
1268                                Template template, int lineNumber,
1269                                List<LocalizableMessage> warnings)
1270          throws InitializationException, MakeLDIFException
1271  {
1272    // The attribute tag must have at least one argument, which is the name of
1273    // the attribute to reference.  It may have a second argument, which is the
1274    // number of characters to use from the attribute value.  The arguments will
1275    // be delimited by colons.
1276    StringTokenizer   tokenizer = new StringTokenizer(tagString, ":");
1277    ArrayList<String> argList   = new ArrayList<>();
1278    while (tokenizer.hasMoreTokens())
1279    {
1280      argList.add(tokenizer.nextToken());
1281    }
1282
1283    String[] args = new String[argList.size()];
1284    argList.toArray(args);
1285
1286    AttributeValueTag tag = new AttributeValueTag();
1287    if (branch != null)
1288    {
1289      tag.initializeForBranch(this, branch, args, lineNumber, warnings);
1290    }
1291    else
1292    {
1293      tag.initializeForTemplate(this, template, args, lineNumber, warnings);
1294    }
1295
1296    return tag;
1297  }
1298
1299  /**
1300   * Retrieves a File object based on the provided path.  If the given path is
1301   * absolute, then that absolute path will be used.  If it is relative, then it
1302   * will first be evaluated relative to the current working directory.  If that
1303   * path doesn't exist, then it will be evaluated relative to the resource
1304   * path.  If that path doesn't exist, then it will be evaluated relative to
1305   * the directory containing the template file.
1306   *
1307   * @param  path  The path provided for the file.
1308   *
1309   * @return  The File object for the specified path, or <CODE>null</CODE> if
1310   *          the specified file could not be found.
1311   */
1312  public File getFile(String path)
1313  {
1314    // First, see if the file exists using the given path.  This will work if
1315    // the file is absolute, or it's relative to the current working directory.
1316    File f = new File(path);
1317    if (f.exists())
1318    {
1319      return f;
1320    }
1321
1322    // If the provided path was absolute, then use it anyway, even though we
1323    // couldn't find the file.
1324    if (f.isAbsolute())
1325    {
1326      return f;
1327    }
1328
1329    // Try a path relative to the resource directory.
1330    String newPath = resourcePath + File.separator + path;
1331    f = new File(newPath);
1332    if (f.exists())
1333    {
1334      return f;
1335    }
1336
1337    // Try a path relative to the template directory, if it's available.
1338    if (templatePath != null)
1339    {
1340      newPath = templatePath = File.separator + path;
1341      f = new File(newPath);
1342      if (f.exists())
1343      {
1344        return f;
1345      }
1346    }
1347
1348    return null;
1349  }
1350
1351  /**
1352   * Retrieves the lines of the specified file as a string array.  If the result
1353   * is already cached, then it will be used.  If the result is not cached, then
1354   * the file data will be cached so that the contents can be re-used if there
1355   * are multiple references to the same file.
1356   *
1357   * @param  file  The file for which to retrieve the contents.
1358   *
1359   * @return  An array containing the lines of the specified file.
1360   *
1361   * @throws  IOException  If a problem occurs while reading the file.
1362   */
1363  public String[] getFileLines(File file) throws IOException
1364  {
1365    String absolutePath = file.getAbsolutePath();
1366    String[] lines = fileLines.get(absolutePath);
1367    if (lines == null)
1368    {
1369      List<String> lineList = readLines(file);
1370
1371      lines = new String[lineList.size()];
1372      lineList.toArray(lines);
1373      lineList.clear();
1374      fileLines.put(absolutePath, lines);
1375    }
1376
1377    return lines;
1378  }
1379
1380  /**
1381   * Generates the LDIF content and writes it to the provided LDIF writer.
1382   *
1383   * @param  entryWriter  The entry writer that should be used to write the
1384   *                      entries.
1385   *
1386   * @return  The result that indicates whether processing should continue.
1387   *
1388   * @throws  IOException  If an error occurs while writing to the LDIF file.
1389   *
1390   * @throws  MakeLDIFException  If some other problem occurs.
1391   */
1392  public TagResult generateLDIF(EntryWriter entryWriter)
1393         throws IOException, MakeLDIFException
1394  {
1395    for (Branch b : branches.values())
1396    {
1397      TagResult result = b.writeEntries(entryWriter);
1398      if (!result.keepProcessingTemplateFile())
1399      {
1400        return result;
1401      }
1402    }
1403
1404    entryWriter.closeEntryWriter();
1405    return TagResult.SUCCESS_RESULT;
1406  }
1407}