001/*
002 * Copyright 2007-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2007-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2007-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldif;
037
038
039
040import java.io.BufferedReader;
041import java.io.Closeable;
042import java.io.File;
043import java.io.FileInputStream;
044import java.io.InputStream;
045import java.io.InputStreamReader;
046import java.io.IOException;
047import java.nio.charset.StandardCharsets;
048import java.text.ParseException;
049import java.util.ArrayList;
050import java.util.Collection;
051import java.util.Iterator;
052import java.util.HashSet;
053import java.util.LinkedHashMap;
054import java.util.List;
055import java.util.Set;
056import java.util.concurrent.BlockingQueue;
057import java.util.concurrent.ArrayBlockingQueue;
058import java.util.concurrent.TimeUnit;
059import java.util.concurrent.atomic.AtomicBoolean;
060import java.nio.charset.Charset;
061
062import com.unboundid.asn1.ASN1OctetString;
063import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
064import com.unboundid.ldap.matchingrules.MatchingRule;
065import com.unboundid.ldap.sdk.Attribute;
066import com.unboundid.ldap.sdk.Control;
067import com.unboundid.ldap.sdk.Entry;
068import com.unboundid.ldap.sdk.Modification;
069import com.unboundid.ldap.sdk.ModificationType;
070import com.unboundid.ldap.sdk.LDAPException;
071import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition;
072import com.unboundid.ldap.sdk.schema.Schema;
073import com.unboundid.util.AggregateInputStream;
074import com.unboundid.util.Base64;
075import com.unboundid.util.Debug;
076import com.unboundid.util.LDAPSDKThreadFactory;
077import com.unboundid.util.NotNull;
078import com.unboundid.util.Nullable;
079import com.unboundid.util.StaticUtils;
080import com.unboundid.util.ThreadSafety;
081import com.unboundid.util.ThreadSafetyLevel;
082import com.unboundid.util.Validator;
083import com.unboundid.util.parallel.AsynchronousParallelProcessor;
084import com.unboundid.util.parallel.Result;
085import com.unboundid.util.parallel.ParallelProcessor;
086import com.unboundid.util.parallel.Processor;
087
088import static com.unboundid.ldif.LDIFMessages.*;
089
090/**
091 * This class provides an LDIF reader, which can be used to read and decode
092 * entries and change records from a data source using the LDAP Data Interchange
093 * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
094 * <BR>
095 * This class is not synchronized.  If multiple threads read from the
096 * LDIFReader, they must be synchronized externally.
097 * <BR><BR>
098 * <H2>Example</H2>
099 * The following example iterates through all entries contained in an LDIF file
100 * and attempts to add them to a directory server:
101 * <PRE>
102 * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
103 *
104 * int entriesRead = 0;
105 * int entriesAdded = 0;
106 * int errorsEncountered = 0;
107 * while (true)
108 * {
109 *   Entry entry;
110 *   try
111 *   {
112 *     entry = ldifReader.readEntry();
113 *     if (entry == null)
114 *     {
115 *       // All entries have been read.
116 *       break;
117 *     }
118 *
119 *     entriesRead++;
120 *   }
121 *   catch (LDIFException le)
122 *   {
123 *     errorsEncountered++;
124 *     if (le.mayContinueReading())
125 *     {
126 *       // A recoverable error occurred while attempting to read a change
127 *       // record, at or near line number le.getLineNumber()
128 *       // The entry will be skipped, but we'll try to keep reading from the
129 *       // LDIF file.
130 *       continue;
131 *     }
132 *     else
133 *     {
134 *       // An unrecoverable error occurred while attempting to read an entry
135 *       // at or near line number le.getLineNumber()
136 *       // No further LDIF processing will be performed.
137 *       break;
138 *     }
139 *   }
140 *   catch (IOException ioe)
141 *   {
142 *     // An I/O error occurred while attempting to read from the LDIF file.
143 *     // No further LDIF processing will be performed.
144 *     errorsEncountered++;
145 *     break;
146 *   }
147 *
148 *   LDAPResult addResult;
149 *   try
150 *   {
151 *     addResult = connection.add(entry);
152 *     // If we got here, then the change should have been processed
153 *     // successfully.
154 *     entriesAdded++;
155 *   }
156 *   catch (LDAPException le)
157 *   {
158 *     // If we got here, then the change attempt failed.
159 *     addResult = le.toLDAPResult();
160 *     errorsEncountered++;
161 *   }
162 * }
163 *
164 * ldifReader.close();
165 * </PRE>
166 */
167@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
168public final class LDIFReader
169       implements Closeable
170{
171  /**
172   * The default buffer size (128KB) that will be used when reading from the
173   * data source.
174   */
175  public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
176
177
178
179  /*
180   * When processing asynchronously, this determines how many of the allocated
181   * worker threads are used to parse each batch of read entries.
182   */
183  private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
184
185
186
187  /**
188   * When processing asynchronously, this specifies the size of the pending and
189   * completed queues.
190   */
191  private static final int ASYNC_QUEUE_SIZE = 500;
192
193
194
195  /**
196   * Special entry used internally to signal that the LDIFReaderEntryTranslator
197   * has signalled that a read Entry should be skipped by returning null,
198   * which normally implies EOF.
199   */
200  @NotNull private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
201
202
203
204  /**
205   * The default base path that will be prepended to relative paths.  It will
206   * end with a trailing slash.
207   */
208  @NotNull private static final String DEFAULT_RELATIVE_BASE_PATH;
209  static
210  {
211    final File currentDir;
212    final String currentDirString = StaticUtils.getSystemProperty("user.dir");
213    if (currentDirString == null)
214    {
215      currentDir = new File(".");
216    }
217    else
218    {
219      currentDir = new File(currentDirString);
220    }
221
222    final String currentDirAbsolutePath = currentDir.getAbsolutePath();
223    if (currentDirAbsolutePath.endsWith(File.separator))
224    {
225      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
226    }
227    else
228    {
229      DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
230    }
231  }
232
233
234
235  /**
236   * A flag that indicates whether to support LDIF records that include
237   * controls.  If this is {@code true} (which is the default, unless the
238   * {@code com.unboundid.ldif.LDIFReader.disableControlSupport} system property
239   * is defined and set to a value of {@code true}), then a change record in
240   * which the line immediately after the DN starts with "control" will be
241   * interpreted as a control specification.
242   */
243  @NotNull private static final AtomicBoolean SUPPORT_CONTROLS;
244  static
245  {
246    final String propertyName =
247         LDIFReader.class.getName() + ".disableControlSupport";
248    final String disableControlSupportPropertyValue =
249         StaticUtils.getSystemProperty(propertyName);
250    if ((disableControlSupportPropertyValue != null) &&
251         (disableControlSupportPropertyValue.equalsIgnoreCase("true")))
252    {
253      SUPPORT_CONTROLS = new AtomicBoolean(false);
254    }
255    else
256    {
257      SUPPORT_CONTROLS = new AtomicBoolean(true);
258    }
259  }
260
261
262
263  /**
264   * The maximum allowed file size that we will attempt to read from a file
265   * referenced by a URL.  A default size limit of 10 megabytes will be imposed
266   * unless overridden by the
267   * {@code com.unboundid.ldif.LDIFReader.maxURLFileSizeBytes} system property.
268   */
269  private static final long MAX_URL_FILE_SIZE;
270  static
271  {
272    // Use a default size of 10 megabytes, but allow it to be o
273    long maxURLFileSize = 10L * 1024L * 1024L;
274
275    final String propertyName =
276         LDIFReader.class.getName() + ".maxURLFileSizeBytes";
277    final String propertyValue = StaticUtils.getSystemProperty(propertyName);
278    if (propertyValue != null)
279    {
280      try
281      {
282        maxURLFileSize = Long.parseLong(propertyValue);
283      }
284      catch (final Exception e)
285      {
286        Debug.debugException(e);
287      }
288    }
289
290    MAX_URL_FILE_SIZE = maxURLFileSize;
291  }
292
293
294
295  // The buffered reader that will be used to read LDIF data.
296  @NotNull private final BufferedReader reader;
297
298  // The behavior that should be exhibited when encountering duplicate attribute
299  // values.
300  @NotNull private volatile DuplicateValueBehavior duplicateValueBehavior;
301
302  // A line number counter.
303  private long lineNumberCounter = 0;
304
305  // The change record translator to use, if any.
306  @Nullable private final LDIFReaderChangeRecordTranslator
307       changeRecordTranslator;
308
309  // The entry translator to use, if any.
310  @Nullable private final LDIFReaderEntryTranslator entryTranslator;
311
312  // The schema that will be used when processing, if applicable.
313  @Nullable private Schema schema;
314
315  // Specifies the base path that will be prepended to relative paths for file
316  // URLs.
317  @NotNull private volatile String relativeBasePath;
318
319  // The behavior that should be exhibited with regard to illegal trailing
320  // spaces in attribute values.
321  @NotNull private volatile TrailingSpaceBehavior trailingSpaceBehavior;
322
323  // True iff we are processing asynchronously.
324  private final boolean isAsync;
325
326  //
327  // The following only apply to asynchronous processing.
328  //
329
330  // Parses entries asynchronously.
331  @Nullable private final
332       AsynchronousParallelProcessor<UnparsedLDIFRecord,LDIFRecord> asyncParser;
333
334  // Set to true when the end of the input is reached.
335  @Nullable private final AtomicBoolean asyncParsingComplete;
336
337  // The records that have been read and parsed.
338  @Nullable private final BlockingQueue<Result<UnparsedLDIFRecord,LDIFRecord>>
339       asyncParsedRecords;
340
341
342
343  /**
344   * Creates a new LDIF reader that will read data from the specified file.
345   *
346   * @param  path  The path to the file from which the data is to be read.  It
347   *               must not be {@code null}.
348   *
349   * @throws  IOException  If a problem occurs while opening the file for
350   *                       reading.
351   */
352  public LDIFReader(@NotNull final String path)
353         throws IOException
354  {
355    this(new FileInputStream(path));
356  }
357
358
359
360  /**
361   * Creates a new LDIF reader that will read data from the specified file
362   * and parses the LDIF records asynchronously using the specified number of
363   * threads.
364   *
365   * @param  path  The path to the file from which the data is to be read.  It
366   *               must not be {@code null}.
367   * @param  numParseThreads  If this value is greater than zero, then the
368   *                          specified number of threads will be used to
369   *                          asynchronously read and parse the LDIF file.
370   *
371   * @throws  IOException  If a problem occurs while opening the file for
372   *                       reading.
373   *
374   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
375   *      constructor for more details about asynchronous processing.
376   */
377  public LDIFReader(@NotNull final String path, final int numParseThreads)
378         throws IOException
379  {
380    this(new FileInputStream(path), numParseThreads);
381  }
382
383
384
385  /**
386   * Creates a new LDIF reader that will read data from the specified file.
387   *
388   * @param  file  The file from which the data is to be read.  It must not be
389   *               {@code null}.
390   *
391   * @throws  IOException  If a problem occurs while opening the file for
392   *                       reading.
393   */
394  public LDIFReader(@NotNull final File file)
395         throws IOException
396  {
397    this(new FileInputStream(file));
398  }
399
400
401
402  /**
403   * Creates a new LDIF reader that will read data from the specified file
404   * and optionally parses the LDIF records asynchronously using the specified
405   * number of threads.
406   *
407   * @param  file             The file from which the data is to be read.  It
408   *                          must not be {@code null}.
409   * @param  numParseThreads  If this value is greater than zero, then the
410   *                          specified number of threads will be used to
411   *                          asynchronously read and parse the LDIF file.
412   *
413   * @throws  IOException  If a problem occurs while opening the file for
414   *                       reading.
415   */
416  public LDIFReader(@NotNull final File file, final int numParseThreads)
417         throws IOException
418  {
419    this(new FileInputStream(file), numParseThreads);
420  }
421
422
423
424  /**
425   * Creates a new LDIF reader that will read data from the specified files in
426   * the order in which they are provided and optionally parses the LDIF records
427   * asynchronously using the specified number of threads.
428   *
429   * @param  files            The files from which the data is to be read.  It
430   *                          must not be {@code null} or empty.
431   * @param  numParseThreads  If this value is greater than zero, then the
432   *                          specified number of threads will be used to
433   *                          asynchronously read and parse the LDIF file.
434   * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
435   *                          before they are returned.  This is normally
436   *                          {@code null}, which causes entries to be returned
437   *                          unaltered. This is particularly useful when
438   *                          parsing the input file in parallel because the
439   *                          entry translation is also done in parallel.
440   *
441   * @throws  IOException  If a problem occurs while opening the file for
442   *                       reading.
443   */
444  public LDIFReader(@NotNull final File[] files, final int numParseThreads,
445                    @Nullable final LDIFReaderEntryTranslator entryTranslator)
446         throws IOException
447  {
448    this(files, numParseThreads, entryTranslator, null);
449  }
450
451
452
453  /**
454   * Creates a new LDIF reader that will read data from the specified files in
455   * the order in which they are provided and optionally parses the LDIF records
456   * asynchronously using the specified number of threads.
457   *
458   * @param  files                   The files from which the data is to be
459   *                                 read.  It must not be {@code null} or
460   *                                 empty.
461   * @param  numParseThreads         If this value is greater than zero, then
462   *                                 the specified number of threads will be
463   *                                 used to asynchronously read and parse the
464   *                                 LDIF file.
465   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
466   *                                 entries before they are returned.  This is
467   *                                 normally {@code null}, which causes entries
468   *                                 to be returned unaltered.  This is
469   *                                 particularly useful when parsing the input
470   *                                 file in parallel because the entry
471   *                                 translation is also done in parallel.
472   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
473   *                                 apply to change records before they are
474   *                                 returned.  This is normally {@code null},
475   *                                 which causes change records to be returned
476   *                                 unaltered.  This is particularly useful
477   *                                 when parsing the input file in parallel
478   *                                 because the change record translation is
479   *                                 also done in parallel.
480   *
481   * @throws  IOException  If a problem occurs while opening the file for
482   *                       reading.
483   */
484  public LDIFReader(@NotNull final File[] files, final int numParseThreads,
485       @Nullable final LDIFReaderEntryTranslator entryTranslator,
486       @Nullable final LDIFReaderChangeRecordTranslator changeRecordTranslator)
487       throws IOException
488  {
489    this(files, numParseThreads, entryTranslator, changeRecordTranslator,
490         "UTF-8");
491  }
492
493
494
495  /**
496   * Creates a new LDIF reader that will read data from the specified files in
497   * the order in which they are provided and optionally parses the LDIF records
498   * asynchronously using the specified number of threads.
499   *
500   * @param  files                   The files from which the data is to be
501   *                                 read.  It must not be {@code null} or
502   *                                 empty.
503   * @param  numParseThreads         If this value is greater than zero, then
504   *                                 the specified number of threads will be
505   *                                 used to asynchronously read and parse the
506   *                                 LDIF file.
507   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
508   *                                 entries before they are returned.  This is
509   *                                 normally {@code null}, which causes entries
510   *                                 to be returned unaltered.  This is
511   *                                 particularly useful when parsing the input
512   *                                 file in parallel because the entry
513   *                                 translation is also done in parallel.
514   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
515   *                                 apply to change records before they are
516   *                                 returned.  This is normally {@code null},
517   *                                 which causes change records to be returned
518   *                                 unaltered.  This is particularly useful
519   *                                 when parsing the input file in parallel
520   *                                 because the change record translation is
521   *                                 also done in parallel.
522   * @param  characterSet            The character set to use when reading from
523   *                                 the input stream.  It must not be
524   *                                 {@code null}.
525   *
526   * @throws  IOException  If a problem occurs while opening the file for
527   *                       reading.
528   */
529  public LDIFReader(@NotNull final File[] files, final int numParseThreads,
530       @Nullable final LDIFReaderEntryTranslator entryTranslator,
531       @Nullable final LDIFReaderChangeRecordTranslator changeRecordTranslator,
532       @NotNull final String characterSet)
533       throws IOException
534  {
535    this(createAggregateInputStream(files), numParseThreads, entryTranslator,
536         changeRecordTranslator, characterSet);
537  }
538
539
540
541  /**
542   * Creates a new aggregate input stream that will read data from the specified
543   * files.  If there are multiple files, then a "padding" file will be inserted
544   * between them to ensure that there is at least one blank line between the
545   * end of one file and the beginning of another.
546   *
547   * @param  files  The files from which the data is to be read.  It must not be
548   *                {@code null} or empty.
549   *
550   * @return  The input stream to use to read data from the provided files.
551   *
552   * @throws  IOException  If a problem is encountered while attempting to
553   *                       create the input stream.
554   */
555  @NotNull()
556  private static InputStream createAggregateInputStream(
557                                  @NotNull final File... files)
558          throws IOException
559  {
560    if (files.length == 0)
561    {
562      throw new IOException(ERR_READ_NO_LDIF_FILES.get());
563    }
564    else
565    {
566      return new AggregateInputStream(true, files);
567    }
568  }
569
570
571
572  /**
573   * Creates a new LDIF reader that will read data from the provided input
574   * stream.
575   *
576   * @param  inputStream  The input stream from which the data is to be read.
577   *                      It must not be {@code null}.
578   */
579  public LDIFReader(@NotNull final InputStream inputStream)
580  {
581    this(inputStream, 0);
582  }
583
584
585
586  /**
587   * Creates a new LDIF reader that will read data from the specified stream
588   * and parses the LDIF records asynchronously using the specified number of
589   * threads.
590   *
591   * @param  inputStream  The input stream from which the data is to be read.
592   *                      It must not be {@code null}.
593   * @param  numParseThreads  If this value is greater than zero, then the
594   *                          specified number of threads will be used to
595   *                          asynchronously read and parse the LDIF file.
596   *
597   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
598   *      constructor for more details about asynchronous processing.
599   */
600  public LDIFReader(@NotNull final InputStream inputStream,
601                    final int numParseThreads)
602  {
603    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
604    this(new BufferedReader(
605         new InputStreamReader(inputStream, StandardCharsets.UTF_8),
606              DEFAULT_BUFFER_SIZE),
607         numParseThreads);
608  }
609
610
611
612  /**
613   * Creates a new LDIF reader that will read data from the specified stream
614   * and parses the LDIF records asynchronously using the specified number of
615   * threads.
616   *
617   * @param  inputStream  The input stream from which the data is to be read.
618   *                      It must not be {@code null}.
619   * @param  numParseThreads  If this value is greater than zero, then the
620   *                          specified number of threads will be used to
621   *                          asynchronously read and parse the LDIF file.
622   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
623   *                         entries before they are returned.  This is normally
624   *                         {@code null}, which causes entries to be returned
625   *                         unaltered. This is particularly useful when parsing
626   *                         the input file in parallel because the entry
627   *                         translation is also done in parallel.
628   *
629   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
630   *      constructor for more details about asynchronous processing.
631   */
632  public LDIFReader(@NotNull final InputStream inputStream,
633                    final int numParseThreads,
634                    @Nullable final LDIFReaderEntryTranslator entryTranslator)
635  {
636    this(inputStream, numParseThreads, entryTranslator, null);
637  }
638
639
640
641  /**
642   * Creates a new LDIF reader that will read data from the specified stream
643   * and parses the LDIF records asynchronously using the specified number of
644   * threads.
645   *
646   * @param  inputStream             The input stream from which the data is to
647   *                                 be read.  It must not be {@code null}.
648   * @param  numParseThreads         If this value is greater than zero, then
649   *                                 the specified number of threads will be
650   *                                 used to asynchronously read and parse the
651   *                                 LDIF file.
652   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
653   *                                 entries before they are returned.  This is
654   *                                 normally {@code null}, which causes entries
655   *                                 to be returned unaltered.  This is
656   *                                 particularly useful when parsing the input
657   *                                 file in parallel because the entry
658   *                                 translation is also done in parallel.
659   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
660   *                                 apply to change records before they are
661   *                                 returned.  This is normally {@code null},
662   *                                 which causes change records to be returned
663   *                                 unaltered.  This is particularly useful
664   *                                 when parsing the input file in parallel
665   *                                 because the change record translation is
666   *                                 also done in parallel.
667   *
668   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
669   *      constructor for more details about asynchronous processing.
670   */
671  public LDIFReader(@NotNull final InputStream inputStream,
672       final int numParseThreads,
673       @Nullable final LDIFReaderEntryTranslator entryTranslator,
674       @Nullable final LDIFReaderChangeRecordTranslator changeRecordTranslator)
675  {
676    // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
677    this(inputStream, numParseThreads, entryTranslator, changeRecordTranslator,
678         "UTF-8");
679  }
680
681
682
683  /**
684   * Creates a new LDIF reader that will read data from the specified stream
685   * and parses the LDIF records asynchronously using the specified number of
686   * threads.
687   *
688   * @param  inputStream             The input stream from which the data is to
689   *                                 be read.  It must not be {@code null}.
690   * @param  numParseThreads         If this value is greater than zero, then
691   *                                 the specified number of threads will be
692   *                                 used to asynchronously read and parse the
693   *                                 LDIF file.
694   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
695   *                                 entries before they are returned.  This is
696   *                                 normally {@code null}, which causes entries
697   *                                 to be returned unaltered.  This is
698   *                                 particularly useful when parsing the input
699   *                                 file in parallel because the entry
700   *                                 translation is also done in parallel.
701   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
702   *                                 apply to change records before they are
703   *                                 returned.  This is normally {@code null},
704   *                                 which causes change records to be returned
705   *                                 unaltered.  This is particularly useful
706   *                                 when parsing the input file in parallel
707   *                                 because the change record translation is
708   *                                 also done in parallel.
709   * @param  characterSet            The character set to use when reading from
710   *                                 the input stream.  It must not be
711   *                                 {@code null}.
712   *
713   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
714   *      constructor for more details about asynchronous processing.
715   */
716  public LDIFReader(@NotNull final InputStream inputStream,
717       final int numParseThreads,
718       @Nullable final LDIFReaderEntryTranslator entryTranslator,
719       @Nullable final LDIFReaderChangeRecordTranslator changeRecordTranslator,
720       @NotNull final String characterSet)
721  {
722    this(new BufferedReader(
723              new InputStreamReader(inputStream, Charset.forName(characterSet)),
724              DEFAULT_BUFFER_SIZE),
725         numParseThreads, entryTranslator, changeRecordTranslator);
726  }
727
728
729
730  /**
731   * Creates a new LDIF reader that will use the provided buffered reader to
732   * read the LDIF data.  The encoding of the underlying Reader must be set to
733   * "UTF-8" as required by RFC 2849.
734   *
735   * @param  reader  The buffered reader that will be used to read the LDIF
736   *                 data.  It must not be {@code null}.
737   */
738  public LDIFReader(@NotNull final BufferedReader reader)
739  {
740    this(reader, 0);
741  }
742
743
744
745  /**
746   * Creates a new LDIF reader that will read data from the specified buffered
747   * reader and parses the LDIF records asynchronously using the specified
748   * number of threads.  The encoding of the underlying Reader must be set to
749   * "UTF-8" as required by RFC 2849.
750   *
751   * @param reader The buffered reader that will be used to read the LDIF data.
752   *               It must not be {@code null}.
753   * @param  numParseThreads  If this value is greater than zero, then the
754   *                          specified number of threads will be used to
755   *                          asynchronously read and parse the LDIF file.
756   *
757   * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
758   *      constructor for more details about asynchronous processing.
759   */
760  public LDIFReader(@NotNull final BufferedReader reader,
761                    final int numParseThreads)
762  {
763    this(reader, numParseThreads, null);
764  }
765
766
767
768  /**
769   * Creates a new LDIF reader that will read data from the specified buffered
770   * reader and parses the LDIF records asynchronously using the specified
771   * number of threads.  The encoding of the underlying Reader must be set to
772   * "UTF-8" as required by RFC 2849.
773   *
774   * @param reader The buffered reader that will be used to read the LDIF data.
775   *               It must not be {@code null}.
776   * @param  numParseThreads  If this value is greater than zero, then the
777   *                          specified number of threads will be used to
778   *                          asynchronously read and parse the LDIF file.
779   *                          This should only be set to greater than zero when
780   *                          performance analysis has demonstrated that reading
781   *                          and parsing the LDIF is a bottleneck.  The default
782   *                          synchronous processing is normally fast enough.
783   *                          There is little benefit in passing in a value
784   *                          greater than four (unless there is an
785   *                          LDIFReaderEntryTranslator that does time-consuming
786   *                          processing).  A value of zero implies the
787   *                          default behavior of reading and parsing LDIF
788   *                          records synchronously when one of the read
789   *                          methods is called.
790   * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
791   *                         entries before they are returned.  This is normally
792   *                         {@code null}, which causes entries to be returned
793   *                         unaltered. This is particularly useful when parsing
794   *                         the input file in parallel because the entry
795   *                         translation is also done in parallel.
796   */
797  public LDIFReader(@NotNull final BufferedReader reader,
798                    final int numParseThreads,
799                    @Nullable final LDIFReaderEntryTranslator entryTranslator)
800  {
801    this(reader, numParseThreads, entryTranslator, null);
802  }
803
804
805
806  /**
807   * Creates a new LDIF reader that will read data from the specified buffered
808   * reader and parses the LDIF records asynchronously using the specified
809   * number of threads.  The encoding of the underlying Reader must be set to
810   * "UTF-8" as required by RFC 2849.
811   *
812   * @param reader                   The buffered reader that will be used to
813   *                                 read the LDIF data.  It must not be
814   *                                 {@code null}.
815   * @param  numParseThreads         If this value is greater than zero, then
816   *                                 the specified number of threads will be
817   *                                 used to asynchronously read and parse the
818   *                                 LDIF file.
819   * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
820   *                                 entries before they are returned.  This is
821   *                                 normally {@code null}, which causes entries
822   *                                 to be returned unaltered.  This is
823   *                                 particularly useful when parsing the input
824   *                                 file in parallel because the entry
825   *                                 translation is also done in parallel.
826   * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
827   *                                 apply to change records before they are
828   *                                 returned.  This is normally {@code null},
829   *                                 which causes change records to be returned
830   *                                 unaltered.  This is particularly useful
831   *                                 when parsing the input file in parallel
832   *                                 because the change record translation is
833   *                                 also done in parallel.
834   */
835  public LDIFReader(@NotNull final BufferedReader reader,
836       final int numParseThreads,
837       @Nullable final LDIFReaderEntryTranslator entryTranslator,
838       @Nullable final LDIFReaderChangeRecordTranslator changeRecordTranslator)
839  {
840    Validator.ensureNotNull(reader);
841    Validator.ensureTrue(numParseThreads >= 0,
842               "LDIFReader.numParseThreads must not be negative.");
843
844    this.reader = reader;
845    this.entryTranslator = entryTranslator;
846    this.changeRecordTranslator = changeRecordTranslator;
847
848    duplicateValueBehavior = DuplicateValueBehavior.STRIP;
849    trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
850
851    relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
852
853    if (numParseThreads == 0)
854    {
855      isAsync = false;
856      asyncParser = null;
857      asyncParsingComplete = null;
858      asyncParsedRecords = null;
859    }
860    else
861    {
862      isAsync = true;
863      asyncParsingComplete = new AtomicBoolean(false);
864
865      // Decodes entries in parallel.
866      final LDAPSDKThreadFactory threadFactory =
867           new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
868      final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
869           new ParallelProcessor<>(
870                new RecordParser(), threadFactory, numParseThreads,
871                ASYNC_MIN_PER_PARSING_THREAD);
872
873      final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
874           ArrayBlockingQueue<>(ASYNC_QUEUE_SIZE);
875
876      // The output queue must be a little more than twice as big as the input
877      // queue to more easily handle being shutdown in the middle of processing
878      // when the queues are full and threads are blocked.
879      asyncParsedRecords = new ArrayBlockingQueue<>(2 * ASYNC_QUEUE_SIZE + 100);
880
881      asyncParser = new AsynchronousParallelProcessor<>(pendingQueue,
882           parallelParser, asyncParsedRecords);
883
884      final LineReaderThread lineReaderThread = new LineReaderThread();
885      lineReaderThread.start();
886    }
887  }
888
889
890
891  /**
892   * Reads entries from the LDIF file with the specified path and returns them
893   * as a {@code List}.  This is a convenience method that should only be used
894   * for data sets that are small enough so that running out of memory isn't a
895   * concern.
896   *
897   * @param  path  The path to the LDIF file containing the entries to be read.
898   *
899   * @return  A list of the entries read from the given LDIF file.
900   *
901   * @throws  IOException  If a problem occurs while attempting to read data
902   *                       from the specified file.
903   *
904   * @throws  LDIFException  If a problem is encountered while attempting to
905   *                         decode data read as LDIF.
906   */
907  @NotNull()
908  public static List<Entry> readEntries(@NotNull final String path)
909         throws IOException, LDIFException
910  {
911    return readEntries(new LDIFReader(path));
912  }
913
914
915
916  /**
917   * Reads entries from the specified LDIF file and returns them as a
918   * {@code List}.  This is a convenience method that should only be used for
919   * data sets that are small enough so that running out of memory isn't a
920   * concern.
921   *
922   * @param  file  A reference to the LDIF file containing the entries to be
923   *               read.
924   *
925   * @return  A list of the entries read from the given LDIF file.
926   *
927   * @throws  IOException  If a problem occurs while attempting to read data
928   *                       from the specified file.
929   *
930   * @throws  LDIFException  If a problem is encountered while attempting to
931   *                         decode data read as LDIF.
932   */
933  @NotNull()
934  public static List<Entry> readEntries(@NotNull final File file)
935         throws IOException, LDIFException
936  {
937    return readEntries(new LDIFReader(file));
938  }
939
940
941
942  /**
943   * Reads and decodes LDIF entries from the provided input stream and
944   * returns them as a {@code List}.  This is a convenience method that should
945   * only be used for data sets that are small enough so that running out of
946   * memory isn't a concern.
947   *
948   * @param  inputStream  The input stream from which the entries should be
949   *                      read.  The input stream will be closed before
950   *                      returning.
951   *
952   * @return  A list of the entries read from the given input stream.
953   *
954   * @throws  IOException  If a problem occurs while attempting to read data
955   *                       from the input stream.
956   *
957   * @throws  LDIFException  If a problem is encountered while attempting to
958   *                         decode data read as LDIF.
959   */
960  @NotNull()
961  public static List<Entry> readEntries(@NotNull final InputStream inputStream)
962         throws IOException, LDIFException
963  {
964    return readEntries(new LDIFReader(inputStream));
965  }
966
967
968
969  /**
970   * Reads entries from the provided LDIF reader and returns them as a list.
971   *
972   * @param  reader  The reader from which the entries should be read.  It will
973   *                 be closed before returning.
974   *
975   * @return  A list of the entries read from the provided reader.
976   *
977   * @throws  IOException  If a problem was encountered while attempting to read
978   *                       data from the LDIF data source.
979   *
980   * @throws  LDIFException  If a problem is encountered while attempting to
981   *                         decode data read as LDIF.
982   */
983  @NotNull()
984  private static List<Entry> readEntries(@NotNull final LDIFReader reader)
985          throws IOException, LDIFException
986  {
987    try
988    {
989      final ArrayList<Entry> entries = new ArrayList<>(10);
990      while (true)
991      {
992        final Entry e = reader.readEntry();
993        if (e == null)
994        {
995          break;
996        }
997
998        entries.add(e);
999      }
1000
1001      return entries;
1002    }
1003    finally
1004    {
1005      reader.close();
1006    }
1007  }
1008
1009
1010
1011  /**
1012   * Closes this LDIF reader and the underlying LDIF source.
1013   *
1014   * @throws  IOException  If a problem occurs while closing the underlying LDIF
1015   *                       source.
1016   */
1017  @Override()
1018  public void close()
1019         throws IOException
1020  {
1021    reader.close();
1022
1023    if (isAsync())
1024    {
1025      // Closing the reader will trigger the LineReaderThread to complete, but
1026      // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
1027      // this, we clear out the completed output queue, which is larger than
1028      // the input queue, so the LineReaderThread will stop reading and
1029      // shutdown the asyncParser.
1030      asyncParsedRecords.clear();
1031    }
1032  }
1033
1034
1035
1036  /**
1037   * Indicates whether to ignore any duplicate values encountered while reading
1038   * LDIF records.
1039   *
1040   * @return  {@code true} if duplicate values should be ignored, or
1041   *          {@code false} if any LDIF records containing duplicate values
1042   *          should be rejected.
1043   *
1044   * @deprecated  Use the {@link #getDuplicateValueBehavior} method instead.
1045   */
1046  @Deprecated()
1047  public boolean ignoreDuplicateValues()
1048  {
1049    return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
1050  }
1051
1052
1053
1054  /**
1055   * Specifies whether to ignore any duplicate values encountered while reading
1056   * LDIF records.
1057   *
1058   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1059   *                                attribute values encountered while reading
1060   *                                LDIF records.
1061   *
1062   * @deprecated  Use the {@link #setDuplicateValueBehavior} method instead.
1063   */
1064  @Deprecated()
1065  public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
1066  {
1067    if (ignoreDuplicateValues)
1068    {
1069      duplicateValueBehavior = DuplicateValueBehavior.STRIP;
1070    }
1071    else
1072    {
1073      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
1074    }
1075  }
1076
1077
1078
1079  /**
1080   * Retrieves the behavior that should be exhibited if the LDIF reader
1081   * encounters an entry with duplicate values.
1082   *
1083   * @return  The behavior that should be exhibited if the LDIF reader
1084   *          encounters an entry with duplicate values.
1085   */
1086  @NotNull()
1087  public DuplicateValueBehavior getDuplicateValueBehavior()
1088  {
1089    return duplicateValueBehavior;
1090  }
1091
1092
1093
1094  /**
1095   * Specifies the behavior that should be exhibited if the LDIF reader
1096   * encounters an entry with duplicate values.
1097   *
1098   * @param  duplicateValueBehavior  The behavior that should be exhibited if
1099   *                                 the LDIF reader encounters an entry with
1100   *                                 duplicate values.
1101   */
1102  public void setDuplicateValueBehavior(
1103                   @NotNull final DuplicateValueBehavior duplicateValueBehavior)
1104  {
1105    this.duplicateValueBehavior = duplicateValueBehavior;
1106  }
1107
1108
1109
1110  /**
1111   * Indicates whether to strip off any illegal trailing spaces that may appear
1112   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1113   * specification strongly recommends that any value which legitimately
1114   * contains trailing spaces be base64-encoded, and any spaces which appear
1115   * after the end of non-base64-encoded values may therefore be considered
1116   * invalid.  If any such trailing spaces are encountered in an LDIF record and
1117   * they are not to be stripped, then an {@link LDIFException} will be thrown
1118   * for that record.
1119   * <BR><BR>
1120   * Note that this applies only to spaces after the end of a value, and not to
1121   * spaces which may appear at the end of a line for a value that is wrapped
1122   * and continued on the next line.
1123   *
1124   * @return  {@code true} if illegal trailing spaces should be stripped off, or
1125   *          {@code false} if LDIF records containing illegal trailing spaces
1126   *          should be rejected.
1127   *
1128   * @deprecated  Use the {@link #getTrailingSpaceBehavior} method instead.
1129   */
1130  @Deprecated()
1131  public boolean stripTrailingSpaces()
1132  {
1133    return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
1134  }
1135
1136
1137
1138  /**
1139   * Specifies whether to strip off any illegal trailing spaces that may appear
1140   * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1141   * specification strongly recommends that any value which legitimately
1142   * contains trailing spaces be base64-encoded, and any spaces which appear
1143   * after the end of non-base64-encoded values may therefore be considered
1144   * invalid.  If any such trailing spaces are encountered in an LDIF record and
1145   * they are not to be stripped, then an {@link LDIFException} will be thrown
1146   * for that record.
1147   * <BR><BR>
1148   * Note that this applies only to spaces after the end of a value, and not to
1149   * spaces which may appear at the end of a line for a value that is wrapped
1150   * and continued on the next line.
1151   *
1152   * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
1153   *                              trailing spaces, or {@code false} if LDIF
1154   *                              records containing them should be rejected.
1155   *
1156   * @deprecated  Use the {@link #setTrailingSpaceBehavior} method instead.
1157   */
1158  @Deprecated()
1159  public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
1160  {
1161    trailingSpaceBehavior = stripTrailingSpaces
1162         ? TrailingSpaceBehavior.STRIP
1163         : TrailingSpaceBehavior.REJECT;
1164  }
1165
1166
1167
1168  /**
1169   * Retrieves the behavior that should be exhibited when encountering attribute
1170   * values which are not base64-encoded but contain trailing spaces.  The LDIF
1171   * specification strongly recommends that any value which legitimately
1172   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1173   * may be configured to automatically strip these spaces, to preserve them, or
1174   * to reject any entry or change record containing them.
1175   *
1176   * @return  The behavior that should be exhibited when encountering attribute
1177   *          values which are not base64-encoded but contain trailing spaces.
1178   */
1179  @NotNull()
1180  public TrailingSpaceBehavior getTrailingSpaceBehavior()
1181  {
1182    return trailingSpaceBehavior;
1183  }
1184
1185
1186
1187  /**
1188   * Specifies the behavior that should be exhibited when encountering attribute
1189   * values which are not base64-encoded but contain trailing spaces.  The LDIF
1190   * specification strongly recommends that any value which legitimately
1191   * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1192   * may be configured to automatically strip these spaces, to preserve them, or
1193   * to reject any entry or change record containing them.
1194   *
1195   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1196   *                                encountering attribute values which are not
1197   *                                base64-encoded but contain trailing spaces.
1198   */
1199  public void setTrailingSpaceBehavior(
1200                   @NotNull final TrailingSpaceBehavior trailingSpaceBehavior)
1201  {
1202    this.trailingSpaceBehavior = trailingSpaceBehavior;
1203  }
1204
1205
1206
1207  /**
1208   * Indicates whether the LDIF reader will attempt to handle LDAP controls
1209   * contained in LDIF records.  See the documentation for the
1210   * {@link #setSupportControls} method for a more complete explanation.
1211   *
1212   * @return  {@code true} if the LDIF reader will attempt to handle controls
1213   *          contained in LDIF records, or {@code false} if it will not and
1214   *          they will be treated as regular attributes.
1215   */
1216  public static boolean supportControls()
1217  {
1218    return SUPPORT_CONTROLS.get();
1219  }
1220
1221
1222
1223  /**
1224   * Specifies whether the LDIF reader will attempt to handle LDAP controls
1225   * contained in LDIF records.  By default, the LDAP SDK will handle controls
1226   * in accordance with RFC 2849, so that if the first unwrapped line after the
1227   * DN starts with "control:", then that record will be assumed to be an LDIF
1228   * change record that contains one or more LDAP controls.  However, this can
1229   * potentially cause a problem in one corner case:  the case in which an LDIF
1230   * record represents an entry in which the first attribute in the entry is
1231   * named "control", and when you're either reading a generic LDIF record or
1232   * you're reading an LDIF change record with {@code defaultAdd} set to
1233   * {@code true}.  In such case, the LDIF reader would assume that the
1234   * "control" attribute is trying to specify an LDAP control, and it would
1235   * either throw an exception if it can't parse the value as a control, or it
1236   * would return an LDIF add change record without the "control" attribute but
1237   * with an LDAP control.  Calling this method with a value of {@code false}
1238   * will disable the LDIF reader's support for controls so that any record in
1239   * which the unwrapped line immediately following the DN starts with
1240   * "control:" will be treated as specifying an attribute named "control"
1241   * rather than an LDAP control.
1242   *
1243   * @param  supportControls  Specifies whether the LDIF reader will attempt to
1244   *                          handle LDAP controls contained in LDIF records.
1245   */
1246  public static void setSupportControls(final boolean supportControls)
1247  {
1248    SUPPORT_CONTROLS.set(supportControls);
1249  }
1250
1251
1252
1253  /**
1254   * Retrieves the base path that will be prepended to relative paths in order
1255   * to obtain an absolute path.  This will only be used for "file:" URLs that
1256   * have paths which do not begin with a slash.
1257   *
1258   * @return  The base path that will be prepended to relative paths in order to
1259   *          obtain an absolute path.
1260   */
1261  @NotNull()
1262  public String getRelativeBasePath()
1263  {
1264    return relativeBasePath;
1265  }
1266
1267
1268
1269  /**
1270   * Specifies the base path that will be prepended to relative paths in order
1271   * to obtain an absolute path.  This will only be used for "file:" URLs that
1272   * have paths which do not begin with a space.
1273   *
1274   * @param  relativeBasePath  The base path that will be prepended to relative
1275   *                           paths in order to obtain an absolute path.
1276   */
1277  public void setRelativeBasePath(@NotNull final String relativeBasePath)
1278  {
1279    setRelativeBasePath(new File(relativeBasePath));
1280  }
1281
1282
1283
1284  /**
1285   * Specifies the base path that will be prepended to relative paths in order
1286   * to obtain an absolute path.  This will only be used for "file:" URLs that
1287   * have paths which do not begin with a space.
1288   *
1289   * @param  relativeBasePath  The base path that will be prepended to relative
1290   *                           paths in order to obtain an absolute path.
1291   */
1292  public void setRelativeBasePath(@NotNull final File relativeBasePath)
1293  {
1294    final String path = relativeBasePath.getAbsolutePath();
1295    if (path.endsWith(File.separator))
1296    {
1297      this.relativeBasePath = path;
1298    }
1299    else
1300    {
1301      this.relativeBasePath = path + File.separator;
1302    }
1303  }
1304
1305
1306
1307  /**
1308   * Retrieves the schema that will be used when reading LDIF records, if
1309   * defined.
1310   *
1311   * @return  The schema that will be used when reading LDIF records, or
1312   *          {@code null} if no schema should be used and all attributes should
1313   *          be treated as case-insensitive strings.
1314   */
1315  @Nullable()
1316  public Schema getSchema()
1317  {
1318    return schema;
1319  }
1320
1321
1322
1323  /**
1324   * Specifies the schema that should be used when reading LDIF records.
1325   *
1326   * @param  schema  The schema that should be used when reading LDIF records,
1327   *                 or {@code null} if no schema should be used and all
1328   *                 attributes should be treated as case-insensitive strings.
1329   */
1330  public void setSchema(@Nullable final Schema schema)
1331  {
1332    this.schema = schema;
1333  }
1334
1335
1336
1337  /**
1338   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1339   * change record.
1340   *
1341   * @return  The record read from the LDIF source, or {@code null} if there are
1342   *          no more entries to be read.
1343   *
1344   * @throws  IOException  If a problem occurs while trying to read from the
1345   *                       LDIF source.
1346   *
1347   * @throws  LDIFException  If the data read could not be parsed as an entry or
1348   *                         an LDIF change record.
1349   */
1350  @Nullable()
1351  public LDIFRecord readLDIFRecord()
1352         throws IOException, LDIFException
1353  {
1354    if (isAsync())
1355    {
1356      return readLDIFRecordAsync();
1357    }
1358    else
1359    {
1360      return readLDIFRecordInternal();
1361    }
1362  }
1363
1364
1365
1366  /**
1367   * Decodes the provided set of lines as an LDIF record.  It may be either an
1368   * entry or an LDIF change record.
1369   *
1370   * @param  ldifLines  The set of lines that comprise the LDIF representation
1371   *                    of the entry or change record.  It must not be
1372   *                    {@code null} or empty.
1373   *
1374   * @return  The decoded LDIF entry or change record.
1375   *
1376   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1377   *                         LDIF record.
1378   */
1379  @NotNull()
1380  public static LDIFRecord decodeLDIFRecord(@NotNull final String... ldifLines)
1381         throws LDIFException
1382  {
1383    return decodeLDIFRecord(DuplicateValueBehavior.STRIP,
1384         TrailingSpaceBehavior.REJECT, null, ldifLines);
1385  }
1386
1387
1388
1389  /**
1390   * Decodes the provided set of lines as an LDIF record.  It may be either an
1391   * entry or an LDIF change record.
1392   *
1393   * @param  duplicateValueBehavior  The behavior that should be exhibited when
1394   *                                 encountering LDIF records that have
1395   *                                 duplicate attribute values.  It must not be
1396   *                                 {@code null}.
1397   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
1398   *                                 encountering attribute values which are not
1399   *                                 base64-encoded but contain trailing spaces.
1400   *                                 It must not be {@code null}.
1401   * @param  schema                  The schema to use when processing the
1402   *                                 change record, or {@code null} if no schema
1403   *                                 should be used and all values should be
1404   *                                 treated as case-insensitive strings.
1405   * @param  ldifLines               The set of lines that comprise the LDIF
1406   *                                 representation of the entry or change
1407   *                                 record.  It must not be {@code null} or
1408   *                                 empty.
1409   *
1410   * @return  The decoded LDIF entry or change record.
1411   *
1412   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1413   *                         LDIF record.
1414   */
1415  @NotNull()
1416  public static LDIFRecord decodeLDIFRecord(
1417              @NotNull final DuplicateValueBehavior duplicateValueBehavior,
1418              @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
1419              @Nullable final Schema schema,
1420              @NotNull final String... ldifLines)
1421         throws LDIFException
1422  {
1423    final UnparsedLDIFRecord unparsedRecord = prepareRecord(
1424         duplicateValueBehavior, trailingSpaceBehavior, schema, ldifLines);
1425    return decodeRecord(unparsedRecord, DEFAULT_RELATIVE_BASE_PATH, schema);
1426  }
1427
1428
1429
1430  /**
1431   * Reads an entry from the LDIF source.
1432   *
1433   * @return  The entry read from the LDIF source, or {@code null} if there are
1434   *          no more entries to be read.
1435   *
1436   * @throws  IOException  If a problem occurs while attempting to read from the
1437   *                       LDIF source.
1438   *
1439   * @throws  LDIFException  If the data read could not be parsed as an entry.
1440   */
1441  @Nullable()
1442  public Entry readEntry()
1443         throws IOException, LDIFException
1444  {
1445    if (isAsync())
1446    {
1447      return readEntryAsync();
1448    }
1449    else
1450    {
1451      return readEntryInternal();
1452    }
1453  }
1454
1455
1456
1457  /**
1458   * Reads an LDIF change record from the LDIF source.  The LDIF record must
1459   * have a changetype.
1460   *
1461   * @return  The change record read from the LDIF source, or {@code null} if
1462   *          there are no more records to be read.
1463   *
1464   * @throws  IOException  If a problem occurs while attempting to read from the
1465   *                       LDIF source.
1466   *
1467   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1468   *                         change record.
1469   */
1470  @Nullable()
1471  public LDIFChangeRecord readChangeRecord()
1472         throws IOException, LDIFException
1473  {
1474    return readChangeRecord(false);
1475  }
1476
1477
1478
1479  /**
1480   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1481   * record does not have a changetype, then it may be assumed to be an add
1482   * change record.
1483   *
1484   * @param  defaultAdd  Indicates whether an LDIF record not containing a
1485   *                     changetype should be retrieved as an add change record.
1486   *                     If this is {@code false} and the record read does not
1487   *                     include a changetype, then an {@link LDIFException}
1488   *                     will be thrown.
1489   *
1490   * @return  The change record read from the LDIF source, or {@code null} if
1491   *          there are no more records to be read.
1492   *
1493   * @throws  IOException  If a problem occurs while attempting to read from the
1494   *                       LDIF source.
1495   *
1496   * @throws  LDIFException  If the data read could not be parsed as an LDIF
1497   *                         change record.
1498   */
1499  @Nullable()
1500  public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1501         throws IOException, LDIFException
1502  {
1503    if (isAsync())
1504    {
1505      return readChangeRecordAsync(defaultAdd);
1506    }
1507    else
1508    {
1509      return readChangeRecordInternal(defaultAdd);
1510    }
1511  }
1512
1513
1514
1515  /**
1516   * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1517   * thread.
1518   *
1519   * @return  The next parsed record or {@code null} if there are no more
1520   *          records to read.
1521   *
1522   * @throws IOException  If IOException was thrown when reading or parsing
1523   *                      the record.
1524   *
1525   * @throws LDIFException If LDIFException was thrown parsing the record.
1526   */
1527  @Nullable()
1528  private LDIFRecord readLDIFRecordAsync()
1529          throws IOException, LDIFException
1530  {
1531    Result<UnparsedLDIFRecord, LDIFRecord> result;
1532    LDIFRecord record = null;
1533    while (record == null)
1534    {
1535      result = readLDIFRecordResultAsync();
1536      if (result == null)
1537      {
1538        return null;
1539      }
1540
1541      record = result.getOutput();
1542
1543      // This is a special value that means we should skip this Entry.  We have
1544      // to use something different than null because null means EOF.
1545      if (record == SKIP_ENTRY)
1546      {
1547        record = null;
1548      }
1549    }
1550    return record;
1551  }
1552
1553
1554
1555  /**
1556   * Reads an entry asynchronously from the LDIF source.
1557   *
1558   * @return The entry read from the LDIF source, or {@code null} if there are
1559   *         no more entries to be read.
1560   *
1561   * @throws IOException   If a problem occurs while attempting to read from the
1562   *                       LDIF source.
1563   * @throws LDIFException If the data read could not be parsed as an entry.
1564   */
1565  @Nullable()
1566  private Entry readEntryAsync()
1567          throws IOException, LDIFException
1568  {
1569    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1570    LDIFRecord record = null;
1571    while (record == null)
1572    {
1573      result = readLDIFRecordResultAsync();
1574      if (result == null)
1575      {
1576        return null;
1577      }
1578
1579      record = result.getOutput();
1580
1581      // This is a special value that means we should skip this Entry.  We have
1582      // to use something different than null because null means EOF.
1583      if (record == SKIP_ENTRY)
1584      {
1585        record = null;
1586      }
1587    }
1588
1589    if (record instanceof Entry)
1590    {
1591      return (Entry) record;
1592    }
1593    else if (record instanceof LDIFChangeRecord)
1594    {
1595      try
1596      {
1597        // Some LDIFChangeRecord can be converted to an Entry.  This is really
1598        // an edge case though.
1599        return ((LDIFChangeRecord)record).toEntry();
1600      }
1601      catch (final LDIFException e)
1602      {
1603        Debug.debugException(e);
1604        final long firstLineNumber = result.getInput().getFirstLineNumber();
1605        throw new LDIFException(e.getExceptionMessage(),
1606                                firstLineNumber, true, e);
1607      }
1608    }
1609
1610    throw new AssertionError("LDIFRecords must either be an Entry or an " +
1611                             "LDIFChangeRecord");
1612  }
1613
1614
1615
1616  /**
1617   * Reads an LDIF change record from the LDIF source asynchronously.
1618   * Optionally, if the LDIF record does not have a changetype, then it may be
1619   * assumed to be an add change record.
1620   *
1621   * @param defaultAdd Indicates whether an LDIF record not containing a
1622   *                   changetype should be retrieved as an add change record.
1623   *                   If this is {@code false} and the record read does not
1624   *                   include a changetype, then an {@link LDIFException} will
1625   *                   be thrown.
1626   *
1627   * @return The change record read from the LDIF source, or {@code null} if
1628   *         there are no more records to be read.
1629   *
1630   * @throws IOException   If a problem occurs while attempting to read from the
1631   *                       LDIF source.
1632   * @throws LDIFException If the data read could not be parsed as an LDIF
1633   *                       change record.
1634   */
1635  @Nullable()
1636  private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1637          throws IOException, LDIFException
1638  {
1639    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1640    LDIFRecord record = null;
1641    while (record == null)
1642    {
1643      result = readLDIFRecordResultAsync();
1644      if (result == null)
1645      {
1646        return null;
1647      }
1648
1649      record = result.getOutput();
1650
1651      // This is a special value that means we should skip this Entry.  We have
1652      // to use something different than null because null means EOF.
1653      if (record == SKIP_ENTRY)
1654      {
1655        record = null;
1656      }
1657    }
1658
1659    if (record instanceof LDIFChangeRecord)
1660    {
1661      return (LDIFChangeRecord) record;
1662    }
1663    else if (record instanceof Entry)
1664    {
1665      if (defaultAdd)
1666      {
1667        return new LDIFAddChangeRecord((Entry) record);
1668      }
1669      else
1670      {
1671        final long firstLineNumber = result.getInput().getFirstLineNumber();
1672        throw new LDIFException(
1673             ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1674             true);
1675      }
1676    }
1677
1678    throw new AssertionError("LDIFRecords must either be an Entry or an " +
1679                             "LDIFChangeRecord");
1680  }
1681
1682
1683
1684  /**
1685   * Reads the next LDIF record, which was read and parsed asynchronously by
1686   * separate threads.
1687   *
1688   * @return  The next LDIF record or {@code null} if there are no more records.
1689   *
1690   * @throws  IOException  If a problem occurs while attempting to read from the
1691   *                       LDIF source.
1692   *
1693   * @throws  LDIFException  If the data read could not be parsed as an entry.
1694   */
1695  @Nullable()
1696  private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1697          throws IOException, LDIFException
1698  {
1699    Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1700
1701    // If the asynchronous reading and parsing is complete, then we don't have
1702    // to block waiting for the next record to show up on the queue.  If there
1703    // isn't a record there, then return null (EOF) right away.
1704    if (asyncParsingComplete.get())
1705    {
1706      result = asyncParsedRecords.poll();
1707    }
1708    else
1709    {
1710      try
1711      {
1712        // We probably could just do a asyncParsedRecords.take() here, but
1713        // there are some edge case error scenarios where
1714        // asyncParsingComplete might be set without a special EOF sentinel
1715        // Result enqueued.  So to guard against this, we have a very cautious
1716        // polling interval of 1 second.  During normal processing, we never
1717        // have to wait for this to expire, when there is something to do
1718        // (like shutdown).
1719        while ((result == null) && (!asyncParsingComplete.get()))
1720        {
1721          result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1722        }
1723
1724        // There's a very small chance that we missed the value, so double-check
1725        if (result == null)
1726        {
1727          result = asyncParsedRecords.poll();
1728        }
1729      }
1730      catch (final InterruptedException e)
1731      {
1732        Debug.debugException(e);
1733        Thread.currentThread().interrupt();
1734        throw new IOException(e);
1735      }
1736    }
1737    if (result == null)
1738    {
1739      return null;
1740    }
1741
1742    rethrow(result.getFailureCause());
1743
1744    // Check if we reached the end of the input
1745    final UnparsedLDIFRecord unparsedRecord = result.getInput();
1746    if (unparsedRecord.isEOF())
1747    {
1748      // This might have been set already by the LineReaderThread, but
1749      // just in case it hasn't gotten to it yet, do so here.
1750      asyncParsingComplete.set(true);
1751
1752      // Enqueue this EOF result again for any other thread that might be
1753      // blocked in asyncParsedRecords.take() even though having multiple
1754      // threads call this method concurrently breaks the contract of this
1755      // class.
1756      try
1757      {
1758        asyncParsedRecords.put(result);
1759      }
1760      catch (final InterruptedException e)
1761      {
1762        // We shouldn't ever get interrupted because the put won't ever block.
1763        // Once we are done reading, this is the only item left in the queue,
1764        // so we should always be able to re-enqueue it.
1765        Debug.debugException(e);
1766        Thread.currentThread().interrupt();
1767      }
1768      return null;
1769    }
1770
1771    return result;
1772  }
1773
1774
1775
1776  /**
1777   * Indicates whether this LDIF reader was constructed to perform asynchronous
1778   * processing.
1779   *
1780   * @return  {@code true} if this LDIFReader was constructed to perform
1781   *          asynchronous processing, or {@code false} if not.
1782   */
1783  private boolean isAsync()
1784  {
1785    return isAsync;
1786  }
1787
1788
1789
1790  /**
1791   * If not {@code null}, rethrows the specified Throwable as either an
1792   * IOException or LDIFException.
1793   *
1794   * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1795   *           is thrown.
1796   *
1797   * @throws IOException   If t is an IOException or a checked Exception that
1798   *                       is not an LDIFException.
1799   * @throws LDIFException  If t is an LDIFException.
1800   */
1801  static void rethrow(@Nullable final Throwable t)
1802         throws IOException, LDIFException
1803  {
1804    if (t == null)
1805    {
1806      return;
1807    }
1808
1809    if (t instanceof IOException)
1810    {
1811      throw (IOException) t;
1812    }
1813    else if (t instanceof LDIFException)
1814    {
1815      throw (LDIFException) t;
1816    }
1817    else if (t instanceof RuntimeException)
1818    {
1819      throw (RuntimeException) t;
1820    }
1821    else if (t instanceof Error)
1822    {
1823      throw (Error) t;
1824    }
1825    else
1826    {
1827      throw new IOException(t);
1828    }
1829  }
1830
1831
1832
1833  /**
1834   * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1835   * change record.
1836   *
1837   * @return The record read from the LDIF source, or {@code null} if there are
1838   *         no more entries to be read.
1839   *
1840   * @throws IOException   If a problem occurs while trying to read from the
1841   *                       LDIF source.
1842   * @throws LDIFException If the data read could not be parsed as an entry or
1843   *                       an LDIF change record.
1844   */
1845  @Nullable()
1846  private LDIFRecord readLDIFRecordInternal()
1847       throws IOException, LDIFException
1848  {
1849    final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1850    return decodeRecord(unparsedRecord, relativeBasePath, schema);
1851  }
1852
1853
1854
1855  /**
1856   * Reads an entry from the LDIF source.
1857   *
1858   * @return The entry read from the LDIF source, or {@code null} if there are
1859   *         no more entries to be read.
1860   *
1861   * @throws IOException   If a problem occurs while attempting to read from the
1862   *                       LDIF source.
1863   * @throws LDIFException If the data read could not be parsed as an entry.
1864   */
1865  @Nullable()
1866  private Entry readEntryInternal()
1867       throws IOException, LDIFException
1868  {
1869    Entry e = null;
1870    while (e == null)
1871    {
1872      final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1873      if (unparsedRecord.isEOF())
1874      {
1875        return null;
1876      }
1877
1878      e = decodeEntry(unparsedRecord, relativeBasePath);
1879      Debug.debugLDIFRead(e);
1880
1881      if (entryTranslator != null)
1882      {
1883        e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1884      }
1885    }
1886    return e;
1887  }
1888
1889
1890
1891  /**
1892   * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1893   * record does not have a changetype, then it may be assumed to be an add
1894   * change record.
1895   *
1896   * @param defaultAdd Indicates whether an LDIF record not containing a
1897   *                   changetype should be retrieved as an add change record.
1898   *                   If this is {@code false} and the record read does not
1899   *                   include a changetype, then an {@link LDIFException} will
1900   *                   be thrown.
1901   *
1902   * @return The change record read from the LDIF source, or {@code null} if
1903   *         there are no more records to be read.
1904   *
1905   * @throws IOException   If a problem occurs while attempting to read from the
1906   *                       LDIF source.
1907   * @throws LDIFException If the data read could not be parsed as an LDIF
1908   *                       change record.
1909   */
1910  @Nullable()
1911  private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1912       throws IOException, LDIFException
1913  {
1914    LDIFChangeRecord r = null;
1915    while (r == null)
1916    {
1917      final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1918      if (unparsedRecord.isEOF())
1919      {
1920        return null;
1921      }
1922
1923      r = decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd,
1924           schema);
1925      Debug.debugLDIFRead(r);
1926
1927      if (changeRecordTranslator != null)
1928      {
1929        r = changeRecordTranslator.translate(r,
1930             unparsedRecord.getFirstLineNumber());
1931      }
1932    }
1933    return r;
1934  }
1935
1936
1937
1938  /**
1939   * Reads a record (either an entry or a change record) from the LDIF source
1940   * and places it in the line list.
1941   *
1942   * @return  The unparsed record that was read.
1943   *
1944   * @throws  IOException  If a problem occurs while attempting to read from the
1945   *                       LDIF source.
1946   *
1947   * @throws  LDIFException  If the data read could not be parsed as a valid
1948   *                         LDIF record.
1949   */
1950  @NotNull()
1951  private UnparsedLDIFRecord readUnparsedRecord()
1952         throws IOException, LDIFException
1953  {
1954    final ArrayList<StringBuilder> lineList = new ArrayList<>(20);
1955    boolean lastWasComment = false;
1956    long firstLineNumber = lineNumberCounter + 1;
1957    while (true)
1958    {
1959      final String line = reader.readLine();
1960      lineNumberCounter++;
1961
1962      if (line == null)
1963      {
1964        // We've hit the end of the LDIF source.  If we haven't read any entry
1965        // data, then return null.  Otherwise, the last entry wasn't followed by
1966        // a blank line, which is OK, and we should decode that entry.
1967        if (lineList.isEmpty())
1968        {
1969          return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1970               duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1971        }
1972        else
1973        {
1974          break;
1975        }
1976      }
1977
1978      if (line.isEmpty())
1979      {
1980        // It's a blank line.  If we have read entry data, then this signals the
1981        // end of the entry.  Otherwise, it's an extra space between entries,
1982        // which is OK.
1983        lastWasComment = false;
1984        if (lineList.isEmpty())
1985        {
1986          firstLineNumber++;
1987          continue;
1988        }
1989        else
1990        {
1991          break;
1992        }
1993      }
1994
1995      if (line.charAt(0) == ' ')
1996      {
1997        // The line starts with a space, which means that it must be a
1998        // continuation of the previous line.  This is true even if the last
1999        // line was a comment.
2000        if (lastWasComment)
2001        {
2002          // What we've read is part of a comment, so we don't care about its
2003          // content.
2004        }
2005        else if (lineList.isEmpty())
2006        {
2007          throw new LDIFException(
2008                         ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
2009                         lineNumberCounter, false);
2010        }
2011        else
2012        {
2013          lineList.get(lineList.size() - 1).append(line.substring(1));
2014          lastWasComment = false;
2015        }
2016      }
2017      else if (line.charAt(0) == '#')
2018      {
2019        lastWasComment = true;
2020      }
2021      else
2022      {
2023        // We want to make sure that we skip over the "version:" line if it
2024        // exists, but that should only occur at the beginning of an entry where
2025        // it can't be confused with a possible "version" attribute.
2026        if (lineList.isEmpty() && line.startsWith("version:"))
2027        {
2028          lastWasComment = true;
2029        }
2030        else
2031        {
2032          lineList.add(new StringBuilder(line));
2033          lastWasComment = false;
2034        }
2035      }
2036    }
2037
2038    return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2039         trailingSpaceBehavior, schema, firstLineNumber);
2040  }
2041
2042
2043
2044  /**
2045   * Decodes the provided set of LDIF lines as an entry.  The provided set of
2046   * lines must contain exactly one entry.  Long lines may be wrapped as per the
2047   * LDIF specification, and it is acceptable to have one or more blank lines
2048   * following the entry. A default trailing space behavior of
2049   * {@link TrailingSpaceBehavior#REJECT} will be used.
2050   *
2051   * @param  ldifLines  The set of lines that comprise the LDIF representation
2052   *                    of the entry.  It must not be {@code null} or empty.
2053   *
2054   * @return  The entry read from LDIF.
2055   *
2056   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
2057   *                         entry.
2058   */
2059  @NotNull()
2060  public static Entry decodeEntry(@NotNull final String... ldifLines)
2061         throws LDIFException
2062  {
2063    final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
2064         TrailingSpaceBehavior.REJECT, null, ldifLines),
2065         DEFAULT_RELATIVE_BASE_PATH);
2066    Debug.debugLDIFRead(e);
2067    return e;
2068  }
2069
2070
2071
2072  /**
2073   * Decodes the provided set of LDIF lines as an entry.  The provided set of
2074   * lines must contain exactly one entry.  Long lines may be wrapped as per the
2075   * LDIF specification, and it is acceptable to have one or more blank lines
2076   * following the entry. A default trailing space behavior of
2077   * {@link TrailingSpaceBehavior#REJECT} will be used.
2078   *
2079   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2080   *                                attribute values encountered while parsing.
2081   * @param  schema                 The schema to use when parsing the record,
2082   *                                if applicable.
2083   * @param  ldifLines              The set of lines that comprise the LDIF
2084   *                                representation of the entry.  It must not be
2085   *                                {@code null} or empty.
2086   *
2087   * @return  The entry read from LDIF.
2088   *
2089   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
2090   *                         entry.
2091   */
2092  @NotNull()
2093  public static Entry decodeEntry(final boolean ignoreDuplicateValues,
2094                                  @Nullable final Schema schema,
2095                                  @NotNull final String... ldifLines)
2096         throws LDIFException
2097  {
2098    return decodeEntry(ignoreDuplicateValues, TrailingSpaceBehavior.REJECT,
2099         schema, ldifLines);
2100  }
2101
2102
2103
2104  /**
2105   * Decodes the provided set of LDIF lines as an entry.  The provided set of
2106   * lines must contain exactly one entry.  Long lines may be wrapped as per the
2107   * LDIF specification, and it is acceptable to have one or more blank lines
2108   * following the entry.
2109   *
2110   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2111   *                                attribute values encountered while parsing.
2112   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
2113   *                                encountering attribute values which are not
2114   *                                base64-encoded but contain trailing spaces.
2115   *                                It must not be {@code null}.
2116   * @param  schema                 The schema to use when parsing the record,
2117   *                                if applicable.
2118   * @param  ldifLines              The set of lines that comprise the LDIF
2119   *                                representation of the entry.  It must not be
2120   *                                {@code null} or empty.
2121   *
2122   * @return  The entry read from LDIF.
2123   *
2124   * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
2125   *                         entry.
2126   */
2127  @NotNull()
2128  public static Entry decodeEntry(
2129                     final boolean ignoreDuplicateValues,
2130                     @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
2131                     @Nullable final Schema schema,
2132                     @NotNull final String... ldifLines)
2133         throws LDIFException
2134  {
2135    final Entry e = decodeEntry(prepareRecord(
2136              (ignoreDuplicateValues
2137                   ? DuplicateValueBehavior.STRIP
2138                   : DuplicateValueBehavior.REJECT),
2139         trailingSpaceBehavior, schema, ldifLines),
2140         DEFAULT_RELATIVE_BASE_PATH);
2141    Debug.debugLDIFRead(e);
2142    return e;
2143  }
2144
2145
2146
2147  /**
2148   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2149   * provided set of lines must contain exactly one change record and it must
2150   * include a changetype.  Long lines may be wrapped as per the LDIF
2151   * specification, and it is acceptable to have one or more blank lines
2152   * following the entry.
2153   *
2154   * @param  ldifLines  The set of lines that comprise the LDIF representation
2155   *                    of the change record.  It must not be {@code null} or
2156   *                    empty.
2157   *
2158   * @return  The change record read from LDIF.
2159   *
2160   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2161   *                         change record.
2162   */
2163  @NotNull()
2164  public static LDIFChangeRecord decodeChangeRecord(
2165                                      @NotNull final String... ldifLines)
2166         throws LDIFException
2167  {
2168    return decodeChangeRecord(false, ldifLines);
2169  }
2170
2171
2172
2173  /**
2174   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2175   * provided set of lines must contain exactly one change record.  Long lines
2176   * may be wrapped as per the LDIF specification, and it is acceptable to have
2177   * one or more blank lines following the entry.
2178   *
2179   * @param  defaultAdd  Indicates whether an LDIF record not containing a
2180   *                     changetype should be retrieved as an add change record.
2181   *                     If this is {@code false} and the record read does not
2182   *                     include a changetype, then an {@link LDIFException}
2183   *                     will be thrown.
2184   * @param  ldifLines  The set of lines that comprise the LDIF representation
2185   *                    of the change record.  It must not be {@code null} or
2186   *                    empty.
2187   *
2188   * @return  The change record read from LDIF.
2189   *
2190   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2191   *                         change record.
2192   */
2193  @NotNull()
2194  public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
2195                                      @NotNull final String... ldifLines)
2196         throws LDIFException
2197  {
2198    final LDIFChangeRecord r =
2199         decodeChangeRecord(
2200              prepareRecord(DuplicateValueBehavior.STRIP,
2201                   TrailingSpaceBehavior.REJECT, null, ldifLines),
2202              DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
2203    Debug.debugLDIFRead(r);
2204    return r;
2205  }
2206
2207
2208
2209  /**
2210   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2211   * provided set of lines must contain exactly one change record.  Long lines
2212   * may be wrapped as per the LDIF specification, and it is acceptable to have
2213   * one or more blank lines following the entry.
2214   *
2215   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2216   *                                attribute values encountered while parsing.
2217   * @param  schema                 The schema to use when processing the change
2218   *                                record, or {@code null} if no schema should
2219   *                                be used and all values should be treated as
2220   *                                case-insensitive strings.
2221   * @param  defaultAdd             Indicates whether an LDIF record not
2222   *                                containing a changetype should be retrieved
2223   *                                as an add change record.  If this is
2224   *                                {@code false} and the record read does not
2225   *                                include a changetype, then an
2226   *                                {@link LDIFException} will be thrown.
2227   * @param  ldifLines              The set of lines that comprise the LDIF
2228   *                                representation of the change record.  It
2229   *                                must not be {@code null} or empty.
2230   *
2231   * @return  The change record read from LDIF.
2232   *
2233   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2234   *                         change record.
2235   */
2236  @NotNull()
2237  public static LDIFChangeRecord decodeChangeRecord(
2238                     final boolean ignoreDuplicateValues,
2239                     @Nullable final Schema schema,
2240                     final boolean defaultAdd,
2241                     @NotNull final String... ldifLines)
2242         throws LDIFException
2243  {
2244    return decodeChangeRecord(ignoreDuplicateValues,
2245         TrailingSpaceBehavior.REJECT, schema, defaultAdd, ldifLines);
2246  }
2247
2248
2249
2250  /**
2251   * Decodes the provided set of LDIF lines as an LDIF change record.  The
2252   * provided set of lines must contain exactly one change record.  Long lines
2253   * may be wrapped as per the LDIF specification, and it is acceptable to have
2254   * one or more blank lines following the entry.
2255   *
2256   * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2257   *                                attribute values encountered while parsing.
2258   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
2259   *                                encountering attribute values which are not
2260   *                                base64-encoded but contain trailing spaces.
2261   *                                It must not be {@code null}.
2262   * @param  schema                 The schema to use when processing the change
2263   *                                record, or {@code null} if no schema should
2264   *                                be used and all values should be treated as
2265   *                                case-insensitive strings.
2266   * @param  defaultAdd             Indicates whether an LDIF record not
2267   *                                containing a changetype should be retrieved
2268   *                                as an add change record.  If this is
2269   *                                {@code false} and the record read does not
2270   *                                include a changetype, then an
2271   *                                {@link LDIFException} will be thrown.
2272   * @param  ldifLines              The set of lines that comprise the LDIF
2273   *                                representation of the change record.  It
2274   *                                must not be {@code null} or empty.
2275   *
2276   * @return  The change record read from LDIF.
2277   *
2278   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2279   *                         change record.
2280   */
2281  @NotNull()
2282  public static LDIFChangeRecord decodeChangeRecord(
2283                     final boolean ignoreDuplicateValues,
2284                     @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
2285                     @Nullable final Schema schema,
2286                     final boolean defaultAdd,
2287                     @NotNull final String... ldifLines)
2288         throws LDIFException
2289  {
2290    final LDIFChangeRecord r = decodeChangeRecord(
2291         prepareRecord(
2292              (ignoreDuplicateValues
2293                   ? DuplicateValueBehavior.STRIP
2294                   : DuplicateValueBehavior.REJECT),
2295              trailingSpaceBehavior, schema, ldifLines),
2296         DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
2297    Debug.debugLDIFRead(r);
2298    return r;
2299  }
2300
2301
2302
2303  /**
2304   * Parses the provided set of lines into a list of {@code StringBuilder}
2305   * objects suitable for decoding into an entry or LDIF change record.
2306   * Comments will be ignored and wrapped lines will be unwrapped.
2307   *
2308   * @param  duplicateValueBehavior  The behavior that should be exhibited if
2309   *                                 the LDIF reader encounters an entry with
2310   *                                 duplicate values.
2311   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2312   *                                 encountering attribute values which are not
2313   *                                 base64-encoded but contain trailing spaces.
2314   * @param  schema                  The schema to use when parsing the record,
2315   *                                 if applicable.
2316   * @param  ldifLines               The set of lines that comprise the record
2317   *                                 to decode.  It must not be {@code null} or
2318   *                                 empty.
2319   *
2320   * @return  The prepared list of {@code StringBuilder} objects ready to be
2321   *          decoded.
2322   *
2323   * @throws  LDIFException  If the provided lines do not contain valid LDIF
2324   *                         content.
2325   */
2326  @NotNull()
2327  private static UnparsedLDIFRecord prepareRecord(
2328               @NotNull final DuplicateValueBehavior duplicateValueBehavior,
2329               @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
2330               @Nullable final Schema schema,
2331               @NotNull final String... ldifLines)
2332          throws LDIFException
2333  {
2334    Validator.ensureNotNull(ldifLines);
2335    Validator.ensureFalse(ldifLines.length == 0,
2336         "LDIFReader.prepareRecord.ldifLines must not be empty.");
2337
2338    boolean lastWasComment = false;
2339    final ArrayList<StringBuilder> lineList = new ArrayList<>(ldifLines.length);
2340    for (int i=0; i < ldifLines.length; i++)
2341    {
2342      final String line = ldifLines[i];
2343      if (line.isEmpty())
2344      {
2345        // This is only acceptable if there are no more non-empty lines in the
2346        // array.
2347        for (int j=i+1; j < ldifLines.length; j++)
2348        {
2349          if (! ldifLines[j].isEmpty())
2350          {
2351            throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
2352                                    ldifLines, null);
2353          }
2354
2355          // If we've gotten here, then we know that we're at the end of the
2356          // entry.  If we have read data, then we can decode it as an entry.
2357          // Otherwise, there was no real data in the provided LDIF lines.
2358          if (lineList.isEmpty())
2359          {
2360            throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
2361                                    ldifLines, null);
2362          }
2363          else
2364          {
2365            return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2366                 trailingSpaceBehavior, schema, 0);
2367          }
2368        }
2369      }
2370
2371      if (line.charAt(0) == ' ')
2372      {
2373        if (i > 0)
2374        {
2375          if (! lastWasComment)
2376          {
2377            lineList.get(lineList.size() - 1).append(line.substring(1));
2378          }
2379        }
2380        else
2381        {
2382          throw new LDIFException(
2383                         ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
2384                         true, ldifLines, null);
2385        }
2386      }
2387      else if (line.charAt(0) == '#')
2388      {
2389        lastWasComment = true;
2390      }
2391      else
2392      {
2393        lineList.add(new StringBuilder(line));
2394        lastWasComment = false;
2395      }
2396    }
2397
2398    if (lineList.isEmpty())
2399    {
2400      throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
2401    }
2402    else
2403    {
2404      return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2405           trailingSpaceBehavior, schema, 0);
2406    }
2407  }
2408
2409
2410
2411  /**
2412   * Decodes the unparsed record that was read from the LDIF source.  It may be
2413   * either an entry or an LDIF change record.
2414   *
2415   * @param  unparsedRecord    The unparsed LDIF record that was read from the
2416   *                           input.  It must not be {@code null} or empty.
2417   * @param  relativeBasePath  The base path that will be prepended to relative
2418   *                           paths in order to obtain an absolute path.
2419   * @param  schema            The schema to use when parsing.
2420   *
2421   * @return  The parsed record, or {@code null} if there are no more entries to
2422   *          be read.
2423   *
2424   * @throws  LDIFException  If the data read could not be parsed as an entry or
2425   *                         an LDIF change record.
2426   */
2427  @NotNull()
2428  private static LDIFRecord decodeRecord(
2429                      @NotNull final UnparsedLDIFRecord unparsedRecord,
2430                      @NotNull final String relativeBasePath,
2431                      @Nullable final Schema schema)
2432       throws LDIFException
2433  {
2434    // If there was an error reading from the input, then we rethrow it here.
2435    final Exception readError = unparsedRecord.getFailureCause();
2436    if (readError != null)
2437    {
2438      if (readError instanceof LDIFException)
2439      {
2440        // If the error was an LDIFException, which will normally be the case,
2441        // then rethrow it with all of the same state.  We could just
2442        //   throw (LDIFException) readError;
2443        // but that's considered bad form.
2444        final LDIFException ldifEx = (LDIFException) readError;
2445        throw new LDIFException(ldifEx.getMessage(),
2446                                ldifEx.getLineNumber(),
2447                                ldifEx.mayContinueReading(),
2448                                ldifEx.getDataLines(),
2449                                ldifEx.getCause());
2450      }
2451      else
2452      {
2453        throw new LDIFException(StaticUtils.getExceptionMessage(readError),
2454             -1, true, readError);
2455      }
2456    }
2457
2458    if (unparsedRecord.isEOF())
2459    {
2460      return null;
2461    }
2462
2463    final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
2464    if (unparsedRecord.getLineList() == null)
2465    {
2466      return null;  // We can get here if there was an error reading the lines.
2467    }
2468
2469    final LDIFRecord r;
2470    if (lineList.size() == 1)
2471    {
2472      r = decodeEntry(unparsedRecord, relativeBasePath);
2473    }
2474    else
2475    {
2476      final String lowerSecondLine =
2477           StaticUtils.toLowerCase(lineList.get(1).toString());
2478      if (lowerSecondLine.startsWith("changetype:"))
2479      {
2480        r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema);
2481      }
2482      else if (lowerSecondLine.startsWith("control:") && SUPPORT_CONTROLS.get())
2483      {
2484        r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema);
2485      }
2486      else
2487      {
2488        r = decodeEntry(unparsedRecord, relativeBasePath);
2489      }
2490    }
2491
2492    Debug.debugLDIFRead(r);
2493    return r;
2494  }
2495
2496
2497
2498  /**
2499   * Decodes the provided set of LDIF lines as an entry.  The provided list must
2500   * not contain any blank lines or comments, and lines are not allowed to be
2501   * wrapped.
2502   *
2503   * @param  unparsedRecord   The unparsed LDIF record that was read from the
2504   *                          input.  It must not be {@code null} or empty.
2505   * @param  relativeBasePath  The base path that will be prepended to relative
2506   *                           paths in order to obtain an absolute path.
2507   *
2508   * @return  The entry read from LDIF.
2509   *
2510   * @throws  LDIFException  If the provided LDIF data cannot be read as an
2511   *                         entry.
2512   */
2513  @NotNull()
2514  private static Entry decodeEntry(
2515                            @NotNull final UnparsedLDIFRecord unparsedRecord,
2516                            @NotNull final String relativeBasePath)
2517          throws LDIFException
2518  {
2519    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2520    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2521
2522    final Iterator<StringBuilder> iterator = ldifLines.iterator();
2523
2524    // The first line must start with either "version:" or "dn:".  If the first
2525    // line starts with "version:" then the second must start with "dn:".
2526    StringBuilder line = iterator.next();
2527    handleTrailingSpaces(line, null, firstLineNumber,
2528         unparsedRecord.getTrailingSpaceBehavior());
2529    int colonPos = line.indexOf(":");
2530    if ((colonPos > 0) &&
2531        line.substring(0, colonPos).equalsIgnoreCase("version"))
2532    {
2533      // The first line is "version:".  Under most conditions, this will be
2534      // handled by the LDIF reader, but this can happen if you call
2535      // decodeEntry with a set of data that includes a version.  At any rate,
2536      // read the next line, which must specify the DN.
2537      line = iterator.next();
2538      handleTrailingSpaces(line, null, firstLineNumber,
2539           unparsedRecord.getTrailingSpaceBehavior());
2540    }
2541
2542    colonPos = line.indexOf(":");
2543    if ((colonPos < 0) ||
2544         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2545    {
2546      throw new LDIFException(
2547           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2548           firstLineNumber, true, ldifLines, null);
2549    }
2550
2551    final String dn;
2552    final int length = line.length();
2553    if (length == (colonPos+1))
2554    {
2555      // The colon was the last character on the line.  This is acceptable and
2556      // indicates that the entry has the null DN.
2557      dn = "";
2558    }
2559    else if (line.charAt(colonPos+1) == ':')
2560    {
2561      // Skip over any spaces leading up to the value, and then the rest of the
2562      // string is the base64-encoded DN.
2563      int pos = colonPos+2;
2564      while ((pos < length) && (line.charAt(pos) == ' '))
2565      {
2566        pos++;
2567      }
2568
2569      try
2570      {
2571        final byte[] dnBytes = Base64.decode(line.substring(pos));
2572        dn = StaticUtils.toUTF8String(dnBytes);
2573      }
2574      catch (final ParseException pe)
2575      {
2576        Debug.debugException(pe);
2577        throw new LDIFException(
2578                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2579                                                            pe.getMessage()),
2580                       firstLineNumber, true, ldifLines, pe);
2581      }
2582      catch (final Exception e)
2583      {
2584        Debug.debugException(e);
2585        throw new LDIFException(
2586                       ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2587                       firstLineNumber, true, ldifLines, e);
2588      }
2589    }
2590    else
2591    {
2592      // Skip over any spaces leading up to the value, and then the rest of the
2593      // string is the DN.
2594      int pos = colonPos+1;
2595      while ((pos < length) && (line.charAt(pos) == ' '))
2596      {
2597        pos++;
2598      }
2599
2600      dn = line.substring(pos);
2601    }
2602
2603
2604    // The remaining lines must be the attributes for the entry.  However, we
2605    // will allow the case in which an entry does not have any attributes, to be
2606    // able to support reading search result entries in which no attributes were
2607    // returned.
2608    if (! iterator.hasNext())
2609    {
2610      return new Entry(dn, unparsedRecord.getSchema());
2611    }
2612
2613    return new Entry(dn, unparsedRecord.getSchema(),
2614         parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2615              unparsedRecord.getTrailingSpaceBehavior(),
2616              unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2617              firstLineNumber));
2618  }
2619
2620
2621
2622  /**
2623   * Decodes the provided set of LDIF lines as a change record.  The provided
2624   * list must not contain any blank lines or comments, and lines are not
2625   * allowed to be wrapped.
2626   *
2627   * @param  unparsedRecord    The unparsed LDIF record that was read from the
2628   *                           input.  It must not be {@code null} or empty.
2629   * @param  relativeBasePath  The base path that will be prepended to relative
2630   *                           paths in order to obtain an absolute path.
2631   * @param  defaultAdd        Indicates whether an LDIF record not containing a
2632   *                           changetype should be retrieved as an add change
2633   *                           record.  If this is {@code false} and the record
2634   *                           read does not include a changetype, then an
2635   *                           {@link LDIFException} will be thrown.
2636   * @param  schema            The schema to use in parsing.
2637   *
2638   * @return  The change record read from LDIF.
2639   *
2640   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2641   *                         change record.
2642   */
2643  @NotNull()
2644  private static LDIFChangeRecord decodeChangeRecord(
2645                      @NotNull final UnparsedLDIFRecord unparsedRecord,
2646                      @NotNull final String relativeBasePath,
2647                      final boolean defaultAdd,
2648                      @Nullable final Schema schema)
2649          throws LDIFException
2650  {
2651    final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2652    final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2653
2654    Iterator<StringBuilder> iterator = ldifLines.iterator();
2655
2656    // The first line must start with either "version:" or "dn:".  If the first
2657    // line starts with "version:" then the second must start with "dn:".
2658    StringBuilder line = iterator.next();
2659    handleTrailingSpaces(line, null, firstLineNumber,
2660         unparsedRecord.getTrailingSpaceBehavior());
2661    int colonPos = line.indexOf(":");
2662    int linesRead = 1;
2663    if ((colonPos > 0) &&
2664        line.substring(0, colonPos).equalsIgnoreCase("version"))
2665    {
2666      // The first line is "version:".  Under most conditions, this will be
2667      // handled by the LDIF reader, but this can happen if you call
2668      // decodeEntry with a set of data that includes a version.  At any rate,
2669      // read the next line, which must specify the DN.
2670      line = iterator.next();
2671      linesRead++;
2672      handleTrailingSpaces(line, null, firstLineNumber,
2673           unparsedRecord.getTrailingSpaceBehavior());
2674    }
2675
2676    colonPos = line.indexOf(":");
2677    if ((colonPos < 0) ||
2678         (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2679    {
2680      throw new LDIFException(
2681           ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2682           firstLineNumber, true, ldifLines, null);
2683    }
2684
2685    final String dn;
2686    final int length = line.length();
2687    if (length == (colonPos+1))
2688    {
2689      // The colon was the last character on the line.  This is acceptable and
2690      // indicates that the entry has the null DN.
2691      dn = "";
2692    }
2693    else if (line.charAt(colonPos+1) == ':')
2694    {
2695      // Skip over any spaces leading up to the value, and then the rest of the
2696      // string is the base64-encoded DN.
2697      int pos = colonPos+2;
2698      while ((pos < length) && (line.charAt(pos) == ' '))
2699      {
2700        pos++;
2701      }
2702
2703      try
2704      {
2705        final byte[] dnBytes = Base64.decode(line.substring(pos));
2706        dn = StaticUtils.toUTF8String(dnBytes);
2707      }
2708      catch (final ParseException pe)
2709      {
2710        Debug.debugException(pe);
2711        throw new LDIFException(
2712                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2713                                                               pe.getMessage()),
2714                       firstLineNumber, true, ldifLines, pe);
2715      }
2716      catch (final Exception e)
2717      {
2718        Debug.debugException(e);
2719        throw new LDIFException(
2720                       ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2721                                                               e),
2722                       firstLineNumber, true, ldifLines, e);
2723      }
2724    }
2725    else
2726    {
2727      // Skip over any spaces leading up to the value, and then the rest of the
2728      // string is the DN.
2729      int pos = colonPos+1;
2730      while ((pos < length) && (line.charAt(pos) == ' '))
2731      {
2732        pos++;
2733      }
2734
2735      dn = line.substring(pos);
2736    }
2737
2738
2739    // An LDIF change record may contain zero or more controls, with the end of
2740    // the controls signified by the changetype.  The changetype element must be
2741    // present, unless defaultAdd is true in which case the first thing that is
2742    // neither control or changetype will trigger the start of add attribute
2743    // parsing.  If support for controls has been disabled, then a control
2744    // element before the changetype will be treated in the same way as any
2745    // other attribute before the changetype, and it will either be treated as
2746    // an attribute as part of an add change record (if defaultAdd is true) or
2747    // the LDIF record will be rejected as missing the changetype.
2748    if (! iterator.hasNext())
2749    {
2750      throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2751                              firstLineNumber, true, ldifLines, null);
2752    }
2753
2754    String changeType;
2755    ArrayList<Control> controls = null;
2756    while (true)
2757    {
2758      line = iterator.next();
2759      handleTrailingSpaces(line, dn, firstLineNumber,
2760           unparsedRecord.getTrailingSpaceBehavior());
2761      colonPos = line.indexOf(":");
2762      if (colonPos < 0)
2763      {
2764        throw new LDIFException(
2765             ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2766             firstLineNumber, true, ldifLines, null);
2767      }
2768
2769      final String token = StaticUtils.toLowerCase(line.substring(0, colonPos));
2770      if (token.equals("control") && SUPPORT_CONTROLS.get())
2771      {
2772        if (controls == null)
2773        {
2774          controls = new ArrayList<>(5);
2775        }
2776
2777        controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2778             relativeBasePath));
2779      }
2780      else if (token.equals("changetype"))
2781      {
2782        changeType =
2783             decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2784        break;
2785      }
2786      else if (defaultAdd)
2787      {
2788        // The line we read wasn't a control or changetype declaration, so we'll
2789        // assume it's an attribute in an add record.  However, we're not ready
2790        // for that yet, and since we can't rewind an iterator we'll create a
2791        // new one that hasn't yet gotten to this line.
2792        changeType = "add";
2793        iterator = ldifLines.iterator();
2794        for (int i=0; i < linesRead; i++)
2795        {
2796          iterator.next();
2797        }
2798        break;
2799      }
2800      else
2801      {
2802        throw new LDIFException(
2803             ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2804                  firstLineNumber),
2805             firstLineNumber, true, ldifLines, null);
2806      }
2807
2808      linesRead++;
2809    }
2810
2811
2812    // Make sure that the change type is acceptable and then decode the rest of
2813    // the change record accordingly.
2814    final String lowerChangeType = StaticUtils.toLowerCase(changeType);
2815    if (lowerChangeType.equals("add"))
2816    {
2817      // There must be at least one more line.  If not, then that's an error.
2818      // Otherwise, parse the rest of the data as attribute-value pairs.
2819      if (iterator.hasNext())
2820      {
2821        final Collection<Attribute> attrs =
2822             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2823                  unparsedRecord.getTrailingSpaceBehavior(),
2824                  unparsedRecord.getSchema(), ldifLines, iterator,
2825                  relativeBasePath, firstLineNumber);
2826        final Attribute[] attributes = new Attribute[attrs.size()];
2827        final Iterator<Attribute> attrIterator = attrs.iterator();
2828        for (int i=0; i < attributes.length; i++)
2829        {
2830          attributes[i] = attrIterator.next();
2831        }
2832
2833        return new LDIFAddChangeRecord(dn, attributes, controls);
2834      }
2835      else
2836      {
2837        throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2838                                firstLineNumber, true, ldifLines, null);
2839      }
2840    }
2841    else if (lowerChangeType.equals("delete"))
2842    {
2843      // There shouldn't be any more data.  If there is, then that's an error.
2844      // Otherwise, we can just return the delete change record with what we
2845      // already know.
2846      if (iterator.hasNext())
2847      {
2848        throw new LDIFException(
2849                       ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2850                       firstLineNumber, true, ldifLines, null);
2851      }
2852      else
2853      {
2854        return new LDIFDeleteChangeRecord(dn, controls);
2855      }
2856    }
2857    else if (lowerChangeType.equals("modify"))
2858    {
2859      // There must be at least one more line.  If not, then that's an error.
2860      // Otherwise, parse the rest of the data as a set of modifications.
2861      if (iterator.hasNext())
2862      {
2863        final Modification[] mods = parseModifications(dn,
2864             unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2865             relativeBasePath, firstLineNumber, schema);
2866        return new LDIFModifyChangeRecord(dn, mods, controls);
2867      }
2868      else
2869      {
2870        throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2871                                firstLineNumber, true, ldifLines, null);
2872      }
2873    }
2874    else if (lowerChangeType.equals("moddn") ||
2875             lowerChangeType.equals("modrdn"))
2876    {
2877      // There must be at least one more line.  If not, then that's an error.
2878      // Otherwise, parse the rest of the data as a set of modifications.
2879      if (iterator.hasNext())
2880      {
2881        return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2882             unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2883      }
2884      else
2885      {
2886        throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2887                                firstLineNumber, true, ldifLines, null);
2888      }
2889    }
2890    else
2891    {
2892      throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2893                                                         firstLineNumber),
2894                              firstLineNumber, true, ldifLines, null);
2895    }
2896  }
2897
2898
2899
2900  /**
2901   * Decodes information about a control from the provided line.
2902   *
2903   * @param  line              The line to process.
2904   * @param  colonPos          The position of the colon that separates the
2905   *                           control token string from tbe encoded control.
2906   * @param  firstLineNumber   The line number for the start of the record.
2907   * @param  ldifLines         The lines that comprise the LDIF representation
2908   *                           of the full record being parsed.
2909   * @param  relativeBasePath  The base path that will be prepended to relative
2910   *                           paths in order to obtain an absolute path.
2911   *
2912   * @return  The decoded control.
2913   *
2914   * @throws  LDIFException  If a problem is encountered while trying to decode
2915   *                         the changetype.
2916   */
2917  @NotNull()
2918  private static Control decodeControl(@NotNull final StringBuilder line,
2919                              final int colonPos, final long firstLineNumber,
2920                              @NotNull final ArrayList<StringBuilder> ldifLines,
2921                              @NotNull final String relativeBasePath)
2922          throws LDIFException
2923  {
2924    final String controlString;
2925    int length = line.length();
2926    if (length == (colonPos+1))
2927    {
2928      // The colon was the last character on the line.  This is not
2929      // acceptable.
2930      throw new LDIFException(
2931           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2932           firstLineNumber, true, ldifLines, null);
2933    }
2934    else if (line.charAt(colonPos+1) == ':')
2935    {
2936      // Skip over any spaces leading up to the value, and then the rest of
2937      // the string is the base64-encoded control representation.  This is
2938      // unusual and unnecessary, but is nevertheless acceptable.
2939      int pos = colonPos+2;
2940      while ((pos < length) && (line.charAt(pos) == ' '))
2941      {
2942        pos++;
2943      }
2944
2945      try
2946      {
2947        final byte[] controlBytes = Base64.decode(line.substring(pos));
2948        controlString =  StaticUtils.toUTF8String(controlBytes);
2949      }
2950      catch (final ParseException pe)
2951      {
2952        Debug.debugException(pe);
2953        throw new LDIFException(
2954                       ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2955                            firstLineNumber, pe.getMessage()),
2956                       firstLineNumber, true, ldifLines, pe);
2957      }
2958      catch (final Exception e)
2959      {
2960        Debug.debugException(e);
2961        throw new LDIFException(
2962             ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2963             firstLineNumber, true, ldifLines, e);
2964      }
2965    }
2966    else
2967    {
2968      // Skip over any spaces leading up to the value, and then the rest of
2969      // the string is the encoded control.
2970      int pos = colonPos+1;
2971      while ((pos < length) && (line.charAt(pos) == ' '))
2972      {
2973        pos++;
2974      }
2975
2976      controlString = line.substring(pos);
2977    }
2978
2979    // If the resulting control definition is empty, then that's invalid.
2980    if (controlString.isEmpty())
2981    {
2982      throw new LDIFException(
2983           ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2984           firstLineNumber, true, ldifLines, null);
2985    }
2986
2987
2988    // The first element of the control must be the OID, and it must be followed
2989    // by a space (to separate it from the criticality), a colon (to separate it
2990    // from the value and indicate a default criticality of false), or the end
2991    // of the line (to indicate a default criticality of false and no value).
2992    String oid = null;
2993    boolean hasCriticality = false;
2994    boolean hasValue = false;
2995    int pos = 0;
2996    length = controlString.length();
2997    while (pos < length)
2998    {
2999      final char c = controlString.charAt(pos);
3000      if (c == ':')
3001      {
3002        // This indicates that there is no criticality and that the value
3003        // immediately follows the OID.
3004        oid = controlString.substring(0, pos++);
3005        hasValue = true;
3006        break;
3007      }
3008      else if (c == ' ')
3009      {
3010        // This indicates that there is a criticality.  We don't know anything
3011        // about the presence of a value yet.
3012        oid = controlString.substring(0, pos++);
3013        hasCriticality = true;
3014        break;
3015      }
3016      else
3017      {
3018        pos++;
3019      }
3020    }
3021
3022    if (oid == null)
3023    {
3024      // This indicates that the string representation of the control is only
3025      // the OID.
3026      return new Control(controlString, false);
3027    }
3028
3029
3030    // See if we need to read the criticality.  If so, then do so now.
3031    // Otherwise, assume a default criticality of false.
3032    final boolean isCritical;
3033    if (hasCriticality)
3034    {
3035      // Skip over any spaces before the criticality.
3036      while (controlString.charAt(pos) == ' ')
3037      {
3038        pos++;
3039      }
3040
3041      // Read until we find a colon or the end of the string.
3042      final int criticalityStartPos = pos;
3043      while (pos < length)
3044      {
3045        final char c = controlString.charAt(pos);
3046        if (c == ':')
3047        {
3048          hasValue = true;
3049          break;
3050        }
3051        else
3052        {
3053          pos++;
3054        }
3055      }
3056
3057      final String criticalityString =
3058           StaticUtils.toLowerCase(controlString.substring(criticalityStartPos,
3059                pos));
3060      if (criticalityString.equals("true"))
3061      {
3062        isCritical = true;
3063      }
3064      else if (criticalityString.equals("false"))
3065      {
3066        isCritical = false;
3067      }
3068      else
3069      {
3070        throw new LDIFException(
3071             ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
3072                  firstLineNumber),
3073             firstLineNumber, true, ldifLines, null);
3074      }
3075
3076      if (hasValue)
3077      {
3078        pos++;
3079      }
3080    }
3081    else
3082    {
3083      isCritical = false;
3084    }
3085
3086    // See if we need to read the value.  If so, then do so now.  It may be
3087    // a string, or it may be base64-encoded.  It could conceivably even be read
3088    // from a URL.
3089    final ASN1OctetString value;
3090    if (hasValue)
3091    {
3092      // The character immediately after the colon that precedes the value may
3093      // be one of the following:
3094      // - A second colon (optionally followed by a single space) to indicate
3095      //   that the value is base64-encoded.
3096      // - A less-than symbol to indicate that the value should be read from a
3097      //   location specified by a URL.
3098      // - A single space that precedes the non-base64-encoded value.
3099      // - The first character of the non-base64-encoded value.
3100      switch (controlString.charAt(pos))
3101      {
3102        case ':':
3103          try
3104          {
3105            if (controlString.length() == (pos+1))
3106            {
3107              value = new ASN1OctetString();
3108            }
3109            else if (controlString.charAt(pos+1) == ' ')
3110            {
3111              value = new ASN1OctetString(
3112                   Base64.decode(controlString.substring(pos+2)));
3113            }
3114            else
3115            {
3116              value = new ASN1OctetString(
3117                   Base64.decode(controlString.substring(pos+1)));
3118            }
3119          }
3120          catch (final Exception e)
3121          {
3122            Debug.debugException(e);
3123            throw new LDIFException(
3124                 ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
3125                      firstLineNumber, StaticUtils.getExceptionMessage(e)),
3126                 firstLineNumber, true, ldifLines, e);
3127          }
3128          break;
3129        case '<':
3130          try
3131          {
3132            final String urlString;
3133            if (controlString.charAt(pos+1) == ' ')
3134            {
3135              urlString = controlString.substring(pos+2);
3136            }
3137            else
3138            {
3139              urlString = controlString.substring(pos+1);
3140            }
3141            value = new ASN1OctetString(retrieveURLBytes(urlString,
3142                 relativeBasePath, firstLineNumber));
3143          }
3144          catch (final Exception e)
3145          {
3146            Debug.debugException(e);
3147            throw new LDIFException(
3148                 ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
3149                      firstLineNumber, StaticUtils.getExceptionMessage(e)),
3150                 firstLineNumber, true, ldifLines, e);
3151          }
3152          break;
3153        case ' ':
3154          value = new ASN1OctetString(controlString.substring(pos+1));
3155          break;
3156        default:
3157          value = new ASN1OctetString(controlString.substring(pos));
3158          break;
3159      }
3160    }
3161    else
3162    {
3163      value = null;
3164    }
3165
3166    return new Control(oid, isCritical, value);
3167  }
3168
3169
3170
3171  /**
3172   * Decodes the changetype element from the provided line.
3173   *
3174   * @param  line             The line to process.
3175   * @param  colonPos         The position of the colon that separates the
3176   *                          changetype string from its value.
3177   * @param  firstLineNumber  The line number for the start of the record.
3178   * @param  ldifLines        The lines that comprise the LDIF representation of
3179   *                          the full record being parsed.
3180   *
3181   * @return  The decoded changetype string.
3182   *
3183   * @throws  LDIFException  If a problem is encountered while trying to decode
3184   *                         the changetype.
3185   */
3186  @NotNull()
3187  private static String decodeChangeType(@NotNull final StringBuilder line,
3188                             final int colonPos, final long firstLineNumber,
3189                             @NotNull final ArrayList<StringBuilder> ldifLines)
3190          throws LDIFException
3191  {
3192    final int length = line.length();
3193    if (length == (colonPos+1))
3194    {
3195      // The colon was the last character on the line.  This is not
3196      // acceptable.
3197      throw new LDIFException(
3198           ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
3199           true, ldifLines, null);
3200    }
3201    else if (line.charAt(colonPos+1) == ':')
3202    {
3203      // Skip over any spaces leading up to the value, and then the rest of
3204      // the string is the base64-encoded changetype.  This is unusual and
3205      // unnecessary, but is nevertheless acceptable.
3206      int pos = colonPos+2;
3207      while ((pos < length) && (line.charAt(pos) == ' '))
3208      {
3209        pos++;
3210      }
3211
3212      try
3213      {
3214        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3215        return StaticUtils.toUTF8String(changeTypeBytes);
3216      }
3217      catch (final ParseException pe)
3218      {
3219        Debug.debugException(pe);
3220        throw new LDIFException(
3221                       ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
3222                                                            pe.getMessage()),
3223                       firstLineNumber, true, ldifLines, pe);
3224      }
3225      catch (final Exception e)
3226      {
3227        Debug.debugException(e);
3228        throw new LDIFException(
3229             ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
3230             firstLineNumber, true, ldifLines, e);
3231      }
3232    }
3233    else
3234    {
3235      // Skip over any spaces leading up to the value, and then the rest of
3236      // the string is the changetype.
3237      int pos = colonPos+1;
3238      while ((pos < length) && (line.charAt(pos) == ' '))
3239      {
3240        pos++;
3241      }
3242
3243      return line.substring(pos);
3244    }
3245  }
3246
3247
3248
3249  /**
3250   * Parses the data available through the provided iterator as a collection of
3251   * attributes suitable for use in an entry or an add change record.
3252   *
3253   * @param  dn                      The DN of the record being read.
3254   * @param  duplicateValueBehavior  The behavior that should be exhibited if
3255   *                                 the LDIF reader encounters an entry with
3256   *                                 duplicate values.
3257   * @param  trailingSpaceBehavior   The behavior that should be exhibited when
3258   *                                 encountering attribute values which are not
3259   *                                 base64-encoded but contain trailing spaces.
3260   * @param  schema                  The schema to use when parsing the
3261   *                                 attributes, or {@code null} if none is
3262   *                                 needed.
3263   * @param  ldifLines               The lines that comprise the LDIF
3264   *                                 representation of the full record being
3265   *                                 parsed.
3266   * @param  iterator                The iterator to use to access the attribute
3267   *                                 lines.
3268   * @param  relativeBasePath        The base path that will be prepended to
3269   *                                 relative paths in order to obtain an
3270   *                                 absolute path.
3271   * @param  firstLineNumber         The line number for the start of the
3272   *                                 record.
3273   *
3274   * @return  The collection of attributes that were read.
3275   *
3276   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3277   *                         set of attributes.
3278   */
3279  @NotNull()
3280  private static ArrayList<Attribute> parseAttributes(@NotNull final String dn,
3281               @NotNull final DuplicateValueBehavior duplicateValueBehavior,
3282               @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
3283               @Nullable final Schema schema,
3284               @NotNull final ArrayList<StringBuilder> ldifLines,
3285               @NotNull final Iterator<StringBuilder> iterator,
3286               @NotNull final String relativeBasePath,
3287               final long firstLineNumber)
3288          throws LDIFException
3289  {
3290    final LinkedHashMap<String,Object> attributes =
3291         new LinkedHashMap<>(StaticUtils.computeMapCapacity(ldifLines.size()));
3292    while (iterator.hasNext())
3293    {
3294      final StringBuilder line = iterator.next();
3295      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3296      final int colonPos = line.indexOf(":");
3297      if (colonPos <= 0)
3298      {
3299        throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3300                                firstLineNumber, true, ldifLines, null);
3301      }
3302
3303      final String attributeName = line.substring(0, colonPos);
3304      final String lowerName     = StaticUtils.toLowerCase(attributeName);
3305
3306      final MatchingRule matchingRule;
3307      if (schema == null)
3308      {
3309        matchingRule = CaseIgnoreStringMatchingRule.getInstance();
3310      }
3311      else
3312      {
3313        matchingRule =
3314             MatchingRule.selectEqualityMatchingRule(attributeName, schema);
3315      }
3316
3317      Attribute attr;
3318      final LDIFAttribute ldifAttr;
3319      final Object attrObject = attributes.get(lowerName);
3320      if (attrObject == null)
3321      {
3322        attr     = null;
3323        ldifAttr = null;
3324      }
3325      else
3326      {
3327        if (attrObject instanceof Attribute)
3328        {
3329          attr     = (Attribute) attrObject;
3330          ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
3331                                       attr.getRawValues()[0]);
3332          attributes.put(lowerName, ldifAttr);
3333        }
3334        else
3335        {
3336          attr     = null;
3337          ldifAttr = (LDIFAttribute) attrObject;
3338        }
3339      }
3340
3341      final int length = line.length();
3342      if (length == (colonPos+1))
3343      {
3344        // This means that the attribute has a zero-length value, which is
3345        // acceptable.
3346        if (attrObject == null)
3347        {
3348          attr = new Attribute(attributeName, matchingRule, "");
3349          attributes.put(lowerName, attr);
3350        }
3351        else
3352        {
3353          try
3354          {
3355            if (! ldifAttr.addValue(new ASN1OctetString(),
3356                       duplicateValueBehavior))
3357            {
3358              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3359              {
3360                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3361                     firstLineNumber, attributeName), firstLineNumber, true,
3362                     ldifLines, null);
3363              }
3364            }
3365          }
3366          catch (final LDAPException le)
3367          {
3368            throw new LDIFException(
3369                 ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3370                      attributeName, StaticUtils.getExceptionMessage(le)),
3371                 firstLineNumber, true, ldifLines, le);
3372          }
3373        }
3374      }
3375      else if (line.charAt(colonPos+1) == ':')
3376      {
3377        // Skip over any spaces leading up to the value, and then the rest of
3378        // the string is the base64-encoded attribute value.
3379        int pos = colonPos+2;
3380        while ((pos < length) && (line.charAt(pos) == ' '))
3381        {
3382          pos++;
3383        }
3384
3385        try
3386        {
3387          final byte[] valueBytes = Base64.decode(line.substring(pos));
3388          if (attrObject == null)
3389          {
3390            attr = new Attribute(attributeName, matchingRule, valueBytes);
3391            attributes.put(lowerName, attr);
3392          }
3393          else
3394          {
3395            try
3396            {
3397              if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
3398                         duplicateValueBehavior))
3399              {
3400                if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3401                {
3402                  throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3403                       firstLineNumber, attributeName), firstLineNumber, true,
3404                       ldifLines, null);
3405                }
3406              }
3407            }
3408            catch (final LDAPException le)
3409            {
3410              throw new LDIFException(
3411                   ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3412                        attributeName, StaticUtils.getExceptionMessage(le)),
3413                   firstLineNumber, true, ldifLines, le);
3414            }
3415          }
3416        }
3417        catch (final ParseException pe)
3418        {
3419          Debug.debugException(pe);
3420          throw new LDIFException(
3421               ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(attributeName,
3422                    firstLineNumber, pe.getMessage()),
3423               firstLineNumber, true, ldifLines, pe);
3424        }
3425      }
3426      else if (line.charAt(colonPos+1) == '<')
3427      {
3428        // Skip over any spaces leading up to the value, and then the rest of
3429        // the string is a URL that indicates where to get the real content.
3430        // At the present time, we'll only support the file URLs.
3431        int pos = colonPos+2;
3432        while ((pos < length) && (line.charAt(pos) == ' '))
3433        {
3434          pos++;
3435        }
3436
3437        final byte[] urlBytes;
3438        final String urlString = line.substring(pos);
3439        try
3440        {
3441          urlBytes =
3442               retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
3443        }
3444        catch (final Exception e)
3445        {
3446          Debug.debugException(e);
3447          throw new LDIFException(
3448               ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3449                    firstLineNumber, e),
3450               firstLineNumber, true, ldifLines, e);
3451        }
3452
3453        if (attrObject == null)
3454        {
3455          attr = new Attribute(attributeName, matchingRule, urlBytes);
3456          attributes.put(lowerName, attr);
3457        }
3458        else
3459        {
3460          try
3461          {
3462            if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
3463                 duplicateValueBehavior))
3464            {
3465              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3466              {
3467                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3468                     firstLineNumber, attributeName), firstLineNumber, true,
3469                     ldifLines, null);
3470              }
3471            }
3472          }
3473          catch (final LDIFException le)
3474          {
3475            Debug.debugException(le);
3476            throw le;
3477          }
3478          catch (final Exception e)
3479          {
3480            Debug.debugException(e);
3481            throw new LDIFException(
3482                 ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3483                      firstLineNumber, e),
3484                 firstLineNumber, true, ldifLines, e);
3485          }
3486        }
3487      }
3488      else
3489      {
3490        // Skip over any spaces leading up to the value, and then the rest of
3491        // the string is the value.
3492        int pos = colonPos+1;
3493        while ((pos < length) && (line.charAt(pos) == ' '))
3494        {
3495          pos++;
3496        }
3497
3498        final String valueString = line.substring(pos);
3499        if (attrObject == null)
3500        {
3501          attr = new Attribute(attributeName, matchingRule, valueString);
3502          attributes.put(lowerName, attr);
3503        }
3504        else
3505        {
3506          try
3507          {
3508            if (! ldifAttr.addValue(new ASN1OctetString(valueString),
3509                       duplicateValueBehavior))
3510            {
3511              if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3512              {
3513                throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3514                     firstLineNumber, attributeName), firstLineNumber, true,
3515                     ldifLines, null);
3516              }
3517            }
3518          }
3519          catch (final LDAPException le)
3520          {
3521            throw new LDIFException(
3522                 ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn, firstLineNumber,
3523                      attributeName, StaticUtils.getExceptionMessage(le)),
3524                 firstLineNumber, true, ldifLines, le);
3525          }
3526        }
3527      }
3528    }
3529
3530    final ArrayList<Attribute> attrList = new ArrayList<>(attributes.size());
3531    for (final Object o : attributes.values())
3532    {
3533      if (o instanceof Attribute)
3534      {
3535        attrList.add((Attribute) o);
3536      }
3537      else
3538      {
3539        attrList.add(((LDIFAttribute) o).toAttribute());
3540      }
3541    }
3542
3543    return attrList;
3544  }
3545
3546
3547
3548  /**
3549   * Retrieves the bytes that make up the file referenced by the given URL.
3550   *
3551   * @param  urlString         The string representation of the URL to retrieve.
3552   * @param  relativeBasePath  The base path that will be prepended to relative
3553   *                           paths in order to obtain an absolute path.
3554   * @param  firstLineNumber   The line number for the start of the record.
3555   *
3556   * @return  The bytes contained in the specified file, or an empty array if
3557   *          the specified file is empty.
3558   *
3559   * @throws  LDIFException  If the provided URL is malformed or references a
3560   *                         nonexistent file.
3561   *
3562   * @throws  IOException  If a problem is encountered while attempting to read
3563   *                       from the target file.
3564   */
3565  @NotNull()
3566  private static byte[] retrieveURLBytes(@NotNull final String urlString,
3567                                         @NotNull final String relativeBasePath,
3568                                         final long firstLineNumber)
3569          throws LDIFException, IOException
3570  {
3571    int pos;
3572    final String path;
3573    final String lowerURLString = StaticUtils.toLowerCase(urlString);
3574    if (lowerURLString.startsWith("file:/"))
3575    {
3576      pos = 6;
3577      while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3578      {
3579        pos++;
3580      }
3581
3582      path = urlString.substring(pos-1);
3583    }
3584    else if (lowerURLString.startsWith("file:"))
3585    {
3586      // A file: URL that doesn't include a slash will be interpreted as a
3587      // relative path.
3588      path = relativeBasePath + urlString.substring(5);
3589    }
3590    else
3591    {
3592      throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3593           firstLineNumber, true);
3594    }
3595
3596    final File f = new File(path);
3597    if (! f.exists())
3598    {
3599      throw new LDIFException(
3600           ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3601           firstLineNumber, true);
3602    }
3603
3604    // In order to conserve memory, we'll only allow values to be read from
3605    // files no larger than 10 megabytes.
3606    final long fileSize = f.length();
3607    if (fileSize > MAX_URL_FILE_SIZE)
3608    {
3609      throw new LDIFException(
3610           ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3611                MAX_URL_FILE_SIZE),
3612           firstLineNumber, true);
3613    }
3614
3615    int fileBytesRemaining = (int) fileSize;
3616    final byte[] fileData = new byte[(int) fileSize];
3617    final FileInputStream fis = new FileInputStream(f);
3618    try
3619    {
3620      int fileBytesRead = 0;
3621      while (fileBytesRead < fileSize)
3622      {
3623        final int bytesRead =
3624             fis.read(fileData, fileBytesRead, fileBytesRemaining);
3625        if (bytesRead < 0)
3626        {
3627          // We hit the end of the file before we expected to.  This shouldn't
3628          // happen unless the file size changed since we first looked at it,
3629          // which we won't allow.
3630          throw new LDIFException(
3631               ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3632                    f.getAbsolutePath()),
3633               firstLineNumber, true);
3634        }
3635
3636        fileBytesRead      += bytesRead;
3637        fileBytesRemaining -= bytesRead;
3638      }
3639
3640      if (fis.read() != -1)
3641      {
3642        // There is still more data to read.  This shouldn't happen unless the
3643        // file size changed since we first looked at it, which we won't allow.
3644        throw new LDIFException(
3645             ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3646             firstLineNumber, true);
3647      }
3648    }
3649    finally
3650    {
3651      fis.close();
3652    }
3653
3654    return fileData;
3655  }
3656
3657
3658
3659  /**
3660   * Parses the data available through the provided iterator into an array of
3661   * modifications suitable for use in a modify change record.
3662   *
3663   * @param  dn                     The DN of the entry being parsed.
3664   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3665   *                                encountering attribute values which are not
3666   *                                base64-encoded but contain trailing spaces.
3667   * @param  ldifLines              The lines that comprise the LDIF
3668   *                                representation of the full record being
3669   *                                parsed.
3670   * @param  iterator               The iterator to use to access the
3671   *                                modification data.
3672   * @param  relativeBasePath       The base path that will be prepended to
3673   *                                relative paths in order to obtain an
3674   *                                absolute path.
3675   * @param  firstLineNumber        The line number for the start of the record.
3676   * @param  schema                 The schema to use in processing.
3677   *
3678   * @return  An array containing the modifications that were read.
3679   *
3680   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3681   *                         set of modifications.
3682   */
3683  @NotNull()
3684  private static Modification[] parseModifications(@NotNull final String dn,
3685               @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
3686               @NotNull final ArrayList<StringBuilder> ldifLines,
3687               @NotNull final Iterator<StringBuilder> iterator,
3688               @NotNull final String relativeBasePath,
3689               final long firstLineNumber, @Nullable final Schema schema)
3690          throws LDIFException
3691  {
3692    final ArrayList<Modification> modList = new ArrayList<>(ldifLines.size());
3693
3694    while (iterator.hasNext())
3695    {
3696      // The first line must start with "add:", "delete:", "replace:", or
3697      // "increment:" followed by an attribute name.
3698      StringBuilder line = iterator.next();
3699      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3700      int colonPos = line.indexOf(":");
3701      if (colonPos < 0)
3702      {
3703        throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3704                                firstLineNumber, true, ldifLines, null);
3705      }
3706
3707      final ModificationType modType;
3708      final String modTypeStr =
3709           StaticUtils.toLowerCase(line.substring(0, colonPos));
3710      if (modTypeStr.equals("add"))
3711      {
3712        modType = ModificationType.ADD;
3713      }
3714      else if (modTypeStr.equals("delete"))
3715      {
3716        modType = ModificationType.DELETE;
3717      }
3718      else if (modTypeStr.equals("replace"))
3719      {
3720        modType = ModificationType.REPLACE;
3721      }
3722      else if (modTypeStr.equals("increment"))
3723      {
3724        modType = ModificationType.INCREMENT;
3725      }
3726      else
3727      {
3728        throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3729                                     firstLineNumber),
3730                                firstLineNumber, true, ldifLines, null);
3731      }
3732
3733      String attributeName;
3734      int length = line.length();
3735      if (length == (colonPos+1))
3736      {
3737        // The colon was the last character on the line.  This is not
3738        // acceptable.
3739        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3740                                     firstLineNumber),
3741                                firstLineNumber, true, ldifLines, null);
3742      }
3743      else if (line.charAt(colonPos+1) == ':')
3744      {
3745        // Skip over any spaces leading up to the value, and then the rest of
3746        // the string is the base64-encoded attribute name.
3747        int pos = colonPos+2;
3748        while ((pos < length) && (line.charAt(pos) == ' '))
3749        {
3750          pos++;
3751        }
3752
3753        try
3754        {
3755          final byte[] dnBytes = Base64.decode(line.substring(pos));
3756          attributeName = StaticUtils.toUTF8String(dnBytes);
3757        }
3758        catch (final ParseException pe)
3759        {
3760          Debug.debugException(pe);
3761          throw new LDIFException(
3762               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3763                    firstLineNumber, pe.getMessage()),
3764               firstLineNumber, true, ldifLines, pe);
3765        }
3766        catch (final Exception e)
3767        {
3768          Debug.debugException(e);
3769          throw new LDIFException(
3770               ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3771                    firstLineNumber, e),
3772               firstLineNumber, true, ldifLines, e);
3773        }
3774      }
3775      else
3776      {
3777        // Skip over any spaces leading up to the value, and then the rest of
3778        // the string is the attribute name.
3779        int pos = colonPos+1;
3780        while ((pos < length) && (line.charAt(pos) == ' '))
3781        {
3782          pos++;
3783        }
3784
3785        attributeName = line.substring(pos);
3786      }
3787
3788      if (attributeName.isEmpty())
3789      {
3790        throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3791                                     firstLineNumber),
3792                                firstLineNumber, true, ldifLines, null);
3793      }
3794
3795
3796      // The next zero or more lines may be the set of attribute values.  Keep
3797      // reading until we reach the end of the iterator or until we find a line
3798      // with just a "-".
3799      final ArrayList<ASN1OctetString> valueList =
3800           new ArrayList<>(ldifLines.size());
3801      while (iterator.hasNext())
3802      {
3803        line = iterator.next();
3804        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3805        if (line.toString().equals("-"))
3806        {
3807          break;
3808        }
3809
3810        colonPos = line.indexOf(":");
3811        if (colonPos < 0)
3812        {
3813          throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3814                                  firstLineNumber, true, ldifLines, null);
3815        }
3816        else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3817        {
3818          // There are a couple of cases in which this might be acceptable:
3819          // - If the two names are logically equivalent, but have an alternate
3820          //   name (or OID) for the target attribute type, or if there are
3821          //   attribute options and the options are just in a different order.
3822          // - If this is the first value for the target attribute and the
3823          //   alternate name includes a "binary" option that the original
3824          //   attribute name did not have.  In this case, all subsequent values
3825          //   will also be required to have the binary option.
3826          final String alternateName = line.substring(0, colonPos);
3827
3828
3829          // Check to see if the base names are equivalent.
3830          boolean baseNameEquivalent = false;
3831          final String expectedBaseName = Attribute.getBaseName(attributeName);
3832          final String alternateBaseName = Attribute.getBaseName(alternateName);
3833          if (alternateBaseName.equalsIgnoreCase(expectedBaseName))
3834          {
3835            baseNameEquivalent = true;
3836          }
3837          else
3838          {
3839            if (schema != null)
3840            {
3841              final AttributeTypeDefinition expectedAT =
3842                   schema.getAttributeType(expectedBaseName);
3843              final AttributeTypeDefinition alternateAT =
3844                   schema.getAttributeType(alternateBaseName);
3845              if ((expectedAT != null) && (alternateAT != null) &&
3846                  expectedAT.equals(alternateAT))
3847              {
3848                baseNameEquivalent = true;
3849              }
3850            }
3851          }
3852
3853
3854          // Check to see if the attribute options are equivalent.
3855          final Set<String> expectedOptions =
3856               Attribute.getOptions(attributeName);
3857          final Set<String> lowerExpectedOptions = new HashSet<>(
3858               StaticUtils.computeMapCapacity(expectedOptions.size()));
3859          for (final String s : expectedOptions)
3860          {
3861            lowerExpectedOptions.add(StaticUtils.toLowerCase(s));
3862          }
3863
3864          final Set<String> alternateOptions =
3865               Attribute.getOptions(alternateName);
3866          final Set<String> lowerAlternateOptions = new HashSet<>(
3867               StaticUtils.computeMapCapacity(alternateOptions.size()));
3868          for (final String s : alternateOptions)
3869          {
3870            lowerAlternateOptions.add(StaticUtils.toLowerCase(s));
3871          }
3872
3873          final boolean optionsEquivalent =
3874               lowerAlternateOptions.equals(lowerExpectedOptions);
3875
3876
3877          if (baseNameEquivalent && optionsEquivalent)
3878          {
3879            // This is fine.  The two attribute descriptions are logically
3880            // equivalent.  We'll continue using the attribute description that
3881            // was provided first.
3882          }
3883          else if (valueList.isEmpty() && baseNameEquivalent &&
3884                   lowerAlternateOptions.remove("binary") &&
3885                   lowerAlternateOptions.equals(lowerExpectedOptions))
3886          {
3887            // This means that the provided value is the first value for the
3888            // attribute, and that the only significant difference is that the
3889            // provided attribute description included an unexpected "binary"
3890            // option.  We'll accept this, but will require any additional
3891            // values for this modification to also include the binary option,
3892            // and we'll use the binary option in the attribute that is
3893            // eventually created.
3894            attributeName = alternateName;
3895          }
3896          else
3897          {
3898            // This means that either the base names are different or the sets
3899            // of options are incompatible.  This is not acceptable.
3900            throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3901                                         firstLineNumber,
3902                                         line.substring(0, colonPos),
3903                                         attributeName),
3904                                    firstLineNumber, true, ldifLines, null);
3905          }
3906        }
3907
3908        length = line.length();
3909        final ASN1OctetString value;
3910        if (length == (colonPos+1))
3911        {
3912          // The colon was the last character on the line.  This is fine.
3913          value = new ASN1OctetString();
3914        }
3915        else if (line.charAt(colonPos+1) == ':')
3916        {
3917          // Skip over any spaces leading up to the value, and then the rest of
3918          // the string is the base64-encoded value.  This is unusual and
3919          // unnecessary, but is nevertheless acceptable.
3920          int pos = colonPos+2;
3921          while ((pos < length) && (line.charAt(pos) == ' '))
3922          {
3923            pos++;
3924          }
3925
3926          try
3927          {
3928            value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3929          }
3930          catch (final ParseException pe)
3931          {
3932            Debug.debugException(pe);
3933            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3934                 attributeName, firstLineNumber, pe.getMessage()),
3935                 firstLineNumber, true, ldifLines, pe);
3936          }
3937          catch (final Exception e)
3938          {
3939            Debug.debugException(e);
3940            throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3941                                         firstLineNumber, e),
3942                                    firstLineNumber, true, ldifLines, e);
3943          }
3944        }
3945        else if (line.charAt(colonPos+1) == '<')
3946        {
3947          // Skip over any spaces leading up to the value, and then the rest of
3948          // the string is a URL that indicates where to get the real content.
3949          // At the present time, we'll only support the file URLs.
3950          int pos = colonPos+2;
3951          while ((pos < length) && (line.charAt(pos) == ' '))
3952          {
3953            pos++;
3954          }
3955
3956          final String urlString = line.substring(pos);
3957          try
3958          {
3959            final byte[] urlBytes =
3960                 retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
3961            value = new ASN1OctetString(urlBytes);
3962          }
3963          catch (final Exception e)
3964          {
3965            Debug.debugException(e);
3966            throw new LDIFException(
3967                 ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3968                      firstLineNumber, e),
3969                 firstLineNumber, true, ldifLines, e);
3970          }
3971        }
3972        else
3973        {
3974          // Skip over any spaces leading up to the value, and then the rest of
3975          // the string is the value.
3976          int pos = colonPos+1;
3977          while ((pos < length) && (line.charAt(pos) == ' '))
3978          {
3979            pos++;
3980          }
3981
3982          value = new ASN1OctetString(line.substring(pos));
3983        }
3984
3985        valueList.add(value);
3986      }
3987
3988      final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3989      valueList.toArray(values);
3990
3991      // If it's an add modification type, then there must be at least one
3992      // value.
3993      if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3994          (values.length == 0))
3995      {
3996        throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3997                                     firstLineNumber),
3998                                firstLineNumber, true, ldifLines, null);
3999      }
4000
4001      // If it's an increment modification type, then there must be exactly one
4002      // value.
4003      if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
4004          (values.length != 1))
4005      {
4006        throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
4007                                     firstLineNumber, attributeName),
4008                                firstLineNumber, true, ldifLines, null);
4009      }
4010
4011      modList.add(new Modification(modType, attributeName, values));
4012    }
4013
4014    final Modification[] mods = new Modification[modList.size()];
4015    modList.toArray(mods);
4016    return mods;
4017  }
4018
4019
4020
4021  /**
4022   * Parses the data available through the provided iterator as the body of a
4023   * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
4024   * newsuperior lines).
4025   *
4026   * @param  ldifLines              The lines that comprise the LDIF
4027   *                                representation of the full record being
4028   *                                parsed.
4029   * @param  iterator               The iterator to use to access the modify DN
4030   *                                data.
4031   * @param  dn                     The current DN of the entry.
4032   * @param  controls               The set of controls to include in the change
4033   *                                record.
4034   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
4035   *                                encountering attribute values which are not
4036   *                                base64-encoded but contain trailing spaces.
4037   * @param  firstLineNumber        The line number for the start of the record.
4038   *
4039   * @return  The decoded modify DN change record.
4040   *
4041   * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
4042   *                         modify DN change record.
4043   */
4044  @NotNull()
4045  private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
4046               @NotNull final ArrayList<StringBuilder> ldifLines,
4047               @NotNull final Iterator<StringBuilder> iterator,
4048               @NotNull final String dn,
4049               @Nullable final List<Control> controls,
4050               @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
4051               final long firstLineNumber)
4052          throws LDIFException
4053  {
4054    // The next line must be the new RDN, and it must start with "newrdn:".
4055    StringBuilder line = iterator.next();
4056    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
4057    int colonPos = line.indexOf(":");
4058    if ((colonPos < 0) ||
4059        (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
4060    {
4061      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
4062                                   firstLineNumber),
4063                              firstLineNumber, true, ldifLines, null);
4064    }
4065
4066    final String newRDN;
4067    int length = line.length();
4068    if (length == (colonPos+1))
4069    {
4070      // The colon was the last character on the line.  This is not acceptable.
4071      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
4072                                   firstLineNumber),
4073                              firstLineNumber, true, ldifLines, null);
4074    }
4075    else if (line.charAt(colonPos+1) == ':')
4076    {
4077      // Skip over any spaces leading up to the value, and then the rest of the
4078      // string is the base64-encoded new RDN.
4079      int pos = colonPos+2;
4080      while ((pos < length) && (line.charAt(pos) == ' '))
4081      {
4082        pos++;
4083      }
4084
4085      try
4086      {
4087        final byte[] dnBytes = Base64.decode(line.substring(pos));
4088        newRDN = StaticUtils.toUTF8String(dnBytes);
4089      }
4090      catch (final ParseException pe)
4091      {
4092        Debug.debugException(pe);
4093        throw new LDIFException(
4094             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
4095                                                               pe.getMessage()),
4096             firstLineNumber, true, ldifLines, pe);
4097      }
4098      catch (final Exception e)
4099      {
4100        Debug.debugException(e);
4101        throw new LDIFException(
4102             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
4103                                                               e),
4104             firstLineNumber, true, ldifLines, e);
4105      }
4106    }
4107    else
4108    {
4109      // Skip over any spaces leading up to the value, and then the rest of the
4110      // string is the new RDN.
4111      int pos = colonPos+1;
4112      while ((pos < length) && (line.charAt(pos) == ' '))
4113      {
4114        pos++;
4115      }
4116
4117      newRDN = line.substring(pos);
4118    }
4119
4120    if (newRDN.isEmpty())
4121    {
4122      throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
4123                                   firstLineNumber),
4124                              firstLineNumber, true, ldifLines, null);
4125    }
4126
4127
4128    // The next line must be the deleteOldRDN flag, and it must start with
4129    // 'deleteoldrdn:'.
4130    if (! iterator.hasNext())
4131    {
4132      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
4133                                   firstLineNumber),
4134                              firstLineNumber, true, ldifLines, null);
4135    }
4136
4137    line = iterator.next();
4138    handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
4139    colonPos = line.indexOf(":");
4140    if ((colonPos < 0) ||
4141        (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
4142    {
4143      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
4144                                   firstLineNumber),
4145                              firstLineNumber, true, ldifLines, null);
4146    }
4147
4148    final String deleteOldRDNStr;
4149    length = line.length();
4150    if (length == (colonPos+1))
4151    {
4152      // The colon was the last character on the line.  This is not acceptable.
4153      throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
4154                                   firstLineNumber),
4155                              firstLineNumber, true, ldifLines, null);
4156    }
4157    else if (line.charAt(colonPos+1) == ':')
4158    {
4159      // Skip over any spaces leading up to the value, and then the rest of the
4160      // string is the base64-encoded value.  This is unusual and
4161      // unnecessary, but is nevertheless acceptable.
4162      int pos = colonPos+2;
4163      while ((pos < length) && (line.charAt(pos) == ' '))
4164      {
4165        pos++;
4166      }
4167
4168      try
4169      {
4170        final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
4171        deleteOldRDNStr = StaticUtils.toUTF8String(changeTypeBytes);
4172      }
4173      catch (final ParseException pe)
4174      {
4175        Debug.debugException(pe);
4176        throw new LDIFException(
4177             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
4178                  firstLineNumber, pe.getMessage()),
4179             firstLineNumber, true, ldifLines, pe);
4180      }
4181      catch (final Exception e)
4182      {
4183        Debug.debugException(e);
4184        throw new LDIFException(
4185             ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
4186                  firstLineNumber, e),
4187             firstLineNumber, true, ldifLines, e);
4188      }
4189    }
4190    else
4191    {
4192      // Skip over any spaces leading up to the value, and then the rest of the
4193      // string is the value.
4194      int pos = colonPos+1;
4195      while ((pos < length) && (line.charAt(pos) == ' '))
4196      {
4197        pos++;
4198      }
4199
4200      deleteOldRDNStr = line.substring(pos);
4201    }
4202
4203    final boolean deleteOldRDN;
4204    if (deleteOldRDNStr.equals("0"))
4205    {
4206      deleteOldRDN = false;
4207    }
4208    else if (deleteOldRDNStr.equals("1"))
4209    {
4210      deleteOldRDN = true;
4211    }
4212    else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
4213             deleteOldRDNStr.equalsIgnoreCase("no"))
4214    {
4215      // This is technically illegal, but we'll allow it.
4216      deleteOldRDN = false;
4217    }
4218    else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
4219             deleteOldRDNStr.equalsIgnoreCase("yes"))
4220    {
4221      // This is also technically illegal, but we'll allow it.
4222      deleteOldRDN = false;
4223    }
4224    else
4225    {
4226      throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
4227                                   deleteOldRDNStr, firstLineNumber),
4228                              firstLineNumber, true, ldifLines, null);
4229    }
4230
4231
4232    // If there is another line, then it must be the new superior DN and it must
4233    // start with "newsuperior:".  If this is absent, then it's fine.
4234    final String newSuperiorDN;
4235    if (iterator.hasNext())
4236    {
4237      line = iterator.next();
4238      handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
4239      colonPos = line.indexOf(":");
4240      if ((colonPos < 0) ||
4241          (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
4242      {
4243        throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
4244                                     firstLineNumber),
4245                                firstLineNumber, true, ldifLines, null);
4246      }
4247
4248      length = line.length();
4249      if (length == (colonPos+1))
4250      {
4251        // The colon was the last character on the line.  This is fine.
4252        newSuperiorDN = "";
4253      }
4254      else if (line.charAt(colonPos+1) == ':')
4255      {
4256        // Skip over any spaces leading up to the value, and then the rest of
4257        // the string is the base64-encoded new superior DN.
4258        int pos = colonPos+2;
4259        while ((pos < length) && (line.charAt(pos) == ' '))
4260        {
4261          pos++;
4262        }
4263
4264        try
4265        {
4266          final byte[] dnBytes = Base64.decode(line.substring(pos));
4267          newSuperiorDN = StaticUtils.toUTF8String(dnBytes);
4268        }
4269        catch (final ParseException pe)
4270        {
4271          Debug.debugException(pe);
4272          throw new LDIFException(
4273               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4274                    firstLineNumber, pe.getMessage()),
4275               firstLineNumber, true, ldifLines, pe);
4276        }
4277        catch (final Exception e)
4278        {
4279          Debug.debugException(e);
4280          throw new LDIFException(
4281               ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4282                    firstLineNumber, e),
4283               firstLineNumber, true, ldifLines, e);
4284        }
4285      }
4286      else
4287      {
4288        // Skip over any spaces leading up to the value, and then the rest of
4289        // the string is the new superior DN.
4290        int pos = colonPos+1;
4291        while ((pos < length) && (line.charAt(pos) == ' '))
4292        {
4293          pos++;
4294        }
4295
4296        newSuperiorDN = line.substring(pos);
4297      }
4298    }
4299    else
4300    {
4301      newSuperiorDN = null;
4302    }
4303
4304
4305    // There must not be any more lines.
4306    if (iterator.hasNext())
4307    {
4308      throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
4309                              firstLineNumber, true, ldifLines, null);
4310    }
4311
4312    return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
4313         newSuperiorDN, controls);
4314  }
4315
4316
4317
4318  /**
4319   * Examines the line contained in the provided buffer to determine whether it
4320   * may contain one or more illegal trailing spaces.  If it does, then those
4321   * spaces will either be stripped out or an exception will be thrown to
4322   * indicate that they are illegal.
4323   *
4324   * @param  buffer                 The buffer to be examined.
4325   * @param  dn                     The DN of the LDIF record being parsed.  It
4326   *                                may be {@code null} if the DN is not yet
4327   *                                known (e.g., because the provided line is
4328   *                                expected to contain that DN).
4329   * @param  firstLineNumber        The approximate line number in the LDIF
4330   *                                source on which the LDIF record begins.
4331   * @param  trailingSpaceBehavior  The behavior that should be exhibited when
4332   *                                encountering attribute values which are not
4333   *                                base64-encoded but contain trailing spaces.
4334   *
4335   * @throws  LDIFException  If the line contained in the provided buffer ends
4336   *                         with one or more illegal trailing spaces and
4337   *                         {@code stripTrailingSpaces} was provided with a
4338   *                         value of {@code false}.
4339   */
4340  private static void handleTrailingSpaces(@NotNull final StringBuilder buffer,
4341               @Nullable final String dn, final long firstLineNumber,
4342               @NotNull final TrailingSpaceBehavior trailingSpaceBehavior)
4343          throws LDIFException
4344  {
4345    int pos = buffer.length() - 1;
4346    boolean trailingFound = false;
4347    while ((pos >= 0) && (buffer.charAt(pos) == ' '))
4348    {
4349      trailingFound = true;
4350      pos--;
4351    }
4352
4353    if (trailingFound && (buffer.charAt(pos) != ':'))
4354    {
4355      switch (trailingSpaceBehavior)
4356      {
4357        case STRIP:
4358          buffer.setLength(pos+1);
4359          break;
4360
4361        case REJECT:
4362          if (dn == null)
4363          {
4364            throw new LDIFException(
4365                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
4366                      buffer.toString()),
4367                 firstLineNumber, true);
4368          }
4369          else
4370          {
4371            throw new LDIFException(
4372                 ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
4373                      firstLineNumber, buffer.toString()),
4374                 firstLineNumber, true);
4375          }
4376
4377        case RETAIN:
4378        default:
4379          // No action will be taken.
4380          break;
4381      }
4382    }
4383  }
4384
4385
4386
4387  /**
4388   * This represents an unparsed LDIFRecord.  It stores the line number of the
4389   * first line of the record and each line of the record.
4390   */
4391  private static final class UnparsedLDIFRecord
4392  {
4393    // The list of lines contained in this record.
4394    @Nullable private final ArrayList<StringBuilder> lineList;
4395
4396    // The first line number for the LDIF record.
4397    private final long firstLineNumber;
4398
4399    // The exception thrown while reading from the input.
4400    @Nullable private final Exception failureCause;
4401
4402    // Indicates whether the end of the input has been reached.
4403    private final boolean isEOF;
4404
4405    // The duplicate value behavior to use when parsing the record.
4406    @NotNull private final DuplicateValueBehavior duplicateValueBehavior;
4407
4408    // The schema to use when parsing the record.
4409    @Nullable private final Schema schema;
4410
4411    // The trailing space behavior to use when parsing the record.
4412    @NotNull private final TrailingSpaceBehavior trailingSpaceBehavior;
4413
4414
4415
4416    /**
4417     * Creates a new instance of this record.
4418     *
4419     * @param  lineList                The lines that comprise the LDIF record.
4420     * @param  duplicateValueBehavior  The behavior to exhibit if the entry
4421     *                                 contains duplicate attribute values.
4422     * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
4423     *                                 encountering trailing spaces in
4424     *                                 non-base64-encoded attribute values.
4425     * @param  schema                  The schema to use when parsing, if
4426     *                                 applicable.
4427     * @param  firstLineNumber         The first line number of the LDIF record.
4428     */
4429    private UnparsedLDIFRecord(@NotNull final ArrayList<StringBuilder> lineList,
4430                 @NotNull final DuplicateValueBehavior duplicateValueBehavior,
4431                 @NotNull final TrailingSpaceBehavior trailingSpaceBehavior,
4432                 @Nullable final Schema schema, final long firstLineNumber)
4433    {
4434      this.lineList               = lineList;
4435      this.firstLineNumber        = firstLineNumber;
4436      this.duplicateValueBehavior = duplicateValueBehavior;
4437      this.trailingSpaceBehavior  = trailingSpaceBehavior;
4438      this.schema                 = schema;
4439
4440      failureCause = null;
4441      isEOF =
4442           (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
4443    }
4444
4445
4446
4447    /**
4448     * Creates a new instance of this record.
4449     *
4450     * @param failureCause  The Exception thrown when reading from the input.
4451     */
4452    private UnparsedLDIFRecord(@NotNull final Exception failureCause)
4453    {
4454      this.failureCause = failureCause;
4455
4456      lineList               = null;
4457      firstLineNumber        = 0;
4458      duplicateValueBehavior = DuplicateValueBehavior.REJECT;
4459      trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
4460      schema                 = null;
4461      isEOF                  = false;
4462    }
4463
4464
4465
4466    /**
4467     * Return the lines that comprise the LDIF record.
4468     *
4469     * @return  The lines that comprise the LDIF record, or {@code null} if this
4470     *          is a failure record.
4471     */
4472    @Nullable()
4473    private ArrayList<StringBuilder> getLineList()
4474    {
4475      return lineList;
4476    }
4477
4478
4479
4480    /**
4481     * Retrieves the behavior to exhibit when encountering duplicate attribute
4482     * values.
4483     *
4484     * @return  The behavior to exhibit when encountering duplicate attribute
4485     *          values.
4486     */
4487    @NotNull()
4488    private DuplicateValueBehavior getDuplicateValueBehavior()
4489    {
4490      return duplicateValueBehavior;
4491    }
4492
4493
4494
4495    /**
4496     * Retrieves the behavior that should be exhibited when encountering
4497     * attribute values which are not base64-encoded but contain trailing
4498     * spaces.  The LDIF specification strongly recommends that any value which
4499     * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
4500     * LDIF parser may be configured to automatically strip these spaces, to
4501     * preserve them, or to reject any entry or change record containing them.
4502     *
4503     * @return  The behavior that should be exhibited when encountering
4504     *          attribute values which are not base64-encoded but contain
4505     *          trailing spaces.
4506     */
4507    @NotNull()
4508    private TrailingSpaceBehavior getTrailingSpaceBehavior()
4509    {
4510      return trailingSpaceBehavior;
4511    }
4512
4513
4514
4515    /**
4516     * Retrieves the schema that should be used when parsing the record, if
4517     * applicable.
4518     *
4519     * @return  The schema that should be used when parsing the record, or
4520     *          {@code null} if none should be used.
4521     */
4522    @Nullable()
4523    private Schema getSchema()
4524    {
4525      return schema;
4526    }
4527
4528
4529
4530    /**
4531     * Return the first line number of the LDIF record.
4532     *
4533     * @return  The first line number of the LDIF record.
4534     */
4535    private long getFirstLineNumber()
4536    {
4537      return firstLineNumber;
4538    }
4539
4540
4541
4542    /**
4543     * Return {@code true} iff the end of the input was reached.
4544     *
4545     * @return  {@code true} iff the end of the input was reached.
4546     */
4547    private boolean isEOF()
4548    {
4549      return isEOF;
4550    }
4551
4552
4553
4554    /**
4555     * Returns the reason that reading the record lines failed.  This normally
4556     * is only non-null if something bad happened to the input stream (like
4557     * a disk read error).
4558     *
4559     * @return  The reason that reading the record lines failed.
4560     */
4561    @Nullable()
4562    private Exception getFailureCause()
4563    {
4564      return failureCause;
4565    }
4566  }
4567
4568
4569  /**
4570   * When processing in asynchronous mode, this thread is responsible for
4571   * reading the raw unparsed records from the input and submitting them for
4572   * processing.
4573   */
4574  private final class LineReaderThread
4575       extends Thread
4576  {
4577    /**
4578     * Creates a new instance of this thread.
4579     */
4580    private LineReaderThread()
4581    {
4582      super("Asynchronous LDIF line reader");
4583      setDaemon(true);
4584    }
4585
4586
4587
4588    /**
4589     * Reads raw, unparsed records from the input and submits them for
4590     * processing until the input is finished or closed.
4591     */
4592    @Override()
4593    public void run()
4594    {
4595      try
4596      {
4597        boolean stopProcessing = false;
4598        while (!stopProcessing)
4599        {
4600          UnparsedLDIFRecord unparsedRecord;
4601          try
4602          {
4603            unparsedRecord = readUnparsedRecord();
4604          }
4605          catch (final IOException e)
4606          {
4607            Debug.debugException(e);
4608            unparsedRecord = new UnparsedLDIFRecord(e);
4609            stopProcessing = true;
4610          }
4611          catch (final Exception e)
4612          {
4613            Debug.debugException(e);
4614            unparsedRecord = new UnparsedLDIFRecord(e);
4615          }
4616
4617          try
4618          {
4619            asyncParser.submit(unparsedRecord);
4620          }
4621          catch (final InterruptedException e)
4622          {
4623            Debug.debugException(e);
4624            // If this thread is interrupted, then someone wants us to stop
4625            // processing, so that's what we'll do.
4626            Thread.currentThread().interrupt();
4627            stopProcessing = true;
4628          }
4629
4630          if ((unparsedRecord == null) || unparsedRecord.isEOF())
4631          {
4632            stopProcessing = true;
4633          }
4634        }
4635      }
4636      finally
4637      {
4638        try
4639        {
4640          asyncParser.shutdown();
4641        }
4642        catch (final InterruptedException e)
4643        {
4644          Debug.debugException(e);
4645          Thread.currentThread().interrupt();
4646        }
4647        finally
4648        {
4649          asyncParsingComplete.set(true);
4650        }
4651      }
4652    }
4653  }
4654
4655
4656
4657  /**
4658   * Used to parse Records asynchronously.
4659   */
4660  private final class RecordParser implements Processor<UnparsedLDIFRecord,
4661                                                        LDIFRecord>
4662  {
4663    /**
4664     * {@inheritDoc}
4665     */
4666    @Override()
4667    public LDIFRecord process(@NotNull final UnparsedLDIFRecord input)
4668           throws LDIFException
4669    {
4670      LDIFRecord record = decodeRecord(input, relativeBasePath, schema);
4671
4672      if ((record instanceof Entry) && (entryTranslator != null))
4673      {
4674        record = entryTranslator.translate((Entry) record,
4675             input.getFirstLineNumber());
4676
4677        if (record == null)
4678        {
4679          record = SKIP_ENTRY;
4680        }
4681      }
4682      if ((record instanceof LDIFChangeRecord) &&
4683          (changeRecordTranslator != null))
4684      {
4685        record = changeRecordTranslator.translate((LDIFChangeRecord) record,
4686             input.getFirstLineNumber());
4687
4688        if (record == null)
4689        {
4690          record = SKIP_ENTRY;
4691        }
4692      }
4693      return record;
4694    }
4695  }
4696}