001    /*
002     * Copyright 2007-2015 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2015 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldif;
022    
023    
024    
025    import java.io.BufferedReader;
026    import java.io.BufferedWriter;
027    import java.io.File;
028    import java.io.FileInputStream;
029    import java.io.FileWriter;
030    import java.io.InputStream;
031    import java.io.InputStreamReader;
032    import java.io.IOException;
033    import java.text.ParseException;
034    import java.util.ArrayList;
035    import java.util.Collection;
036    import java.util.Iterator;
037    import java.util.LinkedHashMap;
038    import java.util.List;
039    import java.util.concurrent.BlockingQueue;
040    import java.util.concurrent.ArrayBlockingQueue;
041    import java.util.concurrent.TimeUnit;
042    import java.util.concurrent.atomic.AtomicBoolean;
043    import java.nio.charset.Charset;
044    
045    import com.unboundid.asn1.ASN1OctetString;
046    import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
047    import com.unboundid.ldap.matchingrules.MatchingRule;
048    import com.unboundid.ldap.sdk.Attribute;
049    import com.unboundid.ldap.sdk.Control;
050    import com.unboundid.ldap.sdk.Entry;
051    import com.unboundid.ldap.sdk.Modification;
052    import com.unboundid.ldap.sdk.ModificationType;
053    import com.unboundid.ldap.sdk.LDAPException;
054    import com.unboundid.ldap.sdk.schema.Schema;
055    import com.unboundid.util.AggregateInputStream;
056    import com.unboundid.util.Base64;
057    import com.unboundid.util.LDAPSDKThreadFactory;
058    import com.unboundid.util.ThreadSafety;
059    import com.unboundid.util.ThreadSafetyLevel;
060    import com.unboundid.util.parallel.AsynchronousParallelProcessor;
061    import com.unboundid.util.parallel.Result;
062    import com.unboundid.util.parallel.ParallelProcessor;
063    import com.unboundid.util.parallel.Processor;
064    
065    import static com.unboundid.ldif.LDIFMessages.*;
066    import static com.unboundid.util.Debug.*;
067    import static com.unboundid.util.StaticUtils.*;
068    import static com.unboundid.util.Validator.*;
069    
070    /**
071     * This class provides an LDIF reader, which can be used to read and decode
072     * entries and change records from a data source using the LDAP Data Interchange
073     * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
074     * <BR>
075     * This class is not synchronized.  If multiple threads read from the
076     * LDIFReader, they must be synchronized externally.
077     * <BR><BR>
078     * <H2>Example</H2>
079     * The following example iterates through all entries contained in an LDIF file
080     * and attempts to add them to a directory server:
081     * <PRE>
082     * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
083     *
084     * int entriesRead = 0;
085     * int entriesAdded = 0;
086     * int errorsEncountered = 0;
087     * while (true)
088     * {
089     *   Entry entry;
090     *   try
091     *   {
092     *     entry = ldifReader.readEntry();
093     *     if (entry == null)
094     *     {
095     *       // All entries have been read.
096     *       break;
097     *     }
098     *
099     *     entriesRead++;
100     *   }
101     *   catch (LDIFException le)
102     *   {
103     *     errorsEncountered++;
104     *     if (le.mayContinueReading())
105     *     {
106     *       // A recoverable error occurred while attempting to read a change
107     *       // record, at or near line number le.getLineNumber()
108     *       // The entry will be skipped, but we'll try to keep reading from the
109     *       // LDIF file.
110     *       continue;
111     *     }
112     *     else
113     *     {
114     *       // An unrecoverable error occurred while attempting to read an entry
115     *       // at or near line number le.getLineNumber()
116     *       // No further LDIF processing will be performed.
117     *       break;
118     *     }
119     *   }
120     *   catch (IOException ioe)
121     *   {
122     *     // An I/O error occurred while attempting to read from the LDIF file.
123     *     // No further LDIF processing will be performed.
124     *     errorsEncountered++;
125     *     break;
126     *   }
127     *
128     *   LDAPResult addResult;
129     *   try
130     *   {
131     *     addResult = connection.add(entry);
132     *     // If we got here, then the change should have been processed
133     *     // successfully.
134     *     entriesAdded++;
135     *   }
136     *   catch (LDAPException le)
137     *   {
138     *     // If we got here, then the change attempt failed.
139     *     addResult = le.toLDAPResult();
140     *     errorsEncountered++;
141     *   }
142     * }
143     *
144     * ldifReader.close();
145     * </PRE>
146     */
147    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
148    public final class LDIFReader
149    {
150      /**
151       * The default buffer size (128KB) that will be used when reading from the
152       * data source.
153       */
154      public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
155    
156    
157    
158      /*
159       * When processing asynchronously, this determines how many of the allocated
160       * worker threads are used to parse each batch of read entries.
161       */
162      private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
163    
164    
165    
166      /**
167       * When processing asynchronously, this specifies the size of the pending and
168       * completed queues.
169       */
170      private static final int ASYNC_QUEUE_SIZE = 500;
171    
172    
173    
174      /**
175       * Special entry used internally to signal that the LDIFReaderEntryTranslator
176       * has signalled that a read Entry should be skipped by returning null,
177       * which normally implies EOF.
178       */
179      private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
180    
181    
182    
183      /**
184       * The default base path that will be prepended to relative paths.  It will
185       * end with a trailing slash.
186       */
187      private static final String DEFAULT_RELATIVE_BASE_PATH;
188      static
189      {
190        final File currentDir;
191        String currentDirString = System.getProperty("user.dir");
192        if (currentDirString == null)
193        {
194          currentDir = new File(".");
195        }
196        else
197        {
198          currentDir = new File(currentDirString);
199        }
200    
201        final String currentDirAbsolutePath = currentDir.getAbsolutePath();
202        if (currentDirAbsolutePath.endsWith(File.separator))
203        {
204          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
205        }
206        else
207        {
208          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
209        }
210      }
211    
212    
213    
214      // The buffered reader that will be used to read LDIF data.
215      private final BufferedReader reader;
216    
217      // The behavior that should be exhibited when encountering duplicate attribute
218      // values.
219      private volatile DuplicateValueBehavior duplicateValueBehavior;
220    
221      // A line number counter.
222      private long lineNumberCounter = 0;
223    
224      private final LDIFReaderEntryTranslator entryTranslator;
225    
226      // The schema that will be used when processing, if applicable.
227      private Schema schema;
228    
229      // Specifies the base path that will be prepended to relative paths for file
230      // URLs.
231      private volatile String relativeBasePath;
232    
233      // The behavior that should be exhibited with regard to illegal trailing
234      // spaces in attribute values.
235      private volatile TrailingSpaceBehavior trailingSpaceBehavior;
236    
237      // True iff we are processing asynchronously.
238      private final boolean isAsync;
239    
240      //
241      // The following only apply to asynchronous processing.
242      //
243    
244      // Parses entries asynchronously.
245      private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord>
246           asyncParser;
247    
248      // Set to true when the end of the input is reached.
249      private final AtomicBoolean asyncParsingComplete;
250    
251      // The records that have been read and parsed.
252      private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>>
253           asyncParsedRecords;
254    
255    
256    
257      /**
258       * Creates a new LDIF reader that will read data from the specified file.
259       *
260       * @param  path  The path to the file from which the data is to be read.  It
261       *               must not be {@code null}.
262       *
263       * @throws  IOException  If a problem occurs while opening the file for
264       *                       reading.
265       */
266      public LDIFReader(final String path)
267             throws IOException
268      {
269        this(new FileInputStream(path));
270      }
271    
272    
273    
274      /**
275       * Creates a new LDIF reader that will read data from the specified file
276       * and parses the LDIF records asynchronously using the specified number of
277       * threads.
278       *
279       * @param  path  The path to the file from which the data is to be read.  It
280       *               must not be {@code null}.
281       * @param  numParseThreads  If this value is greater than zero, then the
282       *                          specified number of threads will be used to
283       *                          asynchronously read and parse the LDIF file.
284       *
285       * @throws  IOException  If a problem occurs while opening the file for
286       *                       reading.
287       *
288       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
289       *      constructor for more details about asynchronous processing.
290       */
291      public LDIFReader(final String path, final int numParseThreads)
292             throws IOException
293      {
294        this(new FileInputStream(path), numParseThreads);
295      }
296    
297    
298    
299      /**
300       * Creates a new LDIF reader that will read data from the specified file.
301       *
302       * @param  file  The file from which the data is to be read.  It must not be
303       *               {@code null}.
304       *
305       * @throws  IOException  If a problem occurs while opening the file for
306       *                       reading.
307       */
308      public LDIFReader(final File file)
309             throws IOException
310      {
311        this(new FileInputStream(file));
312      }
313    
314    
315    
316      /**
317       * Creates a new LDIF reader that will read data from the specified file
318       * and optionally parses the LDIF records asynchronously using the specified
319       * number of threads.
320       *
321       * @param  file             The file from which the data is to be read.  It
322       *                          must not be {@code null}.
323       * @param  numParseThreads  If this value is greater than zero, then the
324       *                          specified number of threads will be used to
325       *                          asynchronously read and parse the LDIF file.
326       *
327       * @throws  IOException  If a problem occurs while opening the file for
328       *                       reading.
329       */
330      public LDIFReader(final File file, final int numParseThreads)
331             throws IOException
332      {
333        this(new FileInputStream(file), numParseThreads);
334      }
335    
336    
337    
338      /**
339       * Creates a new LDIF reader that will read data from the specified files in
340       * the order in which they are provided and optionally parses the LDIF records
341       * asynchronously using the specified number of threads.
342       *
343       * @param  files            The files from which the data is to be read.  It
344       *                          must not be {@code null} or empty.
345       * @param  numParseThreads  If this value is greater than zero, then the
346       *                          specified number of threads will be used to
347       *                          asynchronously read and parse the LDIF file.
348       * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
349       *                          before they are returned.  This is normally
350       *                          {@code null}, which causes entries to be returned
351       *                          unaltered. This is particularly useful when
352       *                          parsing the input file in parallel because the
353       *                          entry translation is also done in parallel.
354       *
355       * @throws  IOException  If a problem occurs while opening the file for
356       *                       reading.
357       */
358      public LDIFReader(final File[] files, final int numParseThreads,
359                        final LDIFReaderEntryTranslator entryTranslator)
360             throws IOException
361      {
362        this(createAggregateInputStream(files), numParseThreads, entryTranslator);
363      }
364    
365    
366    
367      /**
368       * Creates a new aggregate input stream that will read data from the specified
369       * files.  If there are multiple files, then a "padding" file will be inserted
370       * between them to ensure that there is at least one blank line between the
371       * end of one file and the beginning of another.
372       *
373       * @param  files  The files from which the data is to be read.  It must not be
374       *                {@code null} or empty.
375       *
376       * @return  The input stream to use to read data from the provided files.
377       *
378       * @throws  IOException  If a problem is encountered while attempting to
379       *                       create the input stream.
380       */
381      private static InputStream createAggregateInputStream(final File... files)
382              throws IOException
383      {
384        if (files.length == 0)
385        {
386          throw new IOException(ERR_READ_NO_LDIF_FILES.get());
387        }
388        else if (files.length == 1)
389        {
390          return new FileInputStream(files[0]);
391        }
392        else
393        {
394          final File spacerFile =
395               File.createTempFile("ldif-reader-spacer", ".ldif");
396          spacerFile.deleteOnExit();
397    
398          final BufferedWriter spacerWriter =
399               new BufferedWriter(new FileWriter(spacerFile));
400          try
401          {
402            spacerWriter.newLine();
403            spacerWriter.newLine();
404          }
405          finally
406          {
407            spacerWriter.close();
408          }
409    
410          final File[] returnArray = new File[(files.length * 2) - 1];
411          returnArray[0] = files[0];
412    
413          int pos = 1;
414          for (int i=1; i < files.length; i++)
415          {
416            returnArray[pos++] = spacerFile;
417            returnArray[pos++] = files[i];
418          }
419    
420          return new AggregateInputStream(returnArray);
421        }
422      }
423    
424    
425    
426      /**
427       * Creates a new LDIF reader that will read data from the provided input
428       * stream.
429       *
430       * @param  inputStream  The input stream from which the data is to be read.
431       *                      It must not be {@code null}.
432       */
433      public LDIFReader(final InputStream inputStream)
434      {
435        this(inputStream, 0);
436      }
437    
438    
439    
440      /**
441       * Creates a new LDIF reader that will read data from the specified stream
442       * and parses the LDIF records asynchronously using the specified number of
443       * threads.
444       *
445       * @param  inputStream  The input stream from which the data is to be read.
446       *                      It must not be {@code null}.
447       * @param  numParseThreads  If this value is greater than zero, then the
448       *                          specified number of threads will be used to
449       *                          asynchronously read and parse the LDIF file.
450       *
451       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
452       *      constructor for more details about asynchronous processing.
453       */
454      public LDIFReader(final InputStream inputStream, final int numParseThreads)
455      {
456        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
457        this(new BufferedReader(new InputStreamReader(inputStream,
458                                                      Charset.forName("UTF-8")),
459                                DEFAULT_BUFFER_SIZE),
460             numParseThreads);
461      }
462    
463    
464    
465      /**
466       * Creates a new LDIF reader that will read data from the specified stream
467       * and parses the LDIF records asynchronously using the specified number of
468       * threads.
469       *
470       * @param  inputStream  The input stream from which the data is to be read.
471       *                      It must not be {@code null}.
472       * @param  numParseThreads  If this value is greater than zero, then the
473       *                          specified number of threads will be used to
474       *                          asynchronously read and parse the LDIF file.
475       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
476       *                         entries before they are returned.  This is normally
477       *                         {@code null}, which causes entries to be returned
478       *                         unaltered. This is particularly useful when parsing
479       *                         the input file in parallel because the entry
480       *                         translation is also done in parallel.
481       *
482       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
483       *      constructor for more details about asynchronous processing.
484       */
485      public LDIFReader(final InputStream inputStream, final int numParseThreads,
486                        final LDIFReaderEntryTranslator entryTranslator)
487      {
488        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
489        this(new BufferedReader(new InputStreamReader(inputStream,
490                                                      Charset.forName("UTF-8")),
491                                DEFAULT_BUFFER_SIZE),
492             numParseThreads, entryTranslator);
493      }
494    
495    
496    
497      /**
498       * Creates a new LDIF reader that will use the provided buffered reader to
499       * read the LDIF data.  The encoding of the underlying Reader must be set to
500       * "UTF-8" as required by RFC 2849.
501       *
502       * @param  reader  The buffered reader that will be used to read the LDIF
503       *                 data.  It must not be {@code null}.
504       */
505      public LDIFReader(final BufferedReader reader)
506      {
507        this(reader, 0);
508      }
509    
510    
511    
512      /**
513       * Creates a new LDIF reader that will read data from the specified buffered
514       * reader and parses the LDIF records asynchronously using the specified
515       * number of threads.  The encoding of the underlying Reader must be set to
516       * "UTF-8" as required by RFC 2849.
517       *
518       * @param reader The buffered reader that will be used to read the LDIF data.
519       *               It must not be {@code null}.
520       * @param  numParseThreads  If this value is greater than zero, then the
521       *                          specified number of threads will be used to
522       *                          asynchronously read and parse the LDIF file.
523       *
524       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
525       *      constructor for more details about asynchronous processing.
526       */
527      public LDIFReader(final BufferedReader reader, final int numParseThreads)
528      {
529        this(reader, numParseThreads, null);
530      }
531    
532    
533    
534      /**
535       * Creates a new LDIF reader that will read data from the specified buffered
536       * reader and parses the LDIF records asynchronously using the specified
537       * number of threads.  The encoding of the underlying Reader must be set to
538       * "UTF-8" as required by RFC 2849.
539       *
540       * @param reader The buffered reader that will be used to read the LDIF data.
541       *               It must not be {@code null}.
542       * @param  numParseThreads  If this value is greater than zero, then the
543       *                          specified number of threads will be used to
544       *                          asynchronously read and parse the LDIF file.
545       *                          This should only be set to greater than zero when
546       *                          performance analysis has demonstrated that reading
547       *                          and parsing the LDIF is a bottleneck.  The default
548       *                          synchronous processing is normally fast enough.
549       *                          There is little benefit in passing in a value
550       *                          greater than four (unless there is an
551       *                          LDIFReaderEntryTranslator that does time-consuming
552       *                          processing).  A value of zero implies the
553       *                          default behavior of reading and parsing LDIF
554       *                          records synchronously when one of the read
555       *                          methods is called.
556       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
557       *                         entries before they are returned.  This is normally
558       *                         {@code null}, which causes entries to be returned
559       *                         unaltered. This is particularly useful when parsing
560       *                         the input file in parallel because the entry
561       *                         translation is also done in parallel.
562       */
563      public LDIFReader(final BufferedReader reader,
564                        final int numParseThreads,
565                        final LDIFReaderEntryTranslator entryTranslator)
566      {
567        ensureNotNull(reader);
568        ensureTrue(numParseThreads >= 0,
569                   "LDIFReader.numParseThreads must not be negative.");
570    
571        this.reader = reader;
572        this.entryTranslator = entryTranslator;
573    
574        duplicateValueBehavior = DuplicateValueBehavior.STRIP;
575        trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
576    
577        relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
578    
579        if (numParseThreads == 0)
580        {
581          isAsync = false;
582          asyncParser = null;
583          asyncParsingComplete = null;
584          asyncParsedRecords = null;
585        }
586        else
587        {
588          isAsync = true;
589          asyncParsingComplete = new AtomicBoolean(false);
590    
591          // Decodes entries in parallel.
592          final LDAPSDKThreadFactory threadFactory =
593               new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
594          final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
595               new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>(
596                    new RecordParser(), threadFactory, numParseThreads,
597                    ASYNC_MIN_PER_PARSING_THREAD);
598    
599          final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
600               ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE);
601    
602          // The output queue must be a little more than twice as big as the input
603          // queue to more easily handle being shutdown in the middle of processing
604          // when the queues are full and threads are blocked.
605          asyncParsedRecords = new ArrayBlockingQueue
606               <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100);
607    
608          asyncParser = new AsynchronousParallelProcessor
609               <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser,
610                                                asyncParsedRecords);
611    
612          final LineReaderThread lineReaderThread = new LineReaderThread();
613          lineReaderThread.start();
614        }
615      }
616    
617    
618    
619      /**
620       * Reads entries from the LDIF file with the specified path and returns them
621       * as a {@code List}.  This is a convenience method that should only be used
622       * for data sets that are small enough so that running out of memory isn't a
623       * concern.
624       *
625       * @param  path  The path to the LDIF file containing the entries to be read.
626       *
627       * @return  A list of the entries read from the given LDIF file.
628       *
629       * @throws  IOException  If a problem occurs while attempting to read data
630       *                       from the specified file.
631       *
632       * @throws  LDIFException  If a problem is encountered while attempting to
633       *                         decode data read as LDIF.
634       */
635      public static List<Entry> readEntries(final String path)
636             throws IOException, LDIFException
637      {
638        return readEntries(new LDIFReader(path));
639      }
640    
641    
642    
643      /**
644       * Reads entries from the specified LDIF file and returns them as a
645       * {@code List}.  This is a convenience method that should only be used for
646       * data sets that are small enough so that running out of memory isn't a
647       * concern.
648       *
649       * @param  file  A reference to the LDIF file containing the entries to be
650       *               read.
651       *
652       * @return  A list of the entries read from the given LDIF file.
653       *
654       * @throws  IOException  If a problem occurs while attempting to read data
655       *                       from the specified file.
656       *
657       * @throws  LDIFException  If a problem is encountered while attempting to
658       *                         decode data read as LDIF.
659       */
660      public static List<Entry> readEntries(final File file)
661             throws IOException, LDIFException
662      {
663        return readEntries(new LDIFReader(file));
664      }
665    
666    
667    
668      /**
669       * Reads and decodes LDIF entries from the provided input stream and
670       * returns them as a {@code List}.  This is a convenience method that should
671       * only be used for data sets that are small enough so that running out of
672       * memory isn't a concern.
673       *
674       * @param  inputStream  The input stream from which the entries should be
675       *                      read.  The input stream will be closed before
676       *                      returning.
677       *
678       * @return  A list of the entries read from the given input stream.
679       *
680       * @throws  IOException  If a problem occurs while attempting to read data
681       *                       from the input stream.
682       *
683       * @throws  LDIFException  If a problem is encountered while attempting to
684       *                         decode data read as LDIF.
685       */
686      public static List<Entry> readEntries(final InputStream inputStream)
687             throws IOException, LDIFException
688      {
689        return readEntries(new LDIFReader(inputStream));
690      }
691    
692    
693    
694      /**
695       * Reads entries from the provided LDIF reader and returns them as a list.
696       *
697       * @param  reader  The reader from which the entries should be read.  It will
698       *                 be closed before returning.
699       *
700       * @return  A list of the entries read from the provided reader.
701       *
702       * @throws  IOException  If a problem was encountered while attempting to read
703       *                       data from the LDIF data source.
704       *
705       * @throws  LDIFException  If a problem is encountered while attempting to
706       *                         decode data read as LDIF.
707       */
708      private static List<Entry> readEntries(final LDIFReader reader)
709              throws IOException, LDIFException
710      {
711        try
712        {
713          final ArrayList<Entry> entries = new ArrayList<Entry>(10);
714          while (true)
715          {
716            final Entry e = reader.readEntry();
717            if (e == null)
718            {
719              break;
720            }
721    
722            entries.add(e);
723          }
724    
725          return entries;
726        }
727        finally
728        {
729          reader.close();
730        }
731      }
732    
733    
734    
735      /**
736       * Closes this LDIF reader and the underlying LDIF source.
737       *
738       * @throws  IOException  If a problem occurs while closing the underlying LDIF
739       *                       source.
740       */
741      public void close()
742             throws IOException
743      {
744        reader.close();
745    
746        if (isAsync())
747        {
748          // Closing the reader will trigger the LineReaderThread to complete, but
749          // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
750          // this, we clear out the completed output queue, which is larger than
751          // the input queue, so the LineReaderThread will stop reading and
752          // shutdown the asyncParser.
753          asyncParsedRecords.clear();
754        }
755      }
756    
757    
758    
759      /**
760       * Indicates whether to ignore any duplicate values encountered while reading
761       * LDIF records.
762       *
763       * @return  {@code true} if duplicate values should be ignored, or
764       *          {@code false} if any LDIF records containing duplicate values
765       *          should be rejected.
766       *
767       * @deprecated  Use the {@link #getDuplicateValueBehavior} method instead.
768       */
769      @Deprecated()
770      public boolean ignoreDuplicateValues()
771      {
772        return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
773      }
774    
775    
776    
777      /**
778       * Specifies whether to ignore any duplicate values encountered while reading
779       * LDIF records.
780       *
781       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
782       *                                attribute values encountered while reading
783       *                                LDIF records.
784       *
785       * @deprecated  Use the {@link #setDuplicateValueBehavior} method instead.
786       */
787      @Deprecated()
788      public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
789      {
790        if (ignoreDuplicateValues)
791        {
792          duplicateValueBehavior = DuplicateValueBehavior.STRIP;
793        }
794        else
795        {
796          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
797        }
798      }
799    
800    
801    
802      /**
803       * Retrieves the behavior that should be exhibited if the LDIF reader
804       * encounters an entry with duplicate values.
805       *
806       * @return  The behavior that should be exhibited if the LDIF reader
807       *          encounters an entry with duplicate values.
808       */
809      public DuplicateValueBehavior getDuplicateValueBehavior()
810      {
811        return duplicateValueBehavior;
812      }
813    
814    
815    
816      /**
817       * Specifies the behavior that should be exhibited if the LDIF reader
818       * encounters an entry with duplicate values.
819       *
820       * @param  duplicateValueBehavior  The behavior that should be exhibited if
821       *                                 the LDIF reader encounters an entry with
822       *                                 duplicate values.
823       */
824      public void setDuplicateValueBehavior(
825                       final DuplicateValueBehavior duplicateValueBehavior)
826      {
827        this.duplicateValueBehavior = duplicateValueBehavior;
828      }
829    
830    
831    
832      /**
833       * Indicates whether to strip off any illegal trailing spaces that may appear
834       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
835       * specification strongly recommends that any value which legitimately
836       * contains trailing spaces be base64-encoded, and any spaces which appear
837       * after the end of non-base64-encoded values may therefore be considered
838       * invalid.  If any such trailing spaces are encountered in an LDIF record and
839       * they are not to be stripped, then an {@link LDIFException} will be thrown
840       * for that record.
841       * <BR><BR>
842       * Note that this applies only to spaces after the end of a value, and not to
843       * spaces which may appear at the end of a line for a value that is wrapped
844       * and continued on the next line.
845       *
846       * @return  {@code true} if illegal trailing spaces should be stripped off, or
847       *          {@code false} if LDIF records containing illegal trailing spaces
848       *          should be rejected.
849       *
850       * @deprecated  Use the {@link #getTrailingSpaceBehavior} method instead.
851       */
852      @Deprecated()
853      public boolean stripTrailingSpaces()
854      {
855        return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
856      }
857    
858    
859    
860      /**
861       * Specifies whether to strip off any illegal trailing spaces that may appear
862       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
863       * specification strongly recommends that any value which legitimately
864       * contains trailing spaces be base64-encoded, and any spaces which appear
865       * after the end of non-base64-encoded values may therefore be considered
866       * invalid.  If any such trailing spaces are encountered in an LDIF record and
867       * they are not to be stripped, then an {@link LDIFException} will be thrown
868       * for that record.
869       * <BR><BR>
870       * Note that this applies only to spaces after the end of a value, and not to
871       * spaces which may appear at the end of a line for a value that is wrapped
872       * and continued on the next line.
873       *
874       * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
875       *                              trailing spaces, or {@code false} if LDIF
876       *                              records containing them should be rejected.
877       *
878       * @deprecated  Use the {@link #setTrailingSpaceBehavior} method instead.
879       */
880      @Deprecated()
881      public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
882      {
883        trailingSpaceBehavior = stripTrailingSpaces
884             ? TrailingSpaceBehavior.STRIP
885             : TrailingSpaceBehavior.REJECT;
886      }
887    
888    
889    
890      /**
891       * Retrieves the behavior that should be exhibited when encountering attribute
892       * values which are not base64-encoded but contain trailing spaces.  The LDIF
893       * specification strongly recommends that any value which legitimately
894       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
895       * may be configured to automatically strip these spaces, to preserve them, or
896       * to reject any entry or change record containing them.
897       *
898       * @return  The behavior that should be exhibited when encountering attribute
899       *          values which are not base64-encoded but contain trailing spaces.
900       */
901      public TrailingSpaceBehavior getTrailingSpaceBehavior()
902      {
903        return trailingSpaceBehavior;
904      }
905    
906    
907    
908      /**
909       * Specifies the behavior that should be exhibited when encountering attribute
910       * values which are not base64-encoded but contain trailing spaces.  The LDIF
911       * specification strongly recommends that any value which legitimately
912       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
913       * may be configured to automatically strip these spaces, to preserve them, or
914       * to reject any entry or change record containing them.
915       *
916       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
917       *                                encountering attribute values which are not
918       *                                base64-encoded but contain trailing spaces.
919       */
920      public void setTrailingSpaceBehavior(
921                       final TrailingSpaceBehavior trailingSpaceBehavior)
922      {
923        this.trailingSpaceBehavior = trailingSpaceBehavior;
924      }
925    
926    
927    
928      /**
929       * Retrieves the base path that will be prepended to relative paths in order
930       * to obtain an absolute path.  This will only be used for "file:" URLs that
931       * have paths which do not begin with a slash.
932       *
933       * @return  The base path that will be prepended to relative paths in order to
934       *          obtain an absolute path.
935       */
936      public String getRelativeBasePath()
937      {
938        return relativeBasePath;
939      }
940    
941    
942    
943      /**
944       * Specifies the base path that will be prepended to relative paths in order
945       * to obtain an absolute path.  This will only be used for "file:" URLs that
946       * have paths which do not begin with a space.
947       *
948       * @param  relativeBasePath  The base path that will be prepended to relative
949       *                           paths in order to obtain an absolute path.
950       */
951      public void setRelativeBasePath(final String relativeBasePath)
952      {
953        setRelativeBasePath(new File(relativeBasePath));
954      }
955    
956    
957    
958      /**
959       * Specifies the base path that will be prepended to relative paths in order
960       * to obtain an absolute path.  This will only be used for "file:" URLs that
961       * have paths which do not begin with a space.
962       *
963       * @param  relativeBasePath  The base path that will be prepended to relative
964       *                           paths in order to obtain an absolute path.
965       */
966      public void setRelativeBasePath(final File relativeBasePath)
967      {
968        final String path = relativeBasePath.getAbsolutePath();
969        if (path.endsWith(File.separator))
970        {
971          this.relativeBasePath = path;
972        }
973        else
974        {
975          this.relativeBasePath = path + File.separator;
976        }
977      }
978    
979    
980    
981      /**
982       * Retrieves the schema that will be used when reading LDIF records, if
983       * defined.
984       *
985       * @return  The schema that will be used when reading LDIF records, or
986       *          {@code null} if no schema should be used and all attributes should
987       *          be treated as case-insensitive strings.
988       */
989      public Schema getSchema()
990      {
991        return schema;
992      }
993    
994    
995    
996      /**
997       * Specifies the schema that should be used when reading LDIF records.
998       *
999       * @param  schema  The schema that should be used when reading LDIF records,
1000       *                 or {@code null} if no schema should be used and all
1001       *                 attributes should be treated as case-insensitive strings.
1002       */
1003      public void setSchema(final Schema schema)
1004      {
1005        this.schema = schema;
1006      }
1007    
1008    
1009    
1010      /**
1011       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1012       * change record.
1013       *
1014       * @return  The record read from the LDIF source, or {@code null} if there are
1015       *          no more entries to be read.
1016       *
1017       * @throws  IOException  If a problem occurs while trying to read from the
1018       *                       LDIF source.
1019       *
1020       * @throws  LDIFException  If the data read could not be parsed as an entry or
1021       *                         an LDIF change record.
1022       */
1023      public LDIFRecord readLDIFRecord()
1024             throws IOException, LDIFException
1025      {
1026        if (isAsync())
1027        {
1028          return readLDIFRecordAsync();
1029        }
1030        else
1031        {
1032          return readLDIFRecordInternal();
1033        }
1034      }
1035    
1036    
1037    
1038      /**
1039       * Reads an entry from the LDIF source.
1040       *
1041       * @return  The entry read from the LDIF source, or {@code null} if there are
1042       *          no more entries to be read.
1043       *
1044       * @throws  IOException  If a problem occurs while attempting to read from the
1045       *                       LDIF source.
1046       *
1047       * @throws  LDIFException  If the data read could not be parsed as an entry.
1048       */
1049      public Entry readEntry()
1050             throws IOException, LDIFException
1051      {
1052        if (isAsync())
1053        {
1054          return readEntryAsync();
1055        }
1056        else
1057        {
1058          return readEntryInternal();
1059        }
1060      }
1061    
1062    
1063    
1064      /**
1065       * Reads an LDIF change record from the LDIF source.  The LDIF record must
1066       * have a changetype.
1067       *
1068       * @return  The change record read from the LDIF source, or {@code null} if
1069       *          there are no more records to be read.
1070       *
1071       * @throws  IOException  If a problem occurs while attempting to read from the
1072       *                       LDIF source.
1073       *
1074       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1075       *                         change record.
1076       */
1077      public LDIFChangeRecord readChangeRecord()
1078             throws IOException, LDIFException
1079      {
1080        return readChangeRecord(false);
1081      }
1082    
1083    
1084    
1085      /**
1086       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1087       * record does not have a changetype, then it may be assumed to be an add
1088       * change record.
1089       *
1090       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1091       *                     changetype should be retrieved as an add change record.
1092       *                     If this is {@code false} and the record read does not
1093       *                     include a changetype, then an {@link LDIFException}
1094       *                     will be thrown.
1095       *
1096       * @return  The change record read from the LDIF source, or {@code null} if
1097       *          there are no more records to be read.
1098       *
1099       * @throws  IOException  If a problem occurs while attempting to read from the
1100       *                       LDIF source.
1101       *
1102       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1103       *                         change record.
1104       */
1105      public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1106             throws IOException, LDIFException
1107      {
1108        if (isAsync())
1109        {
1110          return readChangeRecordAsync(defaultAdd);
1111        }
1112        else
1113        {
1114          return readChangeRecordInternal(defaultAdd);
1115        }
1116      }
1117    
1118    
1119    
1120      /**
1121       * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1122       * thread.
1123       *
1124       * @return  The next parsed record or {@code null} if there are no more
1125       *          records to read.
1126       *
1127       * @throws IOException  If IOException was thrown when reading or parsing
1128       *                      the record.
1129       *
1130       * @throws LDIFException If LDIFException was thrown parsing the record.
1131       */
1132      private LDIFRecord readLDIFRecordAsync()
1133              throws IOException, LDIFException
1134      {
1135        final Result<UnparsedLDIFRecord, LDIFRecord> result =
1136             readLDIFRecordResultAsync();
1137        if (result == null)
1138        {
1139          return null;
1140        }
1141        else
1142        {
1143          return result.getOutput();
1144        }
1145      }
1146    
1147    
1148    
1149      /**
1150       * Reads an entry asynchronously from the LDIF source.
1151       *
1152       * @return The entry read from the LDIF source, or {@code null} if there are
1153       *         no more entries to be read.
1154       *
1155       * @throws IOException   If a problem occurs while attempting to read from the
1156       *                       LDIF source.
1157       * @throws LDIFException If the data read could not be parsed as an entry.
1158       */
1159      private Entry readEntryAsync()
1160              throws IOException, LDIFException
1161      {
1162        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1163        LDIFRecord record = null;
1164        while (record == null)
1165        {
1166          result = readLDIFRecordResultAsync();
1167          if (result == null)
1168          {
1169            return null;
1170          }
1171    
1172          record = result.getOutput();
1173    
1174          // This is a special value that means we should skip this Entry.  We have
1175          // to use something different than null because null means EOF.
1176          if (record == SKIP_ENTRY)
1177          {
1178            record = null;
1179          }
1180        }
1181    
1182        if (!(record instanceof Entry))
1183        {
1184          try
1185          {
1186            // Some LDIFChangeRecord can be converted to an Entry.  This is really
1187            // an edge case though.
1188            return ((LDIFChangeRecord)record).toEntry();
1189          }
1190          catch (LDIFException e)
1191          {
1192            debugException(e);
1193            final long firstLineNumber = result.getInput().getFirstLineNumber();
1194            throw new LDIFException(e.getExceptionMessage(),
1195                                    firstLineNumber, true, e);
1196          }
1197        }
1198    
1199        return (Entry) record;
1200      }
1201    
1202    
1203    
1204      /**
1205       * Reads an LDIF change record from the LDIF source asynchronously.
1206       * Optionally, if the LDIF record does not have a changetype, then it may be
1207       * assumed to be an add change record.
1208       *
1209       * @param defaultAdd Indicates whether an LDIF record not containing a
1210       *                   changetype should be retrieved as an add change record.
1211       *                   If this is {@code false} and the record read does not
1212       *                   include a changetype, then an {@link LDIFException} will
1213       *                   be thrown.
1214       *
1215       * @return The change record read from the LDIF source, or {@code null} if
1216       *         there are no more records to be read.
1217       *
1218       * @throws IOException   If a problem occurs while attempting to read from the
1219       *                       LDIF source.
1220       * @throws LDIFException If the data read could not be parsed as an LDIF
1221       *                       change record.
1222       */
1223      private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1224              throws IOException, LDIFException
1225      {
1226        final Result<UnparsedLDIFRecord, LDIFRecord> result =
1227             readLDIFRecordResultAsync();
1228        if (result == null)
1229        {
1230          return null;
1231        }
1232    
1233        final LDIFRecord record = result.getOutput();
1234        if (record instanceof LDIFChangeRecord)
1235        {
1236          return (LDIFChangeRecord) record;
1237        }
1238        else if (record instanceof Entry)
1239        {
1240          if (defaultAdd)
1241          {
1242            return new LDIFAddChangeRecord((Entry) record);
1243          }
1244          else
1245          {
1246            final long firstLineNumber = result.getInput().getFirstLineNumber();
1247            throw new LDIFException(
1248                 ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1249                 true);
1250          }
1251        }
1252    
1253        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1254                                 "LDIFChangeRecord");
1255      }
1256    
1257    
1258    
1259      /**
1260       * Reads the next LDIF record, which was read and parsed asynchronously by
1261       * separate threads.
1262       *
1263       * @return  The next LDIF record or {@code null} if there are no more records.
1264       *
1265       * @throws  IOException  If a problem occurs while attempting to read from the
1266       *                       LDIF source.
1267       *
1268       * @throws  LDIFException  If the data read could not be parsed as an entry.
1269       */
1270      private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1271              throws IOException, LDIFException
1272      {
1273        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1274    
1275        // If the asynchronous reading and parsing is complete, then we don't have
1276        // to block waiting for the next record to show up on the queue.  If there
1277        // isn't a record there, then return null (EOF) right away.
1278        if (asyncParsingComplete.get())
1279        {
1280          result = asyncParsedRecords.poll();
1281        }
1282        else
1283        {
1284          try
1285          {
1286            // We probably could just do a asyncParsedRecords.take() here, but
1287            // there are some edge case error scenarios where
1288            // asyncParsingComplete might be set without a special EOF sentinel
1289            // Result enqueued.  So to guard against this, we have a very cautious
1290            // polling interval of 1 second.  During normal processing, we never
1291            // have to wait for this to expire, when there is something to do
1292            // (like shutdown).
1293            while ((result == null) && (!asyncParsingComplete.get()))
1294            {
1295              result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1296            }
1297    
1298            // There's a very small chance that we missed the value, so double-check
1299            if (result == null)
1300            {
1301              result = asyncParsedRecords.poll();
1302            }
1303          }
1304          catch (InterruptedException e)
1305          {
1306            debugException(e);
1307            throw createIOExceptionWithCause(null, e);
1308          }
1309        }
1310        if (result == null)
1311        {
1312          return null;
1313        }
1314    
1315        rethrow(result.getFailureCause());
1316    
1317        // Check if we reached the end of the input
1318        final UnparsedLDIFRecord unparsedRecord = result.getInput();
1319        if (unparsedRecord.isEOF())
1320        {
1321          // This might have been set already by the LineReaderThread, but
1322          // just in case it hasn't gotten to it yet, do so here.
1323          asyncParsingComplete.set(true);
1324    
1325          // Enqueue this EOF result again for any other thread that might be
1326          // blocked in asyncParsedRecords.take() even though having multiple
1327          // threads call this method concurrently breaks the contract of this
1328          // class.
1329          try
1330          {
1331            asyncParsedRecords.put(result);
1332          }
1333          catch (InterruptedException e)
1334          {
1335            // We shouldn't ever get interrupted because the put won't ever block.
1336            // Once we are done reading, this is the only item left in the queue,
1337            // so we should always be able to re-enqueue it.
1338            debugException(e);
1339          }
1340          return null;
1341        }
1342    
1343        return result;
1344      }
1345    
1346    
1347    
1348      /**
1349       * Indicates whether this LDIF reader was constructed to perform asynchronous
1350       * processing.
1351       *
1352       * @return  {@code true} if this LDIFReader was constructed to perform
1353       *          asynchronous processing, or {@code false} if not.
1354       */
1355      private boolean isAsync()
1356      {
1357        return isAsync;
1358      }
1359    
1360    
1361    
1362      /**
1363       * If not {@code null}, rethrows the specified Throwable as either an
1364       * IOException or LDIFException.
1365       *
1366       * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1367       *           is thrown.
1368       *
1369       * @throws IOException   If t is an IOException or a checked Exception that
1370       *                       is not an LDIFException.
1371       * @throws LDIFException  If t is an LDIFException.
1372       */
1373      static void rethrow(final Throwable t)
1374             throws IOException, LDIFException
1375      {
1376        if (t == null)
1377        {
1378          return;
1379        }
1380    
1381        if (t instanceof IOException)
1382        {
1383          throw (IOException) t;
1384        }
1385        else if (t instanceof LDIFException)
1386        {
1387          throw (LDIFException) t;
1388        }
1389        else if (t instanceof RuntimeException)
1390        {
1391          throw (RuntimeException) t;
1392        }
1393        else if (t instanceof Error)
1394        {
1395          throw (Error) t;
1396        }
1397        else
1398        {
1399          throw createIOExceptionWithCause(null, t);
1400        }
1401      }
1402    
1403    
1404    
1405      /**
1406       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1407       * change record.
1408       *
1409       * @return The record read from the LDIF source, or {@code null} if there are
1410       *         no more entries to be read.
1411       *
1412       * @throws IOException   If a problem occurs while trying to read from the
1413       *                       LDIF source.
1414       * @throws LDIFException If the data read could not be parsed as an entry or
1415       *                       an LDIF change record.
1416       */
1417      private LDIFRecord readLDIFRecordInternal()
1418           throws IOException, LDIFException
1419      {
1420        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1421        return decodeRecord(unparsedRecord, relativeBasePath);
1422      }
1423    
1424    
1425    
1426      /**
1427       * Reads an entry from the LDIF source.
1428       *
1429       * @return The entry read from the LDIF source, or {@code null} if there are
1430       *         no more entries to be read.
1431       *
1432       * @throws IOException   If a problem occurs while attempting to read from the
1433       *                       LDIF source.
1434       * @throws LDIFException If the data read could not be parsed as an entry.
1435       */
1436      private Entry readEntryInternal()
1437           throws IOException, LDIFException
1438      {
1439        Entry e = null;
1440        while (e == null)
1441        {
1442          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1443          if (unparsedRecord.isEOF())
1444          {
1445            return null;
1446          }
1447    
1448          e = decodeEntry(unparsedRecord, relativeBasePath);
1449          debugLDIFRead(e);
1450    
1451          if (entryTranslator != null)
1452          {
1453            e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1454          }
1455        }
1456        return e;
1457      }
1458    
1459    
1460    
1461      /**
1462       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1463       * record does not have a changetype, then it may be assumed to be an add
1464       * change record.
1465       *
1466       * @param defaultAdd Indicates whether an LDIF record not containing a
1467       *                   changetype should be retrieved as an add change record.
1468       *                   If this is {@code false} and the record read does not
1469       *                   include a changetype, then an {@link LDIFException} will
1470       *                   be thrown.
1471       *
1472       * @return The change record read from the LDIF source, or {@code null} if
1473       *         there are no more records to be read.
1474       *
1475       * @throws IOException   If a problem occurs while attempting to read from the
1476       *                       LDIF source.
1477       * @throws LDIFException If the data read could not be parsed as an LDIF
1478       *                       change record.
1479       */
1480      private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1481           throws IOException, LDIFException
1482      {
1483        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1484        if (unparsedRecord.isEOF())
1485        {
1486          return null;
1487        }
1488    
1489        final LDIFChangeRecord r =
1490             decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd);
1491        debugLDIFRead(r);
1492        return r;
1493      }
1494    
1495    
1496    
1497      /**
1498       * Reads a record (either an entry or a change record) from the LDIF source
1499       * and places it in the line list.
1500       *
1501       * @return  The line number for the first line of the entry that was read.
1502       *
1503       * @throws  IOException  If a problem occurs while attempting to read from the
1504       *                       LDIF source.
1505       *
1506       * @throws  LDIFException  If the data read could not be parsed as a valid
1507       *                         LDIF record.
1508       */
1509      private UnparsedLDIFRecord readUnparsedRecord()
1510             throws IOException, LDIFException
1511      {
1512        final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20);
1513        boolean lastWasComment = false;
1514        long firstLineNumber = lineNumberCounter + 1;
1515        while (true)
1516        {
1517          final String line = reader.readLine();
1518          lineNumberCounter++;
1519    
1520          if (line == null)
1521          {
1522            // We've hit the end of the LDIF source.  If we haven't read any entry
1523            // data, then return null.  Otherwise, the last entry wasn't followed by
1524            // a blank line, which is OK, and we should decode that entry.
1525            if (lineList.isEmpty())
1526            {
1527              return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1528                   duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1529            }
1530            else
1531            {
1532              break;
1533            }
1534          }
1535    
1536          if (line.length() == 0)
1537          {
1538            // It's a blank line.  If we have read entry data, then this signals the
1539            // end of the entry.  Otherwise, it's an extra space between entries,
1540            // which is OK.
1541            lastWasComment = false;
1542            if (lineList.isEmpty())
1543            {
1544              firstLineNumber++;
1545              continue;
1546            }
1547            else
1548            {
1549              break;
1550            }
1551          }
1552    
1553          if (line.charAt(0) == ' ')
1554          {
1555            // The line starts with a space, which means that it must be a
1556            // continuation of the previous line.  This is true even if the last
1557            // line was a comment.
1558            if (lastWasComment)
1559            {
1560              // What we've read is part of a comment, so we don't care about its
1561              // content.
1562            }
1563            else if (lineList.isEmpty())
1564            {
1565              throw new LDIFException(
1566                             ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1567                             lineNumberCounter, false);
1568            }
1569            else
1570            {
1571              lineList.get(lineList.size() - 1).append(line.substring(1));
1572              lastWasComment = false;
1573            }
1574          }
1575          else if (line.charAt(0) == '#')
1576          {
1577            lastWasComment = true;
1578          }
1579          else
1580          {
1581            // We want to make sure that we skip over the "version:" line if it
1582            // exists, but that should only occur at the beginning of an entry where
1583            // it can't be confused with a possible "version" attribute.
1584            if (lineList.isEmpty() && line.startsWith("version:"))
1585            {
1586              lastWasComment = true;
1587            }
1588            else
1589            {
1590              lineList.add(new StringBuilder(line));
1591              lastWasComment = false;
1592            }
1593          }
1594        }
1595    
1596        return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1597             trailingSpaceBehavior, schema, firstLineNumber);
1598      }
1599    
1600    
1601    
1602      /**
1603       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1604       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1605       * LDIF specification, and it is acceptable to have one or more blank lines
1606       * following the entry.
1607       *
1608       * @param  ldifLines  The set of lines that comprise the LDIF representation
1609       *                    of the entry.  It must not be {@code null} or empty.
1610       *
1611       * @return  The entry read from LDIF.
1612       *
1613       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1614       *                         entry.
1615       */
1616      public static Entry decodeEntry(final String... ldifLines)
1617             throws LDIFException
1618      {
1619        final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1620             TrailingSpaceBehavior.REJECT, null, ldifLines),
1621             DEFAULT_RELATIVE_BASE_PATH);
1622        debugLDIFRead(e);
1623        return e;
1624      }
1625    
1626    
1627    
1628      /**
1629       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1630       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1631       * LDIF specification, and it is acceptable to have one or more blank lines
1632       * following the entry.
1633       *
1634       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1635       *                                attribute values encountered while parsing.
1636       * @param  schema                 The schema to use when parsing the record,
1637       *                                if applicable.
1638       * @param  ldifLines              The set of lines that comprise the LDIF
1639       *                                representation of the entry.  It must not be
1640       *                                {@code null} or empty.
1641       *
1642       * @return  The entry read from LDIF.
1643       *
1644       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1645       *                         entry.
1646       */
1647      public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1648                                      final Schema schema,
1649                                      final String... ldifLines)
1650             throws LDIFException
1651      {
1652        final Entry e = decodeEntry(prepareRecord(
1653                  (ignoreDuplicateValues
1654                       ? DuplicateValueBehavior.STRIP
1655                       : DuplicateValueBehavior.REJECT),
1656                  TrailingSpaceBehavior.REJECT, schema, ldifLines),
1657             DEFAULT_RELATIVE_BASE_PATH);
1658        debugLDIFRead(e);
1659        return e;
1660      }
1661    
1662    
1663    
1664      /**
1665       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1666       * provided set of lines must contain exactly one change record and it must
1667       * include a changetype.  Long lines may be wrapped as per the LDIF
1668       * specification, and it is acceptable to have one or more blank lines
1669       * following the entry.
1670       *
1671       * @param  ldifLines  The set of lines that comprise the LDIF representation
1672       *                    of the change record.  It must not be {@code null} or
1673       *                    empty.
1674       *
1675       * @return  The change record read from LDIF.
1676       *
1677       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1678       *                         change record.
1679       */
1680      public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1681             throws LDIFException
1682      {
1683        return decodeChangeRecord(false, ldifLines);
1684      }
1685    
1686    
1687    
1688      /**
1689       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1690       * provided set of lines must contain exactly one change record.  Long lines
1691       * may be wrapped as per the LDIF specification, and it is acceptable to have
1692       * one or more blank lines following the entry.
1693       *
1694       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1695       *                     changetype should be retrieved as an add change record.
1696       *                     If this is {@code false} and the record read does not
1697       *                     include a changetype, then an {@link LDIFException}
1698       *                     will be thrown.
1699       * @param  ldifLines  The set of lines that comprise the LDIF representation
1700       *                    of the change record.  It must not be {@code null} or
1701       *                    empty.
1702       *
1703       * @return  The change record read from LDIF.
1704       *
1705       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1706       *                         change record.
1707       */
1708      public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1709                                                        final String... ldifLines)
1710             throws LDIFException
1711      {
1712        final LDIFChangeRecord r =
1713             decodeChangeRecord(
1714                  prepareRecord(DuplicateValueBehavior.STRIP,
1715                       TrailingSpaceBehavior.REJECT, null, ldifLines),
1716                  DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1717        debugLDIFRead(r);
1718        return r;
1719      }
1720    
1721    
1722    
1723      /**
1724       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1725       * provided set of lines must contain exactly one change record.  Long lines
1726       * may be wrapped as per the LDIF specification, and it is acceptable to have
1727       * one or more blank lines following the entry.
1728       *
1729       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1730       *                                attribute values encountered while parsing.
1731       * @param  schema                 The schema to use when processing the change
1732       *                                record, or {@code null} if no schema should
1733       *                                be used and all values should be treated as
1734       *                                case-insensitive strings.
1735       * @param  defaultAdd             Indicates whether an LDIF record not
1736       *                                containing a changetype should be retrieved
1737       *                                as an add change record.  If this is
1738       *                                {@code false} and the record read does not
1739       *                                include a changetype, then an
1740       *                                {@link LDIFException} will be thrown.
1741       * @param  ldifLines              The set of lines that comprise the LDIF
1742       *                                representation of the change record.  It
1743       *                                must not be {@code null} or empty.
1744       *
1745       * @return  The change record read from LDIF.
1746       *
1747       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1748       *                         change record.
1749       */
1750      public static LDIFChangeRecord decodeChangeRecord(
1751                                          final boolean ignoreDuplicateValues,
1752                                          final Schema schema,
1753                                          final boolean defaultAdd,
1754                                          final String... ldifLines)
1755             throws LDIFException
1756      {
1757        final LDIFChangeRecord r = decodeChangeRecord(
1758             prepareRecord(
1759                  (ignoreDuplicateValues
1760                       ? DuplicateValueBehavior.STRIP
1761                       : DuplicateValueBehavior.REJECT),
1762                  TrailingSpaceBehavior.REJECT, schema, ldifLines),
1763             DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1764        debugLDIFRead(r);
1765        return r;
1766      }
1767    
1768    
1769    
1770      /**
1771       * Parses the provided set of lines into a list of {@code StringBuilder}
1772       * objects suitable for decoding into an entry or LDIF change record.
1773       * Comments will be ignored and wrapped lines will be unwrapped.
1774       *
1775       * @param  duplicateValueBehavior  The behavior that should be exhibited if
1776       *                                 the LDIF reader encounters an entry with
1777       *                                 duplicate values.
1778       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
1779       *                                 encountering attribute values which are not
1780       *                                 base64-encoded but contain trailing spaces.
1781       * @param  schema                  The schema to use when parsing the record,
1782       *                                 if applicable.
1783       * @param  ldifLines               The set of lines that comprise the record
1784       *                                 to decode.  It must not be {@code null} or
1785       *                                 empty.
1786       *
1787       * @return  The prepared list of {@code StringBuilder} objects ready to be
1788       *          decoded.
1789       *
1790       * @throws  LDIFException  If the provided lines do not contain valid LDIF
1791       *                         content.
1792       */
1793      private static UnparsedLDIFRecord prepareRecord(
1794                          final DuplicateValueBehavior duplicateValueBehavior,
1795                          final TrailingSpaceBehavior trailingSpaceBehavior,
1796                          final Schema schema, final String... ldifLines)
1797              throws LDIFException
1798      {
1799        ensureNotNull(ldifLines);
1800        ensureFalse(ldifLines.length == 0,
1801                    "LDIFReader.prepareRecord.ldifLines must not be empty.");
1802    
1803        boolean lastWasComment = false;
1804        final ArrayList<StringBuilder> lineList =
1805             new ArrayList<StringBuilder>(ldifLines.length);
1806        for (int i=0; i < ldifLines.length; i++)
1807        {
1808          final String line = ldifLines[i];
1809          if (line.length() == 0)
1810          {
1811            // This is only acceptable if there are no more non-empty lines in the
1812            // array.
1813            for (int j=i+1; j < ldifLines.length; j++)
1814            {
1815              if (ldifLines[j].length() > 0)
1816              {
1817                throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
1818                                        ldifLines, null);
1819              }
1820    
1821              // If we've gotten here, then we know that we're at the end of the
1822              // entry.  If we have read data, then we can decode it as an entry.
1823              // Otherwise, there was no real data in the provided LDIF lines.
1824              if (lineList.isEmpty())
1825              {
1826                throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
1827                                        ldifLines, null);
1828              }
1829              else
1830              {
1831                return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1832                     trailingSpaceBehavior, schema, 0);
1833              }
1834            }
1835          }
1836    
1837          if (line.charAt(0) == ' ')
1838          {
1839            if (i > 0)
1840            {
1841              if (! lastWasComment)
1842              {
1843                lineList.get(lineList.size() - 1).append(line.substring(1));
1844              }
1845            }
1846            else
1847            {
1848              throw new LDIFException(
1849                             ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
1850                             true, ldifLines, null);
1851            }
1852          }
1853          else if (line.charAt(0) == '#')
1854          {
1855            lastWasComment = true;
1856          }
1857          else
1858          {
1859            lineList.add(new StringBuilder(line));
1860            lastWasComment = false;
1861          }
1862        }
1863    
1864        if (lineList.isEmpty())
1865        {
1866          throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
1867        }
1868        else
1869        {
1870          return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1871               trailingSpaceBehavior, schema, 0);
1872        }
1873      }
1874    
1875    
1876    
1877      /**
1878       * Decodes the unparsed record that was read from the LDIF source.  It may be
1879       * either an entry or an LDIF change record.
1880       *
1881       * @param  unparsedRecord    The unparsed LDIF record that was read from the
1882       *                           input.  It must not be {@code null} or empty.
1883       * @param  relativeBasePath  The base path that will be prepended to relative
1884       *                           paths in order to obtain an absolute path.
1885       *
1886       * @return  The parsed record, or {@code null} if there are no more entries to
1887       *          be read.
1888       *
1889       * @throws  LDIFException  If the data read could not be parsed as an entry or
1890       *                         an LDIF change record.
1891       */
1892      private static LDIFRecord decodeRecord(
1893                                     final UnparsedLDIFRecord unparsedRecord,
1894                                     final String relativeBasePath)
1895           throws LDIFException
1896      {
1897        // If there was an error reading from the input, then we rethrow it here.
1898        final Exception readError = unparsedRecord.getFailureCause();
1899        if (readError != null)
1900        {
1901          if (readError instanceof LDIFException)
1902          {
1903            // If the error was an LDIFException, which will normally be the case,
1904            // then rethrow it with all of the same state.  We could just
1905            //   throw (LDIFException) readError;
1906            // but that's considered bad form.
1907            final LDIFException ldifEx = (LDIFException) readError;
1908            throw new LDIFException(ldifEx.getMessage(),
1909                                    ldifEx.getLineNumber(),
1910                                    ldifEx.mayContinueReading(),
1911                                    ldifEx.getDataLines(),
1912                                    ldifEx.getCause());
1913          }
1914          else
1915          {
1916            throw new LDIFException(getExceptionMessage(readError),
1917                                    -1, true, readError);
1918          }
1919        }
1920    
1921        if (unparsedRecord.isEOF())
1922        {
1923          return null;
1924        }
1925    
1926        final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
1927        if (unparsedRecord.getLineList() == null)
1928        {
1929          return null;  // We can get here if there was an error reading the lines.
1930        }
1931    
1932        final LDIFRecord r;
1933        if (lineList.size() == 1)
1934        {
1935          r = decodeEntry(unparsedRecord, relativeBasePath);
1936        }
1937        else
1938        {
1939          final String lowerSecondLine = toLowerCase(lineList.get(1).toString());
1940          if (lowerSecondLine.startsWith("control:") ||
1941              lowerSecondLine.startsWith("changetype:"))
1942          {
1943            r = decodeChangeRecord(unparsedRecord, relativeBasePath, true);
1944          }
1945          else
1946          {
1947            r = decodeEntry(unparsedRecord, relativeBasePath);
1948          }
1949        }
1950    
1951        debugLDIFRead(r);
1952        return r;
1953      }
1954    
1955    
1956    
1957      /**
1958       * Decodes the provided set of LDIF lines as an entry.  The provided list must
1959       * not contain any blank lines or comments, and lines are not allowed to be
1960       * wrapped.
1961       *
1962       * @param  unparsedRecord   The unparsed LDIF record that was read from the
1963       *                          input.  It must not be {@code null} or empty.
1964       * @param  relativeBasePath  The base path that will be prepended to relative
1965       *                           paths in order to obtain an absolute path.
1966       *
1967       * @return  The entry read from LDIF.
1968       *
1969       * @throws  LDIFException  If the provided LDIF data cannot be read as an
1970       *                         entry.
1971       */
1972      private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
1973                                       final String relativeBasePath)
1974              throws LDIFException
1975      {
1976        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
1977        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
1978    
1979        final Iterator<StringBuilder> iterator = ldifLines.iterator();
1980    
1981        // The first line must start with either "version:" or "dn:".  If the first
1982        // line starts with "version:" then the second must start with "dn:".
1983        StringBuilder line = iterator.next();
1984        handleTrailingSpaces(line, null, firstLineNumber,
1985             unparsedRecord.getTrailingSpaceBehavior());
1986        int colonPos = line.indexOf(":");
1987        if ((colonPos > 0) &&
1988            line.substring(0, colonPos).equalsIgnoreCase("version"))
1989        {
1990          // The first line is "version:".  Under most conditions, this will be
1991          // handled by the LDIF reader, but this can happen if you call
1992          // decodeEntry with a set of data that includes a version.  At any rate,
1993          // read the next line, which must specify the DN.
1994          line = iterator.next();
1995          handleTrailingSpaces(line, null, firstLineNumber,
1996               unparsedRecord.getTrailingSpaceBehavior());
1997        }
1998    
1999        colonPos = line.indexOf(":");
2000        if ((colonPos < 0) ||
2001             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2002        {
2003          throw new LDIFException(
2004               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2005               firstLineNumber, true, ldifLines, null);
2006        }
2007    
2008        final String dn;
2009        final int length = line.length();
2010        if (length == (colonPos+1))
2011        {
2012          // The colon was the last character on the line.  This is acceptable and
2013          // indicates that the entry has the null DN.
2014          dn = "";
2015        }
2016        else if (line.charAt(colonPos+1) == ':')
2017        {
2018          // Skip over any spaces leading up to the value, and then the rest of the
2019          // string is the base64-encoded DN.
2020          int pos = colonPos+2;
2021          while ((pos < length) && (line.charAt(pos) == ' '))
2022          {
2023            pos++;
2024          }
2025    
2026          try
2027          {
2028            final byte[] dnBytes = Base64.decode(line.substring(pos));
2029            dn = new String(dnBytes, "UTF-8");
2030          }
2031          catch (final ParseException pe)
2032          {
2033            debugException(pe);
2034            throw new LDIFException(
2035                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2036                                                                pe.getMessage()),
2037                           firstLineNumber, true, ldifLines, pe);
2038          }
2039          catch (final Exception e)
2040          {
2041            debugException(e);
2042            throw new LDIFException(
2043                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2044                           firstLineNumber, true, ldifLines, e);
2045          }
2046        }
2047        else
2048        {
2049          // Skip over any spaces leading up to the value, and then the rest of the
2050          // string is the DN.
2051          int pos = colonPos+1;
2052          while ((pos < length) && (line.charAt(pos) == ' '))
2053          {
2054            pos++;
2055          }
2056    
2057          dn = line.substring(pos);
2058        }
2059    
2060    
2061        // The remaining lines must be the attributes for the entry.  However, we
2062        // will allow the case in which an entry does not have any attributes, to be
2063        // able to support reading search result entries in which no attributes were
2064        // returned.
2065        if (! iterator.hasNext())
2066        {
2067          return new Entry(dn, unparsedRecord.getSchema());
2068        }
2069    
2070        return new Entry(dn, unparsedRecord.getSchema(),
2071             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2072                  unparsedRecord.getTrailingSpaceBehavior(),
2073                  unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2074                  firstLineNumber));
2075      }
2076    
2077    
2078    
2079      /**
2080       * Decodes the provided set of LDIF lines as a change record.  The provided
2081       * list must not contain any blank lines or comments, and lines are not
2082       * allowed to be wrapped.
2083       *
2084       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2085       *                           input.  It must not be {@code null} or empty.
2086       * @param  relativeBasePath  The base path that will be prepended to relative
2087       *                           paths in order to obtain an absolute path.
2088       * @param  defaultAdd        Indicates whether an LDIF record not containing a
2089       *                           changetype should be retrieved as an add change
2090       *                           record.  If this is {@code false} and the record
2091       *                           read does not include a changetype, then an
2092       *                           {@link LDIFException} will be thrown.
2093       *
2094       * @return  The change record read from LDIF.
2095       *
2096       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2097       *                         change record.
2098       */
2099      private static LDIFChangeRecord decodeChangeRecord(
2100                                           final UnparsedLDIFRecord unparsedRecord,
2101                                           final String relativeBasePath,
2102                                           final boolean defaultAdd)
2103              throws LDIFException
2104      {
2105        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2106        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2107    
2108        Iterator<StringBuilder> iterator = ldifLines.iterator();
2109    
2110        // The first line must start with either "version:" or "dn:".  If the first
2111        // line starts with "version:" then the second must start with "dn:".
2112        StringBuilder line = iterator.next();
2113        handleTrailingSpaces(line, null, firstLineNumber,
2114             unparsedRecord.getTrailingSpaceBehavior());
2115        int colonPos = line.indexOf(":");
2116        int linesRead = 1;
2117        if ((colonPos > 0) &&
2118            line.substring(0, colonPos).equalsIgnoreCase("version"))
2119        {
2120          // The first line is "version:".  Under most conditions, this will be
2121          // handled by the LDIF reader, but this can happen if you call
2122          // decodeEntry with a set of data that includes a version.  At any rate,
2123          // read the next line, which must specify the DN.
2124          line = iterator.next();
2125          linesRead++;
2126          handleTrailingSpaces(line, null, firstLineNumber,
2127               unparsedRecord.getTrailingSpaceBehavior());
2128        }
2129    
2130        colonPos = line.indexOf(":");
2131        if ((colonPos < 0) ||
2132             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2133        {
2134          throw new LDIFException(
2135               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2136               firstLineNumber, true, ldifLines, null);
2137        }
2138    
2139        final String dn;
2140        int length = line.length();
2141        if (length == (colonPos+1))
2142        {
2143          // The colon was the last character on the line.  This is acceptable and
2144          // indicates that the entry has the null DN.
2145          dn = "";
2146        }
2147        else if (line.charAt(colonPos+1) == ':')
2148        {
2149          // Skip over any spaces leading up to the value, and then the rest of the
2150          // string is the base64-encoded DN.
2151          int pos = colonPos+2;
2152          while ((pos < length) && (line.charAt(pos) == ' '))
2153          {
2154            pos++;
2155          }
2156    
2157          try
2158          {
2159            final byte[] dnBytes = Base64.decode(line.substring(pos));
2160            dn = new String(dnBytes, "UTF-8");
2161          }
2162          catch (final ParseException pe)
2163          {
2164            debugException(pe);
2165            throw new LDIFException(
2166                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2167                                                                   pe.getMessage()),
2168                           firstLineNumber, true, ldifLines, pe);
2169          }
2170          catch (final Exception e)
2171          {
2172            debugException(e);
2173            throw new LDIFException(
2174                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2175                                                                   e),
2176                           firstLineNumber, true, ldifLines, e);
2177          }
2178        }
2179        else
2180        {
2181          // Skip over any spaces leading up to the value, and then the rest of the
2182          // string is the DN.
2183          int pos = colonPos+1;
2184          while ((pos < length) && (line.charAt(pos) == ' '))
2185          {
2186            pos++;
2187          }
2188    
2189          dn = line.substring(pos);
2190        }
2191    
2192    
2193        // An LDIF change record may contain zero or more controls, with the end of
2194        // the controls signified by the changetype.  The changetype element must be
2195        // present, unless defaultAdd is true in which case the first thing that is
2196        // neither control or changetype will trigger the start of add attribute
2197        // parsing.
2198        if (! iterator.hasNext())
2199        {
2200          throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2201                                  firstLineNumber, true, ldifLines, null);
2202        }
2203    
2204        String changeType = null;
2205        ArrayList<Control> controls = null;
2206        while (true)
2207        {
2208          line = iterator.next();
2209          handleTrailingSpaces(line, dn, firstLineNumber,
2210               unparsedRecord.getTrailingSpaceBehavior());
2211          colonPos = line.indexOf(":");
2212          if (colonPos < 0)
2213          {
2214            throw new LDIFException(
2215                 ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2216                 firstLineNumber, true, ldifLines, null);
2217          }
2218    
2219          final String token = toLowerCase(line.substring(0, colonPos));
2220          if (token.equals("control"))
2221          {
2222            if (controls == null)
2223            {
2224              controls = new ArrayList<Control>(5);
2225            }
2226    
2227            controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2228                 relativeBasePath));
2229          }
2230          else if (token.equals("changetype"))
2231          {
2232            changeType =
2233                 decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2234            break;
2235          }
2236          else if (defaultAdd)
2237          {
2238            // The line we read wasn't a control or changetype declaration, so we'll
2239            // assume it's an attribute in an add record.  However, we're not ready
2240            // for that yet, and since we can't rewind an iterator we'll create a
2241            // new one that hasn't yet gotten to this line.
2242            changeType = "add";
2243            iterator = ldifLines.iterator();
2244            for (int i=0; i < linesRead; i++)
2245            {
2246              iterator.next();
2247            }
2248            break;
2249          }
2250          else
2251          {
2252            throw new LDIFException(
2253                 ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2254                      firstLineNumber),
2255                 firstLineNumber, true, ldifLines, null);
2256          }
2257    
2258          linesRead++;
2259        }
2260    
2261    
2262        // Make sure that the change type is acceptable and then decode the rest of
2263        // the change record accordingly.
2264        final String lowerChangeType = toLowerCase(changeType);
2265        if (lowerChangeType.equals("add"))
2266        {
2267          // There must be at least one more line.  If not, then that's an error.
2268          // Otherwise, parse the rest of the data as attribute-value pairs.
2269          if (iterator.hasNext())
2270          {
2271            final Collection<Attribute> attrs =
2272                 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2273                      unparsedRecord.getTrailingSpaceBehavior(),
2274                      unparsedRecord.getSchema(), ldifLines, iterator,
2275                      relativeBasePath, firstLineNumber);
2276            final Attribute[] attributes = new Attribute[attrs.size()];
2277            final Iterator<Attribute> attrIterator = attrs.iterator();
2278            for (int i=0; i < attributes.length; i++)
2279            {
2280              attributes[i] = attrIterator.next();
2281            }
2282    
2283            return new LDIFAddChangeRecord(dn, attributes, controls);
2284          }
2285          else
2286          {
2287            throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2288                                    firstLineNumber, true, ldifLines, null);
2289          }
2290        }
2291        else if (lowerChangeType.equals("delete"))
2292        {
2293          // There shouldn't be any more data.  If there is, then that's an error.
2294          // Otherwise, we can just return the delete change record with what we
2295          // already know.
2296          if (iterator.hasNext())
2297          {
2298            throw new LDIFException(
2299                           ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2300                           firstLineNumber, true, ldifLines, null);
2301          }
2302          else
2303          {
2304            return new LDIFDeleteChangeRecord(dn, controls);
2305          }
2306        }
2307        else if (lowerChangeType.equals("modify"))
2308        {
2309          // There must be at least one more line.  If not, then that's an error.
2310          // Otherwise, parse the rest of the data as a set of modifications.
2311          if (iterator.hasNext())
2312          {
2313            final Modification[] mods = parseModifications(dn,
2314                 unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2315                 firstLineNumber);
2316            return new LDIFModifyChangeRecord(dn, mods, controls);
2317          }
2318          else
2319          {
2320            throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2321                                    firstLineNumber, true, ldifLines, null);
2322          }
2323        }
2324        else if (lowerChangeType.equals("moddn") ||
2325                 lowerChangeType.equals("modrdn"))
2326        {
2327          // There must be at least one more line.  If not, then that's an error.
2328          // Otherwise, parse the rest of the data as a set of modifications.
2329          if (iterator.hasNext())
2330          {
2331            return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2332                 unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2333          }
2334          else
2335          {
2336            throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2337                                    firstLineNumber, true, ldifLines, null);
2338          }
2339        }
2340        else
2341        {
2342          throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2343                                                             firstLineNumber),
2344                                  firstLineNumber, true, ldifLines, null);
2345        }
2346      }
2347    
2348    
2349    
2350      /**
2351       * Decodes information about a control from the provided line.
2352       *
2353       * @param  line              The line to process.
2354       * @param  colonPos          The position of the colon that separates the
2355       *                           control token string from tbe encoded control.
2356       * @param  firstLineNumber   The line number for the start of the record.
2357       * @param  ldifLines         The lines that comprise the LDIF representation
2358       *                           of the full record being parsed.
2359       * @param  relativeBasePath  The base path that will be prepended to relative
2360       *                           paths in order to obtain an absolute path.
2361       *
2362       * @return  The decoded control.
2363       *
2364       * @throws  LDIFException  If a problem is encountered while trying to decode
2365       *                         the changetype.
2366       */
2367      private static Control decodeControl(final StringBuilder line,
2368                                           final int colonPos,
2369                                           final long firstLineNumber,
2370                                           final ArrayList<StringBuilder> ldifLines,
2371                                           final String relativeBasePath)
2372              throws LDIFException
2373      {
2374        final String controlString;
2375        int length = line.length();
2376        if (length == (colonPos+1))
2377        {
2378          // The colon was the last character on the line.  This is not
2379          // acceptable.
2380          throw new LDIFException(
2381               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2382               firstLineNumber, true, ldifLines, null);
2383        }
2384        else if (line.charAt(colonPos+1) == ':')
2385        {
2386          // Skip over any spaces leading up to the value, and then the rest of
2387          // the string is the base64-encoded control representation.  This is
2388          // unusual and unnecessary, but is nevertheless acceptable.
2389          int pos = colonPos+2;
2390          while ((pos < length) && (line.charAt(pos) == ' '))
2391          {
2392            pos++;
2393          }
2394    
2395          try
2396          {
2397            final byte[] controlBytes = Base64.decode(line.substring(pos));
2398            controlString =  new String(controlBytes, "UTF-8");
2399          }
2400          catch (final ParseException pe)
2401          {
2402            debugException(pe);
2403            throw new LDIFException(
2404                           ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2405                                firstLineNumber, pe.getMessage()),
2406                           firstLineNumber, true, ldifLines, pe);
2407          }
2408          catch (final Exception e)
2409          {
2410            debugException(e);
2411            throw new LDIFException(
2412                 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2413                 firstLineNumber, true, ldifLines, e);
2414          }
2415        }
2416        else
2417        {
2418          // Skip over any spaces leading up to the value, and then the rest of
2419          // the string is the encoded control.
2420          int pos = colonPos+1;
2421          while ((pos < length) && (line.charAt(pos) == ' '))
2422          {
2423            pos++;
2424          }
2425    
2426          controlString = line.substring(pos);
2427        }
2428    
2429        // If the resulting control definition is empty, then that's invalid.
2430        if (controlString.length() == 0)
2431        {
2432          throw new LDIFException(
2433               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2434               firstLineNumber, true, ldifLines, null);
2435        }
2436    
2437    
2438        // The first element of the control must be the OID, and it must be followed
2439        // by a space (to separate it from the criticality), a colon (to separate it
2440        // from the value and indicate a default criticality of false), or the end
2441        // of the line (to indicate a default criticality of false and no value).
2442        String oid = null;
2443        boolean hasCriticality = false;
2444        boolean hasValue = false;
2445        int pos = 0;
2446        length = controlString.length();
2447        while (pos < length)
2448        {
2449          final char c = controlString.charAt(pos);
2450          if (c == ':')
2451          {
2452            // This indicates that there is no criticality and that the value
2453            // immediately follows the OID.
2454            oid = controlString.substring(0, pos++);
2455            hasValue = true;
2456            break;
2457          }
2458          else if (c == ' ')
2459          {
2460            // This indicates that there is a criticality.  We don't know anything
2461            // about the presence of a value yet.
2462            oid = controlString.substring(0, pos++);
2463            hasCriticality = true;
2464            break;
2465          }
2466          else
2467          {
2468            pos++;
2469          }
2470        }
2471    
2472        if (oid == null)
2473        {
2474          // This indicates that the string representation of the control is only
2475          // the OID.
2476          return new Control(controlString, false);
2477        }
2478    
2479    
2480        // See if we need to read the criticality.  If so, then do so now.
2481        // Otherwise, assume a default criticality of false.
2482        final boolean isCritical;
2483        if (hasCriticality)
2484        {
2485          // Skip over any spaces before the criticality.
2486          while (controlString.charAt(pos) == ' ')
2487          {
2488            pos++;
2489          }
2490    
2491          // Read until we find a colon or the end of the string.
2492          final int criticalityStartPos = pos;
2493          while (pos < length)
2494          {
2495            final char c = controlString.charAt(pos);
2496            if (c == ':')
2497            {
2498              hasValue = true;
2499              break;
2500            }
2501            else
2502            {
2503              pos++;
2504            }
2505          }
2506    
2507          final String criticalityString =
2508               toLowerCase(controlString.substring(criticalityStartPos, pos));
2509          if (criticalityString.equals("true"))
2510          {
2511            isCritical = true;
2512          }
2513          else if (criticalityString.equals("false"))
2514          {
2515            isCritical = false;
2516          }
2517          else
2518          {
2519            throw new LDIFException(
2520                 ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2521                      firstLineNumber),
2522                 firstLineNumber, true, ldifLines, null);
2523          }
2524    
2525          if (hasValue)
2526          {
2527            pos++;
2528          }
2529        }
2530        else
2531        {
2532          isCritical = false;
2533        }
2534    
2535        // See if we need to read the value.  If so, then do so now.  It may be
2536        // a string, or it may be base64-encoded.  It could conceivably even be read
2537        // from a URL.
2538        final ASN1OctetString value;
2539        if (hasValue)
2540        {
2541          // The character immediately after the colon that precedes the value may
2542          // be one of the following:
2543          // - A second colon (optionally followed by a single space) to indicate
2544          //   that the value is base64-encoded.
2545          // - A less-than symbol to indicate that the value should be read from a
2546          //   location specified by a URL.
2547          // - A single space that precedes the non-base64-encoded value.
2548          // - The first character of the non-base64-encoded value.
2549          switch (controlString.charAt(pos))
2550          {
2551            case ':':
2552              try
2553              {
2554                if (controlString.length() == (pos+1))
2555                {
2556                  value = new ASN1OctetString();
2557                }
2558                else if (controlString.charAt(pos+1) == ' ')
2559                {
2560                  value = new ASN1OctetString(
2561                       Base64.decode(controlString.substring(pos+2)));
2562                }
2563                else
2564                {
2565                  value = new ASN1OctetString(
2566                       Base64.decode(controlString.substring(pos+1)));
2567                }
2568              }
2569              catch (final Exception e)
2570              {
2571                debugException(e);
2572                throw new LDIFException(
2573                     ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2574                          firstLineNumber, getExceptionMessage(e)),
2575                     firstLineNumber, true, ldifLines, e);
2576              }
2577              break;
2578            case '<':
2579              try
2580              {
2581                final String urlString;
2582                if (controlString.charAt(pos+1) == ' ')
2583                {
2584                  urlString = controlString.substring(pos+2);
2585                }
2586                else
2587                {
2588                  urlString = controlString.substring(pos+1);
2589                }
2590                value = new ASN1OctetString(retrieveURLBytes(urlString,
2591                     relativeBasePath, firstLineNumber));
2592              }
2593              catch (final Exception e)
2594              {
2595                debugException(e);
2596                throw new LDIFException(
2597                     ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2598                          firstLineNumber, getExceptionMessage(e)),
2599                     firstLineNumber, true, ldifLines, e);
2600              }
2601              break;
2602            case ' ':
2603              value = new ASN1OctetString(controlString.substring(pos+1));
2604              break;
2605            default:
2606              value = new ASN1OctetString(controlString.substring(pos));
2607              break;
2608          }
2609        }
2610        else
2611        {
2612          value = null;
2613        }
2614    
2615        return new Control(oid, isCritical, value);
2616      }
2617    
2618    
2619    
2620      /**
2621       * Decodes the changetype element from the provided line.
2622       *
2623       * @param  line             The line to process.
2624       * @param  colonPos         The position of the colon that separates the
2625       *                          changetype string from its value.
2626       * @param  firstLineNumber  The line number for the start of the record.
2627       * @param  ldifLines        The lines that comprise the LDIF representation of
2628       *                          the full record being parsed.
2629       *
2630       * @return  The decoded changetype string.
2631       *
2632       * @throws  LDIFException  If a problem is encountered while trying to decode
2633       *                         the changetype.
2634       */
2635      private static String decodeChangeType(final StringBuilder line,
2636                                 final int colonPos, final long firstLineNumber,
2637                                 final ArrayList<StringBuilder> ldifLines)
2638              throws LDIFException
2639      {
2640        final int length = line.length();
2641        if (length == (colonPos+1))
2642        {
2643          // The colon was the last character on the line.  This is not
2644          // acceptable.
2645          throw new LDIFException(
2646               ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2647               true, ldifLines, null);
2648        }
2649        else if (line.charAt(colonPos+1) == ':')
2650        {
2651          // Skip over any spaces leading up to the value, and then the rest of
2652          // the string is the base64-encoded changetype.  This is unusual and
2653          // unnecessary, but is nevertheless acceptable.
2654          int pos = colonPos+2;
2655          while ((pos < length) && (line.charAt(pos) == ' '))
2656          {
2657            pos++;
2658          }
2659    
2660          try
2661          {
2662            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
2663            return new String(changeTypeBytes, "UTF-8");
2664          }
2665          catch (final ParseException pe)
2666          {
2667            debugException(pe);
2668            throw new LDIFException(
2669                           ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
2670                                                                pe.getMessage()),
2671                           firstLineNumber, true, ldifLines, pe);
2672          }
2673          catch (final Exception e)
2674          {
2675            debugException(e);
2676            throw new LDIFException(
2677                 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
2678                 firstLineNumber, true, ldifLines, e);
2679          }
2680        }
2681        else
2682        {
2683          // Skip over any spaces leading up to the value, and then the rest of
2684          // the string is the changetype.
2685          int pos = colonPos+1;
2686          while ((pos < length) && (line.charAt(pos) == ' '))
2687          {
2688            pos++;
2689          }
2690    
2691          return line.substring(pos);
2692        }
2693      }
2694    
2695    
2696    
2697      /**
2698       * Parses the data available through the provided iterator as a collection of
2699       * attributes suitable for use in an entry or an add change record.
2700       *
2701       * @param  dn                      The DN of the record being read.
2702       * @param  duplicateValueBehavior  The behavior that should be exhibited if
2703       *                                 the LDIF reader encounters an entry with
2704       *                                 duplicate values.
2705       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2706       *                                 encountering attribute values which are not
2707       *                                 base64-encoded but contain trailing spaces.
2708       * @param  schema                  The schema to use when parsing the
2709       *                                 attributes, or {@code null} if none is
2710       *                                 needed.
2711       * @param  ldifLines               The lines that comprise the LDIF
2712       *                                 representation of the full record being
2713       *                                 parsed.
2714       * @param  iterator                The iterator to use to access the attribute
2715       *                                 lines.
2716       * @param  relativeBasePath        The base path that will be prepended to
2717       *                                 relative paths in order to obtain an
2718       *                                 absolute path.
2719       * @param  firstLineNumber         The line number for the start of the
2720       *                                 record.
2721       *
2722       * @return  The collection of attributes that were read.
2723       *
2724       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2725       *                         set of attributes.
2726       */
2727      private static ArrayList<Attribute> parseAttributes(final String dn,
2728           final DuplicateValueBehavior duplicateValueBehavior,
2729           final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
2730           final ArrayList<StringBuilder> ldifLines,
2731           final Iterator<StringBuilder> iterator, final String relativeBasePath,
2732           final long firstLineNumber)
2733              throws LDIFException
2734      {
2735        final LinkedHashMap<String,Object> attributes =
2736             new LinkedHashMap<String,Object>(ldifLines.size());
2737        while (iterator.hasNext())
2738        {
2739          final StringBuilder line = iterator.next();
2740          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
2741          final int colonPos = line.indexOf(":");
2742          if (colonPos <= 0)
2743          {
2744            throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
2745                                    firstLineNumber, true, ldifLines, null);
2746          }
2747    
2748          final String attributeName = line.substring(0, colonPos);
2749          final String lowerName     = toLowerCase(attributeName);
2750    
2751          final MatchingRule matchingRule;
2752          if (schema == null)
2753          {
2754            matchingRule = CaseIgnoreStringMatchingRule.getInstance();
2755          }
2756          else
2757          {
2758            matchingRule =
2759                 MatchingRule.selectEqualityMatchingRule(attributeName, schema);
2760          }
2761    
2762          Attribute attr;
2763          final LDIFAttribute ldifAttr;
2764          final Object attrObject = attributes.get(lowerName);
2765          if (attrObject == null)
2766          {
2767            attr     = null;
2768            ldifAttr = null;
2769          }
2770          else
2771          {
2772            if (attrObject instanceof Attribute)
2773            {
2774              attr     = (Attribute) attrObject;
2775              ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
2776                                           attr.getRawValues()[0]);
2777              attributes.put(lowerName, ldifAttr);
2778            }
2779            else
2780            {
2781              attr     = null;
2782              ldifAttr = (LDIFAttribute) attrObject;
2783            }
2784          }
2785    
2786          final int length = line.length();
2787          if (length == (colonPos+1))
2788          {
2789            // This means that the attribute has a zero-length value, which is
2790            // acceptable.
2791            if (attrObject == null)
2792            {
2793              attr = new Attribute(attributeName, matchingRule, "");
2794              attributes.put(lowerName, attr);
2795            }
2796            else
2797            {
2798              try
2799              {
2800                if (! ldifAttr.addValue(new ASN1OctetString(),
2801                           duplicateValueBehavior))
2802                {
2803                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2804                  {
2805                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2806                         firstLineNumber, attributeName), firstLineNumber, true,
2807                         ldifLines, null);
2808                  }
2809                }
2810              }
2811              catch (LDAPException le)
2812              {
2813                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2814                     firstLineNumber, attributeName, getExceptionMessage(le)),
2815                     firstLineNumber, true, ldifLines, le);
2816              }
2817            }
2818          }
2819          else if (line.charAt(colonPos+1) == ':')
2820          {
2821            // Skip over any spaces leading up to the value, and then the rest of
2822            // the string is the base64-encoded attribute value.
2823            int pos = colonPos+2;
2824            while ((pos < length) && (line.charAt(pos) == ' '))
2825            {
2826              pos++;
2827            }
2828    
2829            try
2830            {
2831              final byte[] valueBytes = Base64.decode(line.substring(pos));
2832              if (attrObject == null)
2833              {
2834                attr = new Attribute(attributeName, matchingRule, valueBytes);
2835                attributes.put(lowerName, attr);
2836              }
2837              else
2838              {
2839                try
2840                {
2841                  if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
2842                             duplicateValueBehavior))
2843                  {
2844                    if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2845                    {
2846                      throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2847                           firstLineNumber, attributeName), firstLineNumber, true,
2848                           ldifLines, null);
2849                    }
2850                  }
2851                }
2852                catch (LDAPException le)
2853                {
2854                  throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2855                       firstLineNumber, attributeName, getExceptionMessage(le)),
2856                       firstLineNumber, true, ldifLines, le);
2857                }
2858              }
2859            }
2860            catch (final ParseException pe)
2861            {
2862              debugException(pe);
2863              throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
2864                                           attributeName,  firstLineNumber,
2865                                           pe.getMessage()),
2866                                      firstLineNumber, true, ldifLines, pe);
2867            }
2868          }
2869          else if (line.charAt(colonPos+1) == '<')
2870          {
2871            // Skip over any spaces leading up to the value, and then the rest of
2872            // the string is a URL that indicates where to get the real content.
2873            // At the present time, we'll only support the file URLs.
2874            int pos = colonPos+2;
2875            while ((pos < length) && (line.charAt(pos) == ' '))
2876            {
2877              pos++;
2878            }
2879    
2880            final byte[] urlBytes;
2881            final String urlString = line.substring(pos);
2882            try
2883            {
2884              urlBytes =
2885                   retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
2886            }
2887            catch (final Exception e)
2888            {
2889              debugException(e);
2890              throw new LDIFException(
2891                   ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2892                        firstLineNumber, e),
2893                   firstLineNumber, true, ldifLines, e);
2894            }
2895    
2896            if (attrObject == null)
2897            {
2898              attr = new Attribute(attributeName, matchingRule, urlBytes);
2899              attributes.put(lowerName, attr);
2900            }
2901            else
2902            {
2903              try
2904              {
2905                if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
2906                     duplicateValueBehavior))
2907                {
2908                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2909                  {
2910                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2911                         firstLineNumber, attributeName), firstLineNumber, true,
2912                         ldifLines, null);
2913                  }
2914                }
2915              }
2916              catch (final LDIFException le)
2917              {
2918                debugException(le);
2919                throw le;
2920              }
2921              catch (final Exception e)
2922              {
2923                debugException(e);
2924                throw new LDIFException(
2925                     ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2926                          firstLineNumber, e),
2927                     firstLineNumber, true, ldifLines, e);
2928              }
2929            }
2930          }
2931          else
2932          {
2933            // Skip over any spaces leading up to the value, and then the rest of
2934            // the string is the value.
2935            int pos = colonPos+1;
2936            while ((pos < length) && (line.charAt(pos) == ' '))
2937            {
2938              pos++;
2939            }
2940    
2941            final String valueString = line.substring(pos);
2942            if (attrObject == null)
2943            {
2944              attr = new Attribute(attributeName, matchingRule, valueString);
2945              attributes.put(lowerName, attr);
2946            }
2947            else
2948            {
2949              try
2950              {
2951                if (! ldifAttr.addValue(new ASN1OctetString(valueString),
2952                           duplicateValueBehavior))
2953                {
2954                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2955                  {
2956                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2957                         firstLineNumber, attributeName), firstLineNumber, true,
2958                         ldifLines, null);
2959                  }
2960                }
2961              }
2962              catch (LDAPException le)
2963              {
2964                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2965                     firstLineNumber, attributeName, getExceptionMessage(le)),
2966                     firstLineNumber, true, ldifLines, le);
2967              }
2968            }
2969          }
2970        }
2971    
2972        final ArrayList<Attribute> attrList =
2973             new ArrayList<Attribute>(attributes.size());
2974        for (final Object o : attributes.values())
2975        {
2976          if (o instanceof Attribute)
2977          {
2978            attrList.add((Attribute) o);
2979          }
2980          else
2981          {
2982            attrList.add(((LDIFAttribute) o).toAttribute());
2983          }
2984        }
2985    
2986        return attrList;
2987      }
2988    
2989    
2990    
2991      /**
2992       * Retrieves the bytes that make up the file referenced by the given URL.
2993       *
2994       * @param  urlString         The string representation of the URL to retrieve.
2995       * @param  relativeBasePath  The base path that will be prepended to relative
2996       *                           paths in order to obtain an absolute path.
2997       * @param  firstLineNumber   The line number for the start of the record.
2998       *
2999       * @return  The bytes contained in the specified file, or an empty array if
3000       *          the specified file is empty.
3001       *
3002       * @throws  LDIFException  If the provided URL is malformed or references a
3003       *                         nonexistent file.
3004       *
3005       * @throws  IOException  If a problem is encountered while attempting to read
3006       *                       from the target file.
3007       */
3008      private static byte[] retrieveURLBytes(final String urlString,
3009                                             final String relativeBasePath,
3010                                             final long firstLineNumber)
3011              throws LDIFException, IOException
3012      {
3013        int pos;
3014        String path;
3015        final String lowerURLString = toLowerCase(urlString);
3016        if (lowerURLString.startsWith("file:/"))
3017        {
3018          pos = 6;
3019          while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3020          {
3021            pos++;
3022          }
3023    
3024          path = urlString.substring(pos-1);
3025        }
3026        else if (lowerURLString.startsWith("file:"))
3027        {
3028          // A file: URL that doesn't include a slash will be interpreted as a
3029          // relative path.
3030          path = relativeBasePath + urlString.substring(5);
3031        }
3032        else
3033        {
3034          throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3035               firstLineNumber, true);
3036        }
3037    
3038        final File f = new File(path);
3039        if (! f.exists())
3040        {
3041          throw new LDIFException(
3042               ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3043               firstLineNumber, true);
3044        }
3045    
3046        // In order to conserve memory, we'll only allow values to be read from
3047        // files no larger than 10 megabytes.
3048        final long fileSize = f.length();
3049        if (fileSize > (10 * 1024 * 1024))
3050        {
3051          throw new LDIFException(
3052               ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3053                    (10*1024*1024)),
3054               firstLineNumber, true);
3055        }
3056    
3057        int fileBytesRemaining = (int) fileSize;
3058        final byte[] fileData = new byte[(int) fileSize];
3059        final FileInputStream fis = new FileInputStream(f);
3060        try
3061        {
3062          int fileBytesRead = 0;
3063          while (fileBytesRead < fileSize)
3064          {
3065            final int bytesRead =
3066                 fis.read(fileData, fileBytesRead, fileBytesRemaining);
3067            if (bytesRead < 0)
3068            {
3069              // We hit the end of the file before we expected to.  This shouldn't
3070              // happen unless the file size changed since we first looked at it,
3071              // which we won't allow.
3072              throw new LDIFException(
3073                   ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3074                        f.getAbsolutePath()),
3075                   firstLineNumber, true);
3076            }
3077    
3078            fileBytesRead      += bytesRead;
3079            fileBytesRemaining -= bytesRead;
3080          }
3081    
3082          if (fis.read() != -1)
3083          {
3084            // There is still more data to read.  This shouldn't happen unless the
3085            // file size changed since we first looked at it, which we won't allow.
3086            throw new LDIFException(
3087                 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3088                 firstLineNumber, true);
3089          }
3090        }
3091        finally
3092        {
3093          fis.close();
3094        }
3095    
3096        return fileData;
3097      }
3098    
3099    
3100    
3101      /**
3102       * Parses the data available through the provided iterator into an array of
3103       * modifications suitable for use in a modify change record.
3104       *
3105       * @param  dn                     The DN of the entry being parsed.
3106       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3107       *                                encountering attribute values which are not
3108       *                                base64-encoded but contain trailing spaces.
3109       * @param  ldifLines              The lines that comprise the LDIF
3110       *                                representation of the full record being
3111       *                                parsed.
3112       * @param  iterator               The iterator to use to access the
3113       *                                modification data.
3114       * @param  firstLineNumber        The line number for the start of the record.
3115       *
3116       * @return  An array containing the modifications that were read.
3117       *
3118       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3119       *                         set of modifications.
3120       */
3121      private static Modification[] parseModifications(final String dn,
3122           final TrailingSpaceBehavior trailingSpaceBehavior,
3123           final ArrayList<StringBuilder> ldifLines,
3124           final Iterator<StringBuilder> iterator, final long firstLineNumber)
3125           throws LDIFException
3126      {
3127        final ArrayList<Modification> modList =
3128             new ArrayList<Modification>(ldifLines.size());
3129    
3130        while (iterator.hasNext())
3131        {
3132          // The first line must start with "add:", "delete:", "replace:", or
3133          // "increment:" followed by an attribute name.
3134          StringBuilder line = iterator.next();
3135          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3136          int colonPos = line.indexOf(":");
3137          if (colonPos < 0)
3138          {
3139            throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3140                                    firstLineNumber, true, ldifLines, null);
3141          }
3142    
3143          final ModificationType modType;
3144          final String modTypeStr = toLowerCase(line.substring(0, colonPos));
3145          if (modTypeStr.equals("add"))
3146          {
3147            modType = ModificationType.ADD;
3148          }
3149          else if (modTypeStr.equals("delete"))
3150          {
3151            modType = ModificationType.DELETE;
3152          }
3153          else if (modTypeStr.equals("replace"))
3154          {
3155            modType = ModificationType.REPLACE;
3156          }
3157          else if (modTypeStr.equals("increment"))
3158          {
3159            modType = ModificationType.INCREMENT;
3160          }
3161          else
3162          {
3163            throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3164                                         firstLineNumber),
3165                                    firstLineNumber, true, ldifLines, null);
3166          }
3167    
3168          final String attributeName;
3169          int length = line.length();
3170          if (length == (colonPos+1))
3171          {
3172            // The colon was the last character on the line.  This is not
3173            // acceptable.
3174            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3175                                         firstLineNumber),
3176                                    firstLineNumber, true, ldifLines, null);
3177          }
3178          else if (line.charAt(colonPos+1) == ':')
3179          {
3180            // Skip over any spaces leading up to the value, and then the rest of
3181            // the string is the base64-encoded attribute name.
3182            int pos = colonPos+2;
3183            while ((pos < length) && (line.charAt(pos) == ' '))
3184            {
3185              pos++;
3186            }
3187    
3188            try
3189            {
3190              final byte[] dnBytes = Base64.decode(line.substring(pos));
3191              attributeName = new String(dnBytes, "UTF-8");
3192            }
3193            catch (final ParseException pe)
3194            {
3195              debugException(pe);
3196              throw new LDIFException(
3197                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3198                        firstLineNumber, pe.getMessage()),
3199                   firstLineNumber, true, ldifLines, pe);
3200            }
3201            catch (final Exception e)
3202            {
3203              debugException(e);
3204              throw new LDIFException(
3205                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3206                        firstLineNumber, e),
3207                   firstLineNumber, true, ldifLines, e);
3208            }
3209          }
3210          else
3211          {
3212            // Skip over any spaces leading up to the value, and then the rest of
3213            // the string is the attribute name.
3214            int pos = colonPos+1;
3215            while ((pos < length) && (line.charAt(pos) == ' '))
3216            {
3217              pos++;
3218            }
3219    
3220            attributeName = line.substring(pos);
3221          }
3222    
3223          if (attributeName.length() == 0)
3224          {
3225            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3226                                         firstLineNumber),
3227                                    firstLineNumber, true, ldifLines, null);
3228          }
3229    
3230    
3231          // The next zero or more lines may be the set of attribute values.  Keep
3232          // reading until we reach the end of the iterator or until we find a line
3233          // with just a "-".
3234          final ArrayList<ASN1OctetString> valueList =
3235               new ArrayList<ASN1OctetString>(ldifLines.size());
3236          while (iterator.hasNext())
3237          {
3238            line = iterator.next();
3239            handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3240            if (line.toString().equals("-"))
3241            {
3242              break;
3243            }
3244    
3245            colonPos = line.indexOf(":");
3246            if (colonPos < 0)
3247            {
3248              throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3249                                      firstLineNumber, true, ldifLines, null);
3250            }
3251            else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3252            {
3253              throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3254                                           firstLineNumber,
3255                                           line.substring(0, colonPos),
3256                                           attributeName),
3257                                      firstLineNumber, true, ldifLines, null);
3258            }
3259    
3260            final ASN1OctetString value;
3261            length = line.length();
3262            if (length == (colonPos+1))
3263            {
3264              // The colon was the last character on the line.  This is fine.
3265              value = new ASN1OctetString();
3266            }
3267            else if (line.charAt(colonPos+1) == ':')
3268            {
3269              // Skip over any spaces leading up to the value, and then the rest of
3270              // the string is the base64-encoded value.  This is unusual and
3271              // unnecessary, but is nevertheless acceptable.
3272              int pos = colonPos+2;
3273              while ((pos < length) && (line.charAt(pos) == ' '))
3274              {
3275                pos++;
3276              }
3277    
3278              try
3279              {
3280                value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3281              }
3282              catch (final ParseException pe)
3283              {
3284                debugException(pe);
3285                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3286                     attributeName, firstLineNumber, pe.getMessage()),
3287                     firstLineNumber, true, ldifLines, pe);
3288              }
3289              catch (final Exception e)
3290              {
3291                debugException(e);
3292                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3293                                             firstLineNumber, e),
3294                                        firstLineNumber, true, ldifLines, e);
3295              }
3296            }
3297            else
3298            {
3299              // Skip over any spaces leading up to the value, and then the rest of
3300              // the string is the value.
3301              int pos = colonPos+1;
3302              while ((pos < length) && (line.charAt(pos) == ' '))
3303              {
3304                pos++;
3305              }
3306    
3307              value = new ASN1OctetString(line.substring(pos));
3308            }
3309    
3310            valueList.add(value);
3311          }
3312    
3313          final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3314          valueList.toArray(values);
3315    
3316          // If it's an add modification type, then there must be at least one
3317          // value.
3318          if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3319              (values.length == 0))
3320          {
3321            throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3322                                         firstLineNumber),
3323                                    firstLineNumber, true, ldifLines, null);
3324          }
3325    
3326          // If it's an increment modification type, then there must be exactly one
3327          // value.
3328          if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3329              (values.length != 1))
3330          {
3331            throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3332                                         firstLineNumber, attributeName),
3333                                    firstLineNumber, true, ldifLines, null);
3334          }
3335    
3336          modList.add(new Modification(modType, attributeName, values));
3337        }
3338    
3339        final Modification[] mods = new Modification[modList.size()];
3340        modList.toArray(mods);
3341        return mods;
3342      }
3343    
3344    
3345    
3346      /**
3347       * Parses the data available through the provided iterator as the body of a
3348       * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3349       * newsuperior lines).
3350       *
3351       * @param  ldifLines              The lines that comprise the LDIF
3352       *                                representation of the full record being
3353       *                                parsed.
3354       * @param  iterator               The iterator to use to access the modify DN
3355       *                                data.
3356       * @param  dn                     The current DN of the entry.
3357       * @param  controls               The set of controls to include in the change
3358       *                                record.
3359       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3360       *                                encountering attribute values which are not
3361       *                                base64-encoded but contain trailing spaces.
3362       * @param  firstLineNumber        The line number for the start of the record.
3363       *
3364       * @return  The decoded modify DN change record.
3365       *
3366       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3367       *                         modify DN change record.
3368       */
3369      private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3370           final ArrayList<StringBuilder> ldifLines,
3371           final Iterator<StringBuilder> iterator, final String dn,
3372           final List<Control> controls,
3373           final TrailingSpaceBehavior trailingSpaceBehavior,
3374           final long firstLineNumber)
3375           throws LDIFException
3376      {
3377        // The next line must be the new RDN, and it must start with "newrdn:".
3378        StringBuilder line = iterator.next();
3379        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3380        int colonPos = line.indexOf(":");
3381        if ((colonPos < 0) ||
3382            (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3383        {
3384          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3385                                       firstLineNumber),
3386                                  firstLineNumber, true, ldifLines, null);
3387        }
3388    
3389        final String newRDN;
3390        int length = line.length();
3391        if (length == (colonPos+1))
3392        {
3393          // The colon was the last character on the line.  This is not acceptable.
3394          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3395                                       firstLineNumber),
3396                                  firstLineNumber, true, ldifLines, null);
3397        }
3398        else if (line.charAt(colonPos+1) == ':')
3399        {
3400          // Skip over any spaces leading up to the value, and then the rest of the
3401          // string is the base64-encoded new RDN.
3402          int pos = colonPos+2;
3403          while ((pos < length) && (line.charAt(pos) == ' '))
3404          {
3405            pos++;
3406          }
3407    
3408          try
3409          {
3410            final byte[] dnBytes = Base64.decode(line.substring(pos));
3411            newRDN = new String(dnBytes, "UTF-8");
3412          }
3413          catch (final ParseException pe)
3414          {
3415            debugException(pe);
3416            throw new LDIFException(
3417                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3418                                                                   pe.getMessage()),
3419                 firstLineNumber, true, ldifLines, pe);
3420          }
3421          catch (final Exception e)
3422          {
3423            debugException(e);
3424            throw new LDIFException(
3425                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3426                                                                   e),
3427                 firstLineNumber, true, ldifLines, e);
3428          }
3429        }
3430        else
3431        {
3432          // Skip over any spaces leading up to the value, and then the rest of the
3433          // string is the new RDN.
3434          int pos = colonPos+1;
3435          while ((pos < length) && (line.charAt(pos) == ' '))
3436          {
3437            pos++;
3438          }
3439    
3440          newRDN = line.substring(pos);
3441        }
3442    
3443        if (newRDN.length() == 0)
3444        {
3445          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3446                                       firstLineNumber),
3447                                  firstLineNumber, true, ldifLines, null);
3448        }
3449    
3450    
3451        // The next line must be the deleteOldRDN flag, and it must start with
3452        // 'deleteoldrdn:'.
3453        if (! iterator.hasNext())
3454        {
3455          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3456                                       firstLineNumber),
3457                                  firstLineNumber, true, ldifLines, null);
3458        }
3459    
3460        line = iterator.next();
3461        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3462        colonPos = line.indexOf(":");
3463        if ((colonPos < 0) ||
3464            (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3465        {
3466          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3467                                       firstLineNumber),
3468                                  firstLineNumber, true, ldifLines, null);
3469        }
3470    
3471        final String deleteOldRDNStr;
3472        length = line.length();
3473        if (length == (colonPos+1))
3474        {
3475          // The colon was the last character on the line.  This is not acceptable.
3476          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3477                                       firstLineNumber),
3478                                  firstLineNumber, true, ldifLines, null);
3479        }
3480        else if (line.charAt(colonPos+1) == ':')
3481        {
3482          // Skip over any spaces leading up to the value, and then the rest of the
3483          // string is the base64-encoded value.  This is unusual and
3484          // unnecessary, but is nevertheless acceptable.
3485          int pos = colonPos+2;
3486          while ((pos < length) && (line.charAt(pos) == ' '))
3487          {
3488            pos++;
3489          }
3490    
3491          try
3492          {
3493            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3494            deleteOldRDNStr = new String(changeTypeBytes, "UTF-8");
3495          }
3496          catch (final ParseException pe)
3497          {
3498            debugException(pe);
3499            throw new LDIFException(
3500                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3501                      firstLineNumber, pe.getMessage()),
3502                 firstLineNumber, true, ldifLines, pe);
3503          }
3504          catch (final Exception e)
3505          {
3506            debugException(e);
3507            throw new LDIFException(
3508                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3509                      firstLineNumber, e),
3510                 firstLineNumber, true, ldifLines, e);
3511          }
3512        }
3513        else
3514        {
3515          // Skip over any spaces leading up to the value, and then the rest of the
3516          // string is the value.
3517          int pos = colonPos+1;
3518          while ((pos < length) && (line.charAt(pos) == ' '))
3519          {
3520            pos++;
3521          }
3522    
3523          deleteOldRDNStr = line.substring(pos);
3524        }
3525    
3526        final boolean deleteOldRDN;
3527        if (deleteOldRDNStr.equals("0"))
3528        {
3529          deleteOldRDN = false;
3530        }
3531        else if (deleteOldRDNStr.equals("1"))
3532        {
3533          deleteOldRDN = true;
3534        }
3535        else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3536                 deleteOldRDNStr.equalsIgnoreCase("no"))
3537        {
3538          // This is technically illegal, but we'll allow it.
3539          deleteOldRDN = false;
3540        }
3541        else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3542                 deleteOldRDNStr.equalsIgnoreCase("yes"))
3543        {
3544          // This is also technically illegal, but we'll allow it.
3545          deleteOldRDN = false;
3546        }
3547        else
3548        {
3549          throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3550                                       deleteOldRDNStr, firstLineNumber),
3551                                  firstLineNumber, true, ldifLines, null);
3552        }
3553    
3554    
3555        // If there is another line, then it must be the new superior DN and it must
3556        // start with "newsuperior:".  If this is absent, then it's fine.
3557        final String newSuperiorDN;
3558        if (iterator.hasNext())
3559        {
3560          line = iterator.next();
3561          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3562          colonPos = line.indexOf(":");
3563          if ((colonPos < 0) ||
3564              (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3565          {
3566            throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3567                                         firstLineNumber),
3568                                    firstLineNumber, true, ldifLines, null);
3569          }
3570    
3571          length = line.length();
3572          if (length == (colonPos+1))
3573          {
3574            // The colon was the last character on the line.  This is fine.
3575            newSuperiorDN = "";
3576          }
3577          else if (line.charAt(colonPos+1) == ':')
3578          {
3579            // Skip over any spaces leading up to the value, and then the rest of
3580            // the string is the base64-encoded new superior DN.
3581            int pos = colonPos+2;
3582            while ((pos < length) && (line.charAt(pos) == ' '))
3583            {
3584              pos++;
3585            }
3586    
3587            try
3588            {
3589              final byte[] dnBytes = Base64.decode(line.substring(pos));
3590              newSuperiorDN = new String(dnBytes, "UTF-8");
3591            }
3592            catch (final ParseException pe)
3593            {
3594              debugException(pe);
3595              throw new LDIFException(
3596                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3597                        firstLineNumber, pe.getMessage()),
3598                   firstLineNumber, true, ldifLines, pe);
3599            }
3600            catch (final Exception e)
3601            {
3602              debugException(e);
3603              throw new LDIFException(
3604                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3605                        firstLineNumber, e),
3606                   firstLineNumber, true, ldifLines, e);
3607            }
3608          }
3609          else
3610          {
3611            // Skip over any spaces leading up to the value, and then the rest of
3612            // the string is the new superior DN.
3613            int pos = colonPos+1;
3614            while ((pos < length) && (line.charAt(pos) == ' '))
3615            {
3616              pos++;
3617            }
3618    
3619            newSuperiorDN = line.substring(pos);
3620          }
3621        }
3622        else
3623        {
3624          newSuperiorDN = null;
3625        }
3626    
3627    
3628        // There must not be any more lines.
3629        if (iterator.hasNext())
3630        {
3631          throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
3632                                  firstLineNumber, true, ldifLines, null);
3633        }
3634    
3635        return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
3636             newSuperiorDN, controls);
3637      }
3638    
3639    
3640    
3641      /**
3642       * Examines the line contained in the provided buffer to determine whether it
3643       * may contain one or more illegal trailing spaces.  If it does, then those
3644       * spaces will either be stripped out or an exception will be thrown to
3645       * indicate that they are illegal.
3646       *
3647       * @param  buffer                 The buffer to be examined.
3648       * @param  dn                     The DN of the LDIF record being parsed.  It
3649       *                                may be {@code null} if the DN is not yet
3650       *                                known (e.g., because the provided line is
3651       *                                expected to contain that DN).
3652       * @param  firstLineNumber        The approximate line number in the LDIF
3653       *                                source on which the LDIF record begins.
3654       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3655       *                                encountering attribute values which are not
3656       *                                base64-encoded but contain trailing spaces.
3657       *
3658       * @throws  LDIFException  If the line contained in the provided buffer ends
3659       *                         with one or more illegal trailing spaces and
3660       *                         {@code stripTrailingSpaces} was provided with a
3661       *                         value of {@code false}.
3662       */
3663      private static void handleTrailingSpaces(final StringBuilder buffer,
3664                               final String dn, final long firstLineNumber,
3665                               final TrailingSpaceBehavior trailingSpaceBehavior)
3666              throws LDIFException
3667      {
3668        int pos = buffer.length() - 1;
3669        boolean trailingFound = false;
3670        while ((pos >= 0) && (buffer.charAt(pos) == ' '))
3671        {
3672          trailingFound = true;
3673          pos--;
3674        }
3675    
3676        if (trailingFound && (buffer.charAt(pos) != ':'))
3677        {
3678          switch (trailingSpaceBehavior)
3679          {
3680            case STRIP:
3681              buffer.setLength(pos+1);
3682              break;
3683    
3684            case REJECT:
3685              if (dn == null)
3686              {
3687                throw new LDIFException(
3688                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
3689                          buffer.toString()),
3690                     firstLineNumber, true);
3691              }
3692              else
3693              {
3694                throw new LDIFException(
3695                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
3696                          firstLineNumber, buffer.toString()),
3697                     firstLineNumber, true);
3698              }
3699    
3700            case RETAIN:
3701            default:
3702              // No action will be taken.
3703              break;
3704          }
3705        }
3706      }
3707    
3708    
3709    
3710      /**
3711       * This represents an unparsed LDIFRecord.  It stores the line number of the
3712       * first line of the record and each line of the record.
3713       */
3714      private static final class UnparsedLDIFRecord
3715      {
3716        private final ArrayList<StringBuilder> lineList;
3717        private final long firstLineNumber;
3718        private final Exception failureCause;
3719        private final boolean isEOF;
3720        private final DuplicateValueBehavior duplicateValueBehavior;
3721        private final Schema schema;
3722        private final TrailingSpaceBehavior trailingSpaceBehavior;
3723    
3724    
3725    
3726        /**
3727         * Constructor.
3728         *
3729         * @param  lineList                The lines that comprise the LDIF record.
3730         * @param  duplicateValueBehavior  The behavior to exhibit if the entry
3731         *                                 contains duplicate attribute values.
3732         * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
3733         *                                 encountering trailing spaces in
3734         *                                 non-base64-encoded attribute values.
3735         * @param  schema                  The schema to use when parsing, if
3736         *                                 applicable.
3737         * @param  firstLineNumber         The first line number of the LDIF record.
3738         */
3739        private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
3740                     final DuplicateValueBehavior duplicateValueBehavior,
3741                     final TrailingSpaceBehavior trailingSpaceBehavior,
3742                     final Schema schema, final long firstLineNumber)
3743        {
3744          this.lineList               = lineList;
3745          this.firstLineNumber        = firstLineNumber;
3746          this.duplicateValueBehavior = duplicateValueBehavior;
3747          this.trailingSpaceBehavior  = trailingSpaceBehavior;
3748          this.schema                 = schema;
3749    
3750          failureCause = null;
3751          isEOF =
3752               (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
3753        }
3754    
3755    
3756    
3757        /**
3758         * Constructor.
3759         *
3760         * @param failureCause  The Exception thrown when reading from the input.
3761         */
3762        private UnparsedLDIFRecord(final Exception failureCause)
3763        {
3764          this.failureCause = failureCause;
3765    
3766          lineList               = null;
3767          firstLineNumber        = 0;
3768          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
3769          trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
3770          schema                 = null;
3771          isEOF                  = false;
3772        }
3773    
3774    
3775    
3776        /**
3777         * Return the lines that comprise the LDIF record.
3778         *
3779         * @return  The lines that comprise the LDIF record.
3780         */
3781        private ArrayList<StringBuilder> getLineList()
3782        {
3783          return lineList;
3784        }
3785    
3786    
3787    
3788        /**
3789         * Retrieves the behavior to exhibit when encountering duplicate attribute
3790         * values.
3791         *
3792         * @return  The behavior to exhibit when encountering duplicate attribute
3793         *          values.
3794         */
3795        private DuplicateValueBehavior getDuplicateValueBehavior()
3796        {
3797          return duplicateValueBehavior;
3798        }
3799    
3800    
3801    
3802        /**
3803         * Retrieves the behavior that should be exhibited when encountering
3804         * attribute values which are not base64-encoded but contain trailing
3805         * spaces.  The LDIF specification strongly recommends that any value which
3806         * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
3807         * LDIF parser may be configured to automatically strip these spaces, to
3808         * preserve them, or to reject any entry or change record containing them.
3809         *
3810         * @return  The behavior that should be exhibited when encountering
3811         *          attribute values which are not base64-encoded but contain
3812         *          trailing spaces.
3813         */
3814        private TrailingSpaceBehavior getTrailingSpaceBehavior()
3815        {
3816          return trailingSpaceBehavior;
3817        }
3818    
3819    
3820    
3821        /**
3822         * Retrieves the schema that should be used when parsing the record, if
3823         * applicable.
3824         *
3825         * @return  The schema that should be used when parsing the record, or
3826         *          {@code null} if none should be used.
3827         */
3828        private Schema getSchema()
3829        {
3830          return schema;
3831        }
3832    
3833    
3834    
3835        /**
3836         * Return the first line number of the LDIF record.
3837         *
3838         * @return  The first line number of the LDIF record.
3839         */
3840        private long getFirstLineNumber()
3841        {
3842          return firstLineNumber;
3843        }
3844    
3845    
3846    
3847        /**
3848         * Return {@code true} iff the end of the input was reached.
3849         *
3850         * @return  {@code true} iff the end of the input was reached.
3851         */
3852        private boolean isEOF()
3853        {
3854          return isEOF;
3855        }
3856    
3857    
3858    
3859        /**
3860         * Returns the reason that reading the record lines failed.  This normally
3861         * is only non-null if something bad happened to the input stream (like
3862         * a disk read error).
3863         *
3864         * @return  The reason that reading the record lines failed.
3865         */
3866        private Exception getFailureCause()
3867        {
3868          return failureCause;
3869        }
3870      }
3871    
3872    
3873      /**
3874       * When processing in asynchronous mode, this thread is responsible for
3875       * reading the raw unparsed records from the input and submitting them for
3876       * processing.
3877       */
3878      private final class LineReaderThread
3879           extends Thread
3880      {
3881        /**
3882         * Constructor.
3883         */
3884        private LineReaderThread()
3885        {
3886          super("Asynchronous LDIF line reader");
3887          setDaemon(true);
3888        }
3889    
3890    
3891    
3892        /**
3893         * Reads raw, unparsed records from the input and submits them for
3894         * processing until the input is finished or closed.
3895         */
3896        @Override()
3897        public void run()
3898        {
3899          try
3900          {
3901            boolean stopProcessing = false;
3902            while (!stopProcessing)
3903            {
3904              UnparsedLDIFRecord unparsedRecord = null;
3905              try
3906              {
3907                unparsedRecord = readUnparsedRecord();
3908              }
3909              catch (IOException e)
3910              {
3911                debugException(e);
3912                unparsedRecord = new UnparsedLDIFRecord(e);
3913                stopProcessing = true;
3914              }
3915              catch (Exception e)
3916              {
3917                debugException(e);
3918                unparsedRecord = new UnparsedLDIFRecord(e);
3919              }
3920    
3921              try
3922              {
3923                asyncParser.submit(unparsedRecord);
3924              }
3925              catch (InterruptedException e)
3926              {
3927                debugException(e);
3928                // If this thread is interrupted, then someone wants us to stop
3929                // processing, so that's what we'll do.
3930                stopProcessing = true;
3931              }
3932    
3933              if ((unparsedRecord == null) || (unparsedRecord.isEOF()))
3934              {
3935                stopProcessing = true;
3936              }
3937            }
3938          }
3939          finally
3940          {
3941            try
3942            {
3943              asyncParser.shutdown();
3944            }
3945            catch (InterruptedException e)
3946            {
3947              debugException(e);
3948            }
3949            finally
3950            {
3951              asyncParsingComplete.set(true);
3952            }
3953          }
3954        }
3955      }
3956    
3957    
3958    
3959      /**
3960       * Used to parse Records asynchronously.
3961       */
3962      private final class RecordParser implements Processor<UnparsedLDIFRecord,
3963                                                            LDIFRecord>
3964      {
3965        /**
3966         * {@inheritDoc}
3967         */
3968        public LDIFRecord process(final UnparsedLDIFRecord input)
3969               throws LDIFException
3970        {
3971          LDIFRecord record = decodeRecord(input, relativeBasePath);
3972    
3973          if ((record instanceof Entry) && (entryTranslator != null))
3974          {
3975            record = entryTranslator.translate((Entry) record,
3976                 input.getFirstLineNumber());
3977    
3978            if (record == null)
3979            {
3980              record = SKIP_ENTRY;
3981            }
3982          }
3983          return record;
3984        }
3985      }
3986    }