001    /*
002     * Copyright 2007-2014 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2014 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldif;
022    
023    
024    
025    import java.io.BufferedReader;
026    import java.io.BufferedWriter;
027    import java.io.File;
028    import java.io.FileInputStream;
029    import java.io.FileWriter;
030    import java.io.InputStream;
031    import java.io.InputStreamReader;
032    import java.io.IOException;
033    import java.text.ParseException;
034    import java.util.ArrayList;
035    import java.util.Collection;
036    import java.util.Iterator;
037    import java.util.LinkedHashMap;
038    import java.util.List;
039    import java.util.concurrent.BlockingQueue;
040    import java.util.concurrent.ArrayBlockingQueue;
041    import java.util.concurrent.TimeUnit;
042    import java.util.concurrent.atomic.AtomicBoolean;
043    import java.nio.charset.Charset;
044    
045    import com.unboundid.asn1.ASN1OctetString;
046    import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
047    import com.unboundid.ldap.matchingrules.MatchingRule;
048    import com.unboundid.ldap.sdk.Attribute;
049    import com.unboundid.ldap.sdk.Control;
050    import com.unboundid.ldap.sdk.Entry;
051    import com.unboundid.ldap.sdk.Modification;
052    import com.unboundid.ldap.sdk.ModificationType;
053    import com.unboundid.ldap.sdk.LDAPException;
054    import com.unboundid.ldap.sdk.schema.Schema;
055    import com.unboundid.util.AggregateInputStream;
056    import com.unboundid.util.Base64;
057    import com.unboundid.util.LDAPSDKThreadFactory;
058    import com.unboundid.util.parallel.AsynchronousParallelProcessor;
059    import com.unboundid.util.parallel.Result;
060    import com.unboundid.util.parallel.ParallelProcessor;
061    import com.unboundid.util.parallel.Processor;
062    
063    import static com.unboundid.ldif.LDIFMessages.*;
064    import static com.unboundid.util.Debug.*;
065    import static com.unboundid.util.StaticUtils.*;
066    import static com.unboundid.util.Validator.*;
067    
068    /**
069     * This class provides an LDIF reader, which can be used to read and decode
070     * entries and change records from a data source using the LDAP Data Interchange
071     * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
072     * <BR>
073     * This class is not synchronized.  If multiple threads read from the
074     * LDIFReader, they must be synchronized externally.
075     * <BR><BR>
076     * <H2>Example</H2>
077     * The following example iterates through all entries contained in an LDIF file
078     * and attempts to add them to a directory server:
079     * <PRE>
080     * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
081     *
082     * int entriesRead = 0;
083     * int entriesAdded = 0;
084     * int errorsEncountered = 0;
085     * while (true)
086     * {
087     *   Entry entry;
088     *   try
089     *   {
090     *     entry = ldifReader.readEntry();
091     *     if (entry == null)
092     *     {
093     *       // All entries have been read.
094     *       break;
095     *     }
096     *
097     *     entriesRead++;
098     *   }
099     *   catch (LDIFException le)
100     *   {
101     *     errorsEncountered++;
102     *     if (le.mayContinueReading())
103     *     {
104     *       // A recoverable error occurred while attempting to read a change
105     *       // record, at or near line number le.getLineNumber()
106     *       // The entry will be skipped, but we'll try to keep reading from the
107     *       // LDIF file.
108     *       continue;
109     *     }
110     *     else
111     *     {
112     *       // An unrecoverable error occurred while attempting to read an entry
113     *       // at or near line number le.getLineNumber()
114     *       // No further LDIF processing will be performed.
115     *       break;
116     *     }
117     *   }
118     *   catch (IOException ioe)
119     *   {
120     *     // An I/O error occurred while attempting to read from the LDIF file.
121     *     // No further LDIF processing will be performed.
122     *     errorsEncountered++;
123     *     break;
124     *   }
125     *
126     *   LDAPResult addResult;
127     *   try
128     *   {
129     *     addResult = connection.add(entry);
130     *     // If we got here, then the change should have been processed
131     *     // successfully.
132     *     entriesAdded++;
133     *   }
134     *   catch (LDAPException le)
135     *   {
136     *     // If we got here, then the change attempt failed.
137     *     addResult = le.toLDAPResult();
138     *     errorsEncountered++;
139     *   }
140     * }
141     *
142     * ldifReader.close();
143     * </PRE>
144     */
145    public final class LDIFReader
146    {
147      /**
148       * The default buffer size (128KB) that will be used when reading from the
149       * data source.
150       */
151      public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
152    
153    
154    
155      /*
156       * When processing asynchronously, this determines how many of the allocated
157       * worker threads are used to parse each batch of read entries.
158       */
159      private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
160    
161    
162    
163      /**
164       * When processing asynchronously, this specifies the size of the pending and
165       * completed queues.
166       */
167      private static final int ASYNC_QUEUE_SIZE = 500;
168    
169    
170    
171      /**
172       * Special entry used internally to signal that the LDIFReaderEntryTranslator
173       * has signalled that a read Entry should be skipped by returning null,
174       * which normally implies EOF.
175       */
176      private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
177    
178    
179    
180      /**
181       * The default base path that will be prepended to relative paths.  It will
182       * end with a trailing slash.
183       */
184      private static final String DEFAULT_RELATIVE_BASE_PATH;
185      static
186      {
187        final File currentDir;
188        String currentDirString = System.getProperty("user.dir");
189        if (currentDirString == null)
190        {
191          currentDir = new File(".");
192        }
193        else
194        {
195          currentDir = new File(currentDirString);
196        }
197    
198        final String currentDirAbsolutePath = currentDir.getAbsolutePath();
199        if (currentDirAbsolutePath.endsWith(File.separator))
200        {
201          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
202        }
203        else
204        {
205          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
206        }
207      }
208    
209    
210    
211      // The buffered reader that will be used to read LDIF data.
212      private final BufferedReader reader;
213    
214      // The behavior that should be exhibited when encountering duplicate attribute
215      // values.
216      private volatile DuplicateValueBehavior duplicateValueBehavior;
217    
218      // A line number counter.
219      private long lineNumberCounter = 0;
220    
221      private final LDIFReaderEntryTranslator entryTranslator;
222    
223      // The schema that will be used when processing, if applicable.
224      private Schema schema;
225    
226      // Specifies the base path that will be prepended to relative paths for file
227      // URLs.
228      private volatile String relativeBasePath;
229    
230      // The behavior that should be exhibited with regard to illegal trailing
231      // spaces in attribute values.
232      private volatile TrailingSpaceBehavior trailingSpaceBehavior;
233    
234      // True iff we are processing asynchronously.
235      private final boolean isAsync;
236    
237      //
238      // The following only apply to asynchronous processing.
239      //
240    
241      // Parses entries asynchronously.
242      private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord>
243           asyncParser;
244    
245      // Set to true when the end of the input is reached.
246      private final AtomicBoolean asyncParsingComplete;
247    
248      // The records that have been read and parsed.
249      private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>>
250           asyncParsedRecords;
251    
252    
253    
254      /**
255       * Creates a new LDIF reader that will read data from the specified file.
256       *
257       * @param  path  The path to the file from which the data is to be read.  It
258       *               must not be {@code null}.
259       *
260       * @throws  IOException  If a problem occurs while opening the file for
261       *                       reading.
262       */
263      public LDIFReader(final String path)
264             throws IOException
265      {
266        this(new FileInputStream(path));
267      }
268    
269    
270    
271      /**
272       * Creates a new LDIF reader that will read data from the specified file
273       * and parses the LDIF records asynchronously using the specified number of
274       * threads.
275       *
276       * @param  path  The path to the file from which the data is to be read.  It
277       *               must not be {@code null}.
278       * @param  numParseThreads  If this value is greater than zero, then the
279       *                          specified number of threads will be used to
280       *                          asynchronously read and parse the LDIF file.
281       *
282       * @throws  IOException  If a problem occurs while opening the file for
283       *                       reading.
284       *
285       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
286       *      constructor for more details about asynchronous processing.
287       */
288      public LDIFReader(final String path, final int numParseThreads)
289             throws IOException
290      {
291        this(new FileInputStream(path), numParseThreads);
292      }
293    
294    
295    
296      /**
297       * Creates a new LDIF reader that will read data from the specified file.
298       *
299       * @param  file  The file from which the data is to be read.  It must not be
300       *               {@code null}.
301       *
302       * @throws  IOException  If a problem occurs while opening the file for
303       *                       reading.
304       */
305      public LDIFReader(final File file)
306             throws IOException
307      {
308        this(new FileInputStream(file));
309      }
310    
311    
312    
313      /**
314       * Creates a new LDIF reader that will read data from the specified file
315       * and optionally parses the LDIF records asynchronously using the specified
316       * number of threads.
317       *
318       * @param  file             The file from which the data is to be read.  It
319       *                          must not be {@code null}.
320       * @param  numParseThreads  If this value is greater than zero, then the
321       *                          specified number of threads will be used to
322       *                          asynchronously read and parse the LDIF file.
323       *
324       * @throws  IOException  If a problem occurs while opening the file for
325       *                       reading.
326       */
327      public LDIFReader(final File file, final int numParseThreads)
328             throws IOException
329      {
330        this(new FileInputStream(file), numParseThreads);
331      }
332    
333    
334    
335      /**
336       * Creates a new LDIF reader that will read data from the specified files in
337       * the order in which they are provided and optionally parses the LDIF records
338       * asynchronously using the specified number of threads.
339       *
340       * @param  files            The files from which the data is to be read.  It
341       *                          must not be {@code null} or empty.
342       * @param  numParseThreads  If this value is greater than zero, then the
343       *                          specified number of threads will be used to
344       *                          asynchronously read and parse the LDIF file.
345       * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
346       *                          before they are returned.  This is normally
347       *                          {@code null}, which causes entries to be returned
348       *                          unaltered. This is particularly useful when
349       *                          parsing the input file in parallel because the
350       *                          entry translation is also done in parallel.
351       *
352       * @throws  IOException  If a problem occurs while opening the file for
353       *                       reading.
354       */
355      public LDIFReader(final File[] files, final int numParseThreads,
356                        final LDIFReaderEntryTranslator entryTranslator)
357             throws IOException
358      {
359        this(createAggregateInputStream(files), numParseThreads, entryTranslator);
360      }
361    
362    
363    
364      /**
365       * Creates a new aggregate input stream that will read data from the specified
366       * files.  If there are multiple files, then a "padding" file will be inserted
367       * between them to ensure that there is at least one blank line between the
368       * end of one file and the beginning of another.
369       *
370       * @param  files  The files from which the data is to be read.  It must not be
371       *                {@code null} or empty.
372       *
373       * @return  The input stream to use to read data from the provided files.
374       *
375       * @throws  IOException  If a problem is encountered while attempting to
376       *                       create the input stream.
377       */
378      private static InputStream createAggregateInputStream(final File... files)
379              throws IOException
380      {
381        if (files.length == 0)
382        {
383          throw new IOException(ERR_READ_NO_LDIF_FILES.get());
384        }
385        else if (files.length == 1)
386        {
387          return new FileInputStream(files[0]);
388        }
389        else
390        {
391          final File spacerFile =
392               File.createTempFile("ldif-reader-spacer", ".ldif");
393          spacerFile.deleteOnExit();
394    
395          final BufferedWriter spacerWriter =
396               new BufferedWriter(new FileWriter(spacerFile));
397          try
398          {
399            spacerWriter.newLine();
400            spacerWriter.newLine();
401          }
402          finally
403          {
404            spacerWriter.close();
405          }
406    
407          final File[] returnArray = new File[(files.length * 2) - 1];
408          returnArray[0] = files[0];
409    
410          int pos = 1;
411          for (int i=1; i < files.length; i++)
412          {
413            returnArray[pos++] = spacerFile;
414            returnArray[pos++] = files[i];
415          }
416    
417          return new AggregateInputStream(returnArray);
418        }
419      }
420    
421    
422    
423      /**
424       * Creates a new LDIF reader that will read data from the provided input
425       * stream.
426       *
427       * @param  inputStream  The input stream from which the data is to be read.
428       *                      It must not be {@code null}.
429       */
430      public LDIFReader(final InputStream inputStream)
431      {
432        this(inputStream, 0);
433      }
434    
435    
436    
437      /**
438       * Creates a new LDIF reader that will read data from the specified stream
439       * and parses the LDIF records asynchronously using the specified number of
440       * threads.
441       *
442       * @param  inputStream  The input stream from which the data is to be read.
443       *                      It must not be {@code null}.
444       * @param  numParseThreads  If this value is greater than zero, then the
445       *                          specified number of threads will be used to
446       *                          asynchronously read and parse the LDIF file.
447       *
448       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
449       *      constructor for more details about asynchronous processing.
450       */
451      public LDIFReader(final InputStream inputStream, final int numParseThreads)
452      {
453        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
454        this(new BufferedReader(new InputStreamReader(inputStream,
455                                                      Charset.forName("UTF-8")),
456                                DEFAULT_BUFFER_SIZE),
457             numParseThreads);
458      }
459    
460    
461    
462      /**
463       * Creates a new LDIF reader that will read data from the specified stream
464       * and parses the LDIF records asynchronously using the specified number of
465       * threads.
466       *
467       * @param  inputStream  The input stream from which the data is to be read.
468       *                      It must not be {@code null}.
469       * @param  numParseThreads  If this value is greater than zero, then the
470       *                          specified number of threads will be used to
471       *                          asynchronously read and parse the LDIF file.
472       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
473       *                         entries before they are returned.  This is normally
474       *                         {@code null}, which causes entries to be returned
475       *                         unaltered. This is particularly useful when parsing
476       *                         the input file in parallel because the entry
477       *                         translation is also done in parallel.
478       *
479       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
480       *      constructor for more details about asynchronous processing.
481       */
482      public LDIFReader(final InputStream inputStream, final int numParseThreads,
483                        final LDIFReaderEntryTranslator entryTranslator)
484      {
485        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
486        this(new BufferedReader(new InputStreamReader(inputStream,
487                                                      Charset.forName("UTF-8")),
488                                DEFAULT_BUFFER_SIZE),
489             numParseThreads, entryTranslator);
490      }
491    
492    
493    
494      /**
495       * Creates a new LDIF reader that will use the provided buffered reader to
496       * read the LDIF data.  The encoding of the underlying Reader must be set to
497       * "UTF-8" as required by RFC 2849.
498       *
499       * @param  reader  The buffered reader that will be used to read the LDIF
500       *                 data.  It must not be {@code null}.
501       */
502      public LDIFReader(final BufferedReader reader)
503      {
504        this(reader, 0);
505      }
506    
507    
508    
509      /**
510       * Creates a new LDIF reader that will read data from the specified buffered
511       * reader and parses the LDIF records asynchronously using the specified
512       * number of threads.  The encoding of the underlying Reader must be set to
513       * "UTF-8" as required by RFC 2849.
514       *
515       * @param reader The buffered reader that will be used to read the LDIF data.
516       *               It must not be {@code null}.
517       * @param  numParseThreads  If this value is greater than zero, then the
518       *                          specified number of threads will be used to
519       *                          asynchronously read and parse the LDIF file.
520       *
521       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
522       *      constructor for more details about asynchronous processing.
523       */
524      public LDIFReader(final BufferedReader reader, final int numParseThreads)
525      {
526        this(reader, numParseThreads, null);
527      }
528    
529    
530    
531      /**
532       * Creates a new LDIF reader that will read data from the specified buffered
533       * reader and parses the LDIF records asynchronously using the specified
534       * number of threads.  The encoding of the underlying Reader must be set to
535       * "UTF-8" as required by RFC 2849.
536       *
537       * @param reader The buffered reader that will be used to read the LDIF data.
538       *               It must not be {@code null}.
539       * @param  numParseThreads  If this value is greater than zero, then the
540       *                          specified number of threads will be used to
541       *                          asynchronously read and parse the LDIF file.
542       *                          This should only be set to greater than zero when
543       *                          performance analysis has demonstrated that reading
544       *                          and parsing the LDIF is a bottleneck.  The default
545       *                          synchronous processing is normally fast enough.
546       *                          There is little benefit in passing in a value
547       *                          greater than four (unless there is an
548       *                          LDIFReaderEntryTranslator that does time-consuming
549       *                          processing).  A value of zero implies the
550       *                          default behavior of reading and parsing LDIF
551       *                          records synchronously when one of the read
552       *                          methods is called.
553       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
554       *                         entries before they are returned.  This is normally
555       *                         {@code null}, which causes entries to be returned
556       *                         unaltered. This is particularly useful when parsing
557       *                         the input file in parallel because the entry
558       *                         translation is also done in parallel.
559       */
560      public LDIFReader(final BufferedReader reader,
561                        final int numParseThreads,
562                        final LDIFReaderEntryTranslator entryTranslator)
563      {
564        ensureNotNull(reader);
565        ensureTrue(numParseThreads >= 0,
566                   "LDIFReader.numParseThreads must not be negative.");
567    
568        this.reader = reader;
569        this.entryTranslator = entryTranslator;
570    
571        duplicateValueBehavior = DuplicateValueBehavior.STRIP;
572        trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
573    
574        relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
575    
576        if (numParseThreads == 0)
577        {
578          isAsync = false;
579          asyncParser = null;
580          asyncParsingComplete = null;
581          asyncParsedRecords = null;
582        }
583        else
584        {
585          isAsync = true;
586          asyncParsingComplete = new AtomicBoolean(false);
587    
588          // Decodes entries in parallel.
589          final LDAPSDKThreadFactory threadFactory =
590               new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
591          final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
592               new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>(
593                    new RecordParser(), threadFactory, numParseThreads,
594                    ASYNC_MIN_PER_PARSING_THREAD);
595    
596          final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
597               ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE);
598    
599          // The output queue must be a little more than twice as big as the input
600          // queue to more easily handle being shutdown in the middle of processing
601          // when the queues are full and threads are blocked.
602          asyncParsedRecords = new ArrayBlockingQueue
603               <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100);
604    
605          asyncParser = new AsynchronousParallelProcessor
606               <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser,
607                                                asyncParsedRecords);
608    
609          final LineReaderThread lineReaderThread = new LineReaderThread();
610          lineReaderThread.start();
611        }
612      }
613    
614    
615    
616      /**
617       * Reads entries from the LDIF file with the specified path and returns them
618       * as a {@code List}.  This is a convenience method that should only be used
619       * for data sets that are small enough so that running out of memory isn't a
620       * concern.
621       *
622       * @param  path  The path to the LDIF file containing the entries to be read.
623       *
624       * @return  A list of the entries read from the given LDIF file.
625       *
626       * @throws  IOException  If a problem occurs while attempting to read data
627       *                       from the specified file.
628       *
629       * @throws  LDIFException  If a problem is encountered while attempting to
630       *                         decode data read as LDIF.
631       */
632      public static List<Entry> readEntries(final String path)
633             throws IOException, LDIFException
634      {
635        return readEntries(new LDIFReader(path));
636      }
637    
638    
639    
640      /**
641       * Reads entries from the specified LDIF file and returns them as a
642       * {@code List}.  This is a convenience method that should only be used for
643       * data sets that are small enough so that running out of memory isn't a
644       * concern.
645       *
646       * @param  file  A reference to the LDIF file containing the entries to be
647       *               read.
648       *
649       * @return  A list of the entries read from the given LDIF file.
650       *
651       * @throws  IOException  If a problem occurs while attempting to read data
652       *                       from the specified file.
653       *
654       * @throws  LDIFException  If a problem is encountered while attempting to
655       *                         decode data read as LDIF.
656       */
657      public static List<Entry> readEntries(final File file)
658             throws IOException, LDIFException
659      {
660        return readEntries(new LDIFReader(file));
661      }
662    
663    
664    
665      /**
666       * Reads and decodes LDIF entries from the provided input stream and
667       * returns them as a {@code List}.  This is a convenience method that should
668       * only be used for data sets that are small enough so that running out of
669       * memory isn't a concern.
670       *
671       * @param  inputStream  The input stream from which the entries should be
672       *                      read.  The input stream will be closed before
673       *                      returning.
674       *
675       * @return  A list of the entries read from the given input stream.
676       *
677       * @throws  IOException  If a problem occurs while attempting to read data
678       *                       from the input stream.
679       *
680       * @throws  LDIFException  If a problem is encountered while attempting to
681       *                         decode data read as LDIF.
682       */
683      public static List<Entry> readEntries(final InputStream inputStream)
684             throws IOException, LDIFException
685      {
686        return readEntries(new LDIFReader(inputStream));
687      }
688    
689    
690    
691      /**
692       * Reads entries from the provided LDIF reader and returns them as a list.
693       *
694       * @param  reader  The reader from which the entries should be read.  It will
695       *                 be closed before returning.
696       *
697       * @return  A list of the entries read from the provided reader.
698       *
699       * @throws  IOException  If a problem was encountered while attempting to read
700       *                       data from the LDIF data source.
701       *
702       * @throws  LDIFException  If a problem is encountered while attempting to
703       *                         decode data read as LDIF.
704       */
705      private static List<Entry> readEntries(final LDIFReader reader)
706              throws IOException, LDIFException
707      {
708        try
709        {
710          final ArrayList<Entry> entries = new ArrayList<Entry>(10);
711          while (true)
712          {
713            final Entry e = reader.readEntry();
714            if (e == null)
715            {
716              break;
717            }
718    
719            entries.add(e);
720          }
721    
722          return entries;
723        }
724        finally
725        {
726          reader.close();
727        }
728      }
729    
730    
731    
732      /**
733       * Closes this LDIF reader and the underlying LDIF source.
734       *
735       * @throws  IOException  If a problem occurs while closing the underlying LDIF
736       *                       source.
737       */
738      public void close()
739             throws IOException
740      {
741        reader.close();
742    
743        if (isAsync())
744        {
745          // Closing the reader will trigger the LineReaderThread to complete, but
746          // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
747          // this, we clear out the completed output queue, which is larger than
748          // the input queue, so the LineReaderThread will stop reading and
749          // shutdown the asyncParser.
750          asyncParsedRecords.clear();
751        }
752      }
753    
754    
755    
756      /**
757       * Indicates whether to ignore any duplicate values encountered while reading
758       * LDIF records.
759       *
760       * @return  {@code true} if duplicate values should be ignored, or
761       *          {@code false} if any LDIF records containing duplicate values
762       *          should be rejected.
763       *
764       * @deprecated  Use the {@code getDuplicateValueBehavior} method instead.
765       */
766      @Deprecated()
767      public boolean ignoreDuplicateValues()
768      {
769        return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
770      }
771    
772    
773    
774      /**
775       * Specifies whether to ignore any duplicate values encountered while reading
776       * LDIF records.
777       *
778       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
779       *                                attribute values encountered while reading
780       *                                LDIF records.
781       *
782       * @deprecated  Use the {@code setDuplicateValueBehavior} method instead.
783       */
784      @Deprecated()
785      public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
786      {
787        if (ignoreDuplicateValues)
788        {
789          duplicateValueBehavior = DuplicateValueBehavior.STRIP;
790        }
791        else
792        {
793          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
794        }
795      }
796    
797    
798    
799      /**
800       * Retrieves the behavior that should be exhibited if the LDIF reader
801       * encounters an entry with duplicate values.
802       *
803       * @return  The behavior that should be exhibited if the LDIF reader
804       *          encounters an entry with duplicate values.
805       */
806      public DuplicateValueBehavior getDuplicateValueBehavior()
807      {
808        return duplicateValueBehavior;
809      }
810    
811    
812    
813      /**
814       * Specifies the behavior that should be exhibited if the LDIF reader
815       * encounters an entry with duplicate values.
816       *
817       * @param  duplicateValueBehavior  The behavior that should be exhibited if
818       *                                 the LDIF reader encounters an entry with
819       *                                 duplicate values.
820       */
821      public void setDuplicateValueBehavior(
822                       final DuplicateValueBehavior duplicateValueBehavior)
823      {
824        this.duplicateValueBehavior = duplicateValueBehavior;
825      }
826    
827    
828    
829      /**
830       * Indicates whether to strip off any illegal trailing spaces that may appear
831       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
832       * specification strongly recommends that any value which legitimately
833       * contains trailing spaces be base64-encoded, and any spaces which appear
834       * after the end of non-base64-encoded values may therefore be considered
835       * invalid.  If any such trailing spaces are encountered in an LDIF record and
836       * they are not to be stripped, then an {@code LDIFException} will be thrown
837       * for that record.
838       * <BR><BR>
839       * Note that this applies only to spaces after the end of a value, and not to
840       * spaces which may appear at the end of a line for a value that is wrapped
841       * and continued on the next line.
842       *
843       * @return  {@code true} if illegal trailing spaces should be stripped off, or
844       *          {@code false} if LDIF records containing illegal trailing spaces
845       *          should be rejected.
846       *
847       * @deprecated  Use the {@code getTrailingSpaceBehavior} method instead.
848       */
849      @Deprecated()
850      public boolean stripTrailingSpaces()
851      {
852        return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
853      }
854    
855    
856    
857      /**
858       * Specifies whether to strip off any illegal trailing spaces that may appear
859       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
860       * specification strongly recommends that any value which legitimately
861       * contains trailing spaces be base64-encoded, and any spaces which appear
862       * after the end of non-base64-encoded values may therefore be considered
863       * invalid.  If any such trailing spaces are encountered in an LDIF record and
864       * they are not to be stripped, then an {@code LDIFException} will be thrown
865       * for that record.
866       * <BR><BR>
867       * Note that this applies only to spaces after the end of a value, and not to
868       * spaces which may appear at the end of a line for a value that is wrapped
869       * and continued on the next line.
870       *
871       * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
872       *                              trailing spaces, or {@code false} if LDIF
873       *                              records containing them should be rejected.
874       *
875       * @deprecated  Use the {@code setTrailingSpaceBehavior} method instead.
876       */
877      @Deprecated()
878      public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
879      {
880        trailingSpaceBehavior = stripTrailingSpaces
881             ? TrailingSpaceBehavior.STRIP
882             : TrailingSpaceBehavior.REJECT;
883      }
884    
885    
886    
887      /**
888       * Retrieves the behavior that should be exhibited when encountering attribute
889       * values which are not base64-encoded but contain trailing spaces.  The LDIF
890       * specification strongly recommends that any value which legitimately
891       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
892       * may be configured to automatically strip these spaces, to preserve them, or
893       * to reject any entry or change record containing them.
894       *
895       * @return  The behavior that should be exhibited when encountering attribute
896       *          values which are not base64-encoded but contain trailing spaces.
897       */
898      public TrailingSpaceBehavior getTrailingSpaceBehavior()
899      {
900        return trailingSpaceBehavior;
901      }
902    
903    
904    
905      /**
906       * Specifies the behavior that should be exhibited when encountering attribute
907       * values which are not base64-encoded but contain trailing spaces.  The LDIF
908       * specification strongly recommends that any value which legitimately
909       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
910       * may be configured to automatically strip these spaces, to preserve them, or
911       * to reject any entry or change record containing them.
912       *
913       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
914       *                                encountering attribute values which are not
915       *                                base64-encoded but contain trailing spaces.
916       */
917      public void setTrailingSpaceBehavior(
918                       final TrailingSpaceBehavior trailingSpaceBehavior)
919      {
920        this.trailingSpaceBehavior = trailingSpaceBehavior;
921      }
922    
923    
924    
925      /**
926       * Retrieves the base path that will be prepended to relative paths in order
927       * to obtain an absolute path.  This will only be used for "file:" URLs that
928       * have paths which do not begin with a slash.
929       *
930       * @return  The base path that will be prepended to relative paths in order to
931       *          obtain an absolute path.
932       */
933      public String getRelativeBasePath()
934      {
935        return relativeBasePath;
936      }
937    
938    
939    
940      /**
941       * Specifies the base path that will be prepended to relative paths in order
942       * to obtain an absolute path.  This will only be used for "file:" URLs that
943       * have paths which do not begin with a space.
944       *
945       * @param  relativeBasePath  The base path that will be prepended to relative
946       *                           paths in order to obtain an absolute path.
947       */
948      public void setRelativeBasePath(final String relativeBasePath)
949      {
950        setRelativeBasePath(new File(relativeBasePath));
951      }
952    
953    
954    
955      /**
956       * Specifies the base path that will be prepended to relative paths in order
957       * to obtain an absolute path.  This will only be used for "file:" URLs that
958       * have paths which do not begin with a space.
959       *
960       * @param  relativeBasePath  The base path that will be prepended to relative
961       *                           paths in order to obtain an absolute path.
962       */
963      public void setRelativeBasePath(final File relativeBasePath)
964      {
965        final String path = relativeBasePath.getAbsolutePath();
966        if (path.endsWith(File.separator))
967        {
968          this.relativeBasePath = path;
969        }
970        else
971        {
972          this.relativeBasePath = path + File.separator;
973        }
974      }
975    
976    
977    
978      /**
979       * Retrieves the schema that will be used when reading LDIF records, if
980       * defined.
981       *
982       * @return  The schema that will be used when reading LDIF records, or
983       *          {@code null} if no schema should be used and all attributes should
984       *          be treated as case-insensitive strings.
985       */
986      public Schema getSchema()
987      {
988        return schema;
989      }
990    
991    
992    
993      /**
994       * Specifies the schema that should be used when reading LDIF records.
995       *
996       * @param  schema  The schema that should be used when reading LDIF records,
997       *                 or {@code null} if no schema should be used and all
998       *                 attributes should be treated as case-insensitive strings.
999       */
1000      public void setSchema(final Schema schema)
1001      {
1002        this.schema = schema;
1003      }
1004    
1005    
1006    
1007      /**
1008       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1009       * change record.
1010       *
1011       * @return  The record read from the LDIF source, or {@code null} if there are
1012       *          no more entries to be read.
1013       *
1014       * @throws  IOException  If a problem occurs while trying to read from the
1015       *                       LDIF source.
1016       *
1017       * @throws  LDIFException  If the data read could not be parsed as an entry or
1018       *                         an LDIF change record.
1019       */
1020      public LDIFRecord readLDIFRecord()
1021             throws IOException, LDIFException
1022      {
1023        if (isAsync())
1024        {
1025          return readLDIFRecordAsync();
1026        }
1027        else
1028        {
1029          return readLDIFRecordInternal();
1030        }
1031      }
1032    
1033    
1034    
1035      /**
1036       * Reads an entry from the LDIF source.
1037       *
1038       * @return  The entry read from the LDIF source, or {@code null} if there are
1039       *          no more entries to be read.
1040       *
1041       * @throws  IOException  If a problem occurs while attempting to read from the
1042       *                       LDIF source.
1043       *
1044       * @throws  LDIFException  If the data read could not be parsed as an entry.
1045       */
1046      public Entry readEntry()
1047             throws IOException, LDIFException
1048      {
1049        if (isAsync())
1050        {
1051          return readEntryAsync();
1052        }
1053        else
1054        {
1055          return readEntryInternal();
1056        }
1057      }
1058    
1059    
1060    
1061      /**
1062       * Reads an LDIF change record from the LDIF source.  The LDIF record must
1063       * have a changetype.
1064       *
1065       * @return  The change record read from the LDIF source, or {@code null} if
1066       *          there are no more records to be read.
1067       *
1068       * @throws  IOException  If a problem occurs while attempting to read from the
1069       *                       LDIF source.
1070       *
1071       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1072       *                         change record.
1073       */
1074      public LDIFChangeRecord readChangeRecord()
1075             throws IOException, LDIFException
1076      {
1077        return readChangeRecord(false);
1078      }
1079    
1080    
1081    
1082      /**
1083       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1084       * record does not have a changetype, then it may be assumed to be an add
1085       * change record.
1086       *
1087       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1088       *                     changetype should be retrieved as an add change record.
1089       *                     If this is {@code false} and the record read does not
1090       *                     include a changetype, then an {@code LDIFException}
1091       *                     will be thrown.
1092       *
1093       * @return  The change record read from the LDIF source, or {@code null} if
1094       *          there are no more records to be read.
1095       *
1096       * @throws  IOException  If a problem occurs while attempting to read from the
1097       *                       LDIF source.
1098       *
1099       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1100       *                         change record.
1101       */
1102      public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1103             throws IOException, LDIFException
1104      {
1105        if (isAsync())
1106        {
1107          return readChangeRecordAsync(defaultAdd);
1108        }
1109        else
1110        {
1111          return readChangeRecordInternal(defaultAdd);
1112        }
1113      }
1114    
1115    
1116    
1117      /**
1118       * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1119       * thread.
1120       *
1121       * @return  The next parsed record or {@code null} if there are no more
1122       *          records to read.
1123       *
1124       * @throws IOException  If IOException was thrown when reading or parsing
1125       *                      the record.
1126       *
1127       * @throws LDIFException If LDIFException was thrown parsing the record.
1128       */
1129      private LDIFRecord readLDIFRecordAsync()
1130              throws IOException, LDIFException
1131      {
1132        final Result<UnparsedLDIFRecord, LDIFRecord> result =
1133             readLDIFRecordResultAsync();
1134        if (result == null)
1135        {
1136          return null;
1137        }
1138        else
1139        {
1140          return result.getOutput();
1141        }
1142      }
1143    
1144    
1145    
1146      /**
1147       * Reads an entry asynchronously from the LDIF source.
1148       *
1149       * @return The entry read from the LDIF source, or {@code null} if there are
1150       *         no more entries to be read.
1151       *
1152       * @throws IOException   If a problem occurs while attempting to read from the
1153       *                       LDIF source.
1154       * @throws LDIFException If the data read could not be parsed as an entry.
1155       */
1156      private Entry readEntryAsync()
1157              throws IOException, LDIFException
1158      {
1159        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1160        LDIFRecord record = null;
1161        while (record == null)
1162        {
1163          result = readLDIFRecordResultAsync();
1164          if (result == null)
1165          {
1166            return null;
1167          }
1168    
1169          record = result.getOutput();
1170    
1171          // This is a special value that means we should skip this Entry.  We have
1172          // to use something different than null because null means EOF.
1173          if (record == SKIP_ENTRY)
1174          {
1175            record = null;
1176          }
1177        }
1178    
1179        if (!(record instanceof Entry))
1180        {
1181          try
1182          {
1183            // Some LDIFChangeRecord can be converted to an Entry.  This is really
1184            // an edge case though.
1185            return ((LDIFChangeRecord)record).toEntry();
1186          }
1187          catch (LDIFException e)
1188          {
1189            debugException(e);
1190            final long firstLineNumber = result.getInput().getFirstLineNumber();
1191            throw new LDIFException(e.getExceptionMessage(),
1192                                    firstLineNumber, true, e);
1193          }
1194        }
1195    
1196        return (Entry) record;
1197      }
1198    
1199    
1200    
1201      /**
1202       * Reads an LDIF change record from the LDIF source asynchronously.
1203       * Optionally, if the LDIF record does not have a changetype, then it may be
1204       * assumed to be an add change record.
1205       *
1206       * @param defaultAdd Indicates whether an LDIF record not containing a
1207       *                   changetype should be retrieved as an add change record.
1208       *                   If this is {@code false} and the record read does not
1209       *                   include a changetype, then an {@code LDIFException} will
1210       *                   be thrown.
1211       *
1212       * @return The change record read from the LDIF source, or {@code null} if
1213       *         there are no more records to be read.
1214       *
1215       * @throws IOException   If a problem occurs while attempting to read from the
1216       *                       LDIF source.
1217       * @throws LDIFException If the data read could not be parsed as an LDIF
1218       *                       change record.
1219       */
1220      private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1221              throws IOException, LDIFException
1222      {
1223        final Result<UnparsedLDIFRecord, LDIFRecord> result =
1224             readLDIFRecordResultAsync();
1225        if (result == null)
1226        {
1227          return null;
1228        }
1229    
1230        final LDIFRecord record = result.getOutput();
1231        if (record instanceof LDIFChangeRecord)
1232        {
1233          return (LDIFChangeRecord) record;
1234        }
1235        else if (record instanceof Entry)
1236        {
1237          if (defaultAdd)
1238          {
1239            return new LDIFAddChangeRecord((Entry) record);
1240          }
1241          else
1242          {
1243            final long firstLineNumber = result.getInput().getFirstLineNumber();
1244            throw new LDIFException(
1245                 ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1246                 true);
1247          }
1248        }
1249    
1250        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1251                                 "LDIFChangeRecord");
1252      }
1253    
1254    
1255    
1256      /**
1257       * Reads the next LDIF record, which was read and parsed asynchronously by
1258       * separate threads.
1259       *
1260       * @return  The next LDIF record or {@code null} if there are no more records.
1261       *
1262       * @throws  IOException  If a problem occurs while attempting to read from the
1263       *                       LDIF source.
1264       *
1265       * @throws  LDIFException  If the data read could not be parsed as an entry.
1266       */
1267      private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1268              throws IOException, LDIFException
1269      {
1270        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1271    
1272        // If the asynchronous reading and parsing is complete, then we don't have
1273        // to block waiting for the next record to show up on the queue.  If there
1274        // isn't a record there, then return null (EOF) right away.
1275        if (asyncParsingComplete.get())
1276        {
1277          result = asyncParsedRecords.poll();
1278        }
1279        else
1280        {
1281          try
1282          {
1283            // We probably could just do a asyncParsedRecords.take() here, but
1284            // there are some edge case error scenarios where
1285            // asyncParsingComplete might be set without a special EOF sentinel
1286            // Result enqueued.  So to guard against this, we have a very cautious
1287            // polling interval of 1 second.  During normal processing, we never
1288            // have to wait for this to expire, when there is something to do
1289            // (like shutdown).
1290            while ((result == null) && (!asyncParsingComplete.get()))
1291            {
1292              result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1293            }
1294    
1295            // There's a very small chance that we missed the value, so double-check
1296            if (result == null)
1297            {
1298              result = asyncParsedRecords.poll();
1299            }
1300          }
1301          catch (InterruptedException e)
1302          {
1303            debugException(e);
1304            throw new IOException(getExceptionMessage(e));
1305          }
1306        }
1307        if (result == null)
1308        {
1309          return null;
1310        }
1311    
1312        rethrow(result.getFailureCause());
1313    
1314        // Check if we reached the end of the input
1315        final UnparsedLDIFRecord unparsedRecord = result.getInput();
1316        if (unparsedRecord.isEOF())
1317        {
1318          // This might have been set already by the LineReaderThread, but
1319          // just in case it hasn't gotten to it yet, do so here.
1320          asyncParsingComplete.set(true);
1321    
1322          // Enqueue this EOF result again for any other thread that might be
1323          // blocked in asyncParsedRecords.take() even though having multiple
1324          // threads call this method concurrently breaks the contract of this
1325          // class.
1326          try
1327          {
1328            asyncParsedRecords.put(result);
1329          }
1330          catch (InterruptedException e)
1331          {
1332            // We shouldn't ever get interrupted because the put won't ever block.
1333            // Once we are done reading, this is the only item left in the queue,
1334            // so we should always be able to re-enqueue it.
1335            debugException(e);
1336          }
1337          return null;
1338        }
1339    
1340        return result;
1341      }
1342    
1343    
1344    
1345      /**
1346       * Indicates whether this LDIF reader was constructed to perform asynchronous
1347       * processing.
1348       *
1349       * @return  {@code true} if this LDIFReader was constructed to perform
1350       *          asynchronous processing, or {@code false} if not.
1351       */
1352      private boolean isAsync()
1353      {
1354        return isAsync;
1355      }
1356    
1357    
1358    
1359      /**
1360       * If not {@code null}, rethrows the specified Throwable as either an
1361       * IOException or LDIFException.
1362       *
1363       * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1364       *           is thrown.
1365       *
1366       * @throws IOException   If t is an IOException or a checked Exception that
1367       *                       is not an LDIFException.
1368       * @throws LDIFException  If t is an LDIFException.
1369       */
1370      static void rethrow(final Throwable t)
1371             throws IOException, LDIFException
1372      {
1373        if (t == null)
1374        {
1375          return;
1376        }
1377    
1378        if (t instanceof IOException)
1379        {
1380          throw (IOException) t;
1381        }
1382        else if (t instanceof LDIFException)
1383        {
1384          throw (LDIFException) t;
1385        }
1386        else if (t instanceof RuntimeException)
1387        {
1388          throw (RuntimeException) t;
1389        }
1390        else if (t instanceof Error)
1391        {
1392          throw (Error) t;
1393        }
1394        else
1395        {
1396          throw new IOException(getExceptionMessage(t));
1397        }
1398      }
1399    
1400    
1401    
1402      /**
1403       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1404       * change record.
1405       *
1406       * @return The record read from the LDIF source, or {@code null} if there are
1407       *         no more entries to be read.
1408       *
1409       * @throws IOException   If a problem occurs while trying to read from the
1410       *                       LDIF source.
1411       * @throws LDIFException If the data read could not be parsed as an entry or
1412       *                       an LDIF change record.
1413       */
1414      private LDIFRecord readLDIFRecordInternal()
1415           throws IOException, LDIFException
1416      {
1417        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1418        return decodeRecord(unparsedRecord, relativeBasePath);
1419      }
1420    
1421    
1422    
1423      /**
1424       * Reads an entry from the LDIF source.
1425       *
1426       * @return The entry read from the LDIF source, or {@code null} if there are
1427       *         no more entries to be read.
1428       *
1429       * @throws IOException   If a problem occurs while attempting to read from the
1430       *                       LDIF source.
1431       * @throws LDIFException If the data read could not be parsed as an entry.
1432       */
1433      private Entry readEntryInternal()
1434           throws IOException, LDIFException
1435      {
1436        Entry e = null;
1437        while (e == null)
1438        {
1439          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1440          if (unparsedRecord.isEOF())
1441          {
1442            return null;
1443          }
1444    
1445          e = decodeEntry(unparsedRecord, relativeBasePath);
1446          debugLDIFRead(e);
1447    
1448          if (entryTranslator != null)
1449          {
1450            e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1451          }
1452        }
1453        return e;
1454      }
1455    
1456    
1457    
1458      /**
1459       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1460       * record does not have a changetype, then it may be assumed to be an add
1461       * change record.
1462       *
1463       * @param defaultAdd Indicates whether an LDIF record not containing a
1464       *                   changetype should be retrieved as an add change record.
1465       *                   If this is {@code false} and the record read does not
1466       *                   include a changetype, then an {@code LDIFException} will
1467       *                   be thrown.
1468       *
1469       * @return The change record read from the LDIF source, or {@code null} if
1470       *         there are no more records to be read.
1471       *
1472       * @throws IOException   If a problem occurs while attempting to read from the
1473       *                       LDIF source.
1474       * @throws LDIFException If the data read could not be parsed as an LDIF
1475       *                       change record.
1476       */
1477      private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1478           throws IOException, LDIFException
1479      {
1480        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1481        if (unparsedRecord.isEOF())
1482        {
1483          return null;
1484        }
1485    
1486        final LDIFChangeRecord r =
1487             decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd);
1488        debugLDIFRead(r);
1489        return r;
1490      }
1491    
1492    
1493    
1494      /**
1495       * Reads a record (either an entry or a change record) from the LDIF source
1496       * and places it in the line list.
1497       *
1498       * @return  The line number for the first line of the entry that was read.
1499       *
1500       * @throws  IOException  If a problem occurs while attempting to read from the
1501       *                       LDIF source.
1502       *
1503       * @throws  LDIFException  If the data read could not be parsed as a valid
1504       *                         LDIF record.
1505       */
1506      private UnparsedLDIFRecord readUnparsedRecord()
1507             throws IOException, LDIFException
1508      {
1509        final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20);
1510        boolean lastWasComment = false;
1511        long firstLineNumber = lineNumberCounter + 1;
1512        while (true)
1513        {
1514          final String line = reader.readLine();
1515          lineNumberCounter++;
1516    
1517          if (line == null)
1518          {
1519            // We've hit the end of the LDIF source.  If we haven't read any entry
1520            // data, then return null.  Otherwise, the last entry wasn't followed by
1521            // a blank line, which is OK, and we should decode that entry.
1522            if (lineList.isEmpty())
1523            {
1524              return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1525                   duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1526            }
1527            else
1528            {
1529              break;
1530            }
1531          }
1532    
1533          if (line.length() == 0)
1534          {
1535            // It's a blank line.  If we have read entry data, then this signals the
1536            // end of the entry.  Otherwise, it's an extra space between entries,
1537            // which is OK.
1538            lastWasComment = false;
1539            if (lineList.isEmpty())
1540            {
1541              firstLineNumber++;
1542              continue;
1543            }
1544            else
1545            {
1546              break;
1547            }
1548          }
1549    
1550          if (line.charAt(0) == ' ')
1551          {
1552            // The line starts with a space, which means that it must be a
1553            // continuation of the previous line.  This is true even if the last
1554            // line was a comment.
1555            if (lastWasComment)
1556            {
1557              // What we've read is part of a comment, so we don't care about its
1558              // content.
1559            }
1560            else if (lineList.isEmpty())
1561            {
1562              throw new LDIFException(
1563                             ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1564                             lineNumberCounter, false);
1565            }
1566            else
1567            {
1568              lineList.get(lineList.size() - 1).append(line.substring(1));
1569              lastWasComment = false;
1570            }
1571          }
1572          else if (line.charAt(0) == '#')
1573          {
1574            lastWasComment = true;
1575          }
1576          else
1577          {
1578            // We want to make sure that we skip over the "version:" line if it
1579            // exists, but that should only occur at the beginning of an entry where
1580            // it can't be confused with a possible "version" attribute.
1581            if (lineList.isEmpty() && line.startsWith("version:"))
1582            {
1583              lastWasComment = true;
1584            }
1585            else
1586            {
1587              lineList.add(new StringBuilder(line));
1588              lastWasComment = false;
1589            }
1590          }
1591        }
1592    
1593        return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1594             trailingSpaceBehavior, schema, firstLineNumber);
1595      }
1596    
1597    
1598    
1599      /**
1600       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1601       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1602       * LDIF specification, and it is acceptable to have one or more blank lines
1603       * following the entry.
1604       *
1605       * @param  ldifLines  The set of lines that comprise the LDIF representation
1606       *                    of the entry.  It must not be {@code null} or empty.
1607       *
1608       * @return  The entry read from LDIF.
1609       *
1610       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1611       *                         entry.
1612       */
1613      public static Entry decodeEntry(final String... ldifLines)
1614             throws LDIFException
1615      {
1616        final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1617             TrailingSpaceBehavior.REJECT, null, ldifLines),
1618             DEFAULT_RELATIVE_BASE_PATH);
1619        debugLDIFRead(e);
1620        return e;
1621      }
1622    
1623    
1624    
1625      /**
1626       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1627       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1628       * LDIF specification, and it is acceptable to have one or more blank lines
1629       * following the entry.
1630       *
1631       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1632       *                                attribute values encountered while parsing.
1633       * @param  schema                 The schema to use when parsing the record,
1634       *                                if applicable.
1635       * @param  ldifLines              The set of lines that comprise the LDIF
1636       *                                representation of the entry.  It must not be
1637       *                                {@code null} or empty.
1638       *
1639       * @return  The entry read from LDIF.
1640       *
1641       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1642       *                         entry.
1643       */
1644      public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1645                                      final Schema schema,
1646                                      final String... ldifLines)
1647             throws LDIFException
1648      {
1649        final Entry e = decodeEntry(prepareRecord(
1650                  (ignoreDuplicateValues
1651                       ? DuplicateValueBehavior.STRIP
1652                       : DuplicateValueBehavior.REJECT),
1653                  TrailingSpaceBehavior.REJECT, schema, ldifLines),
1654             DEFAULT_RELATIVE_BASE_PATH);
1655        debugLDIFRead(e);
1656        return e;
1657      }
1658    
1659    
1660    
1661      /**
1662       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1663       * provided set of lines must contain exactly one change record and it must
1664       * include a changetype.  Long lines may be wrapped as per the LDIF
1665       * specification, and it is acceptable to have one or more blank lines
1666       * following the entry.
1667       *
1668       * @param  ldifLines  The set of lines that comprise the LDIF representation
1669       *                    of the change record.  It must not be {@code null} or
1670       *                    empty.
1671       *
1672       * @return  The change record read from LDIF.
1673       *
1674       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1675       *                         change record.
1676       */
1677      public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1678             throws LDIFException
1679      {
1680        return decodeChangeRecord(false, ldifLines);
1681      }
1682    
1683    
1684    
1685      /**
1686       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1687       * provided set of lines must contain exactly one change record.  Long lines
1688       * may be wrapped as per the LDIF specification, and it is acceptable to have
1689       * one or more blank lines following the entry.
1690       *
1691       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1692       *                     changetype should be retrieved as an add change record.
1693       *                     If this is {@code false} and the record read does not
1694       *                     include a changetype, then an {@code LDIFException}
1695       *                     will be thrown.
1696       * @param  ldifLines  The set of lines that comprise the LDIF representation
1697       *                    of the change record.  It must not be {@code null} or
1698       *                    empty.
1699       *
1700       * @return  The change record read from LDIF.
1701       *
1702       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1703       *                         change record.
1704       */
1705      public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1706                                                        final String... ldifLines)
1707             throws LDIFException
1708      {
1709        final LDIFChangeRecord r =
1710             decodeChangeRecord(
1711                  prepareRecord(DuplicateValueBehavior.STRIP,
1712                       TrailingSpaceBehavior.REJECT, null, ldifLines),
1713                  DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1714        debugLDIFRead(r);
1715        return r;
1716      }
1717    
1718    
1719    
1720      /**
1721       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1722       * provided set of lines must contain exactly one change record.  Long lines
1723       * may be wrapped as per the LDIF specification, and it is acceptable to have
1724       * one or more blank lines following the entry.
1725       *
1726       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1727       *                                attribute values encountered while parsing.
1728       * @param  schema                 The schema to use when processing the change
1729       *                                record, or {@code null} if no schema should
1730       *                                be used and all values should be treated as
1731       *                                case-insensitive strings.
1732       * @param  defaultAdd             Indicates whether an LDIF record not
1733       *                                containing a changetype should be retrieved
1734       *                                as an add change record.  If this is
1735       *                                {@code false} and the record read does not
1736       *                                include a changetype, then an
1737       *                                {@code LDIFException} will be thrown.
1738       * @param  ldifLines              The set of lines that comprise the LDIF
1739       *                                representation of the change record.  It
1740       *                                must not be {@code null} or empty.
1741       *
1742       * @return  The change record read from LDIF.
1743       *
1744       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1745       *                         change record.
1746       */
1747      public static LDIFChangeRecord decodeChangeRecord(
1748                                          final boolean ignoreDuplicateValues,
1749                                          final Schema schema,
1750                                          final boolean defaultAdd,
1751                                          final String... ldifLines)
1752             throws LDIFException
1753      {
1754        final LDIFChangeRecord r = decodeChangeRecord(
1755             prepareRecord(
1756                  (ignoreDuplicateValues
1757                       ? DuplicateValueBehavior.STRIP
1758                       : DuplicateValueBehavior.REJECT),
1759                  TrailingSpaceBehavior.REJECT, schema, ldifLines),
1760             DEFAULT_RELATIVE_BASE_PATH, defaultAdd);
1761        debugLDIFRead(r);
1762        return r;
1763      }
1764    
1765    
1766    
1767      /**
1768       * Parses the provided set of lines into a list of {@code StringBuilder}
1769       * objects suitable for decoding into an entry or LDIF change record.
1770       * Comments will be ignored and wrapped lines will be unwrapped.
1771       *
1772       * @param  duplicateValueBehavior  The behavior that should be exhibited if
1773       *                                 the LDIF reader encounters an entry with
1774       *                                 duplicate values.
1775       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
1776       *                                 encountering attribute values which are not
1777       *                                 base64-encoded but contain trailing spaces.
1778       * @param  schema                  The schema to use when parsing the record,
1779       *                                 if applicable.
1780       * @param  ldifLines               The set of lines that comprise the record
1781       *                                 to decode.  It must not be {@code null} or
1782       *                                 empty.
1783       *
1784       * @return  The prepared list of {@code StringBuilder} objects ready to be
1785       *          decoded.
1786       *
1787       * @throws  LDIFException  If the provided lines do not contain valid LDIF
1788       *                         content.
1789       */
1790      private static UnparsedLDIFRecord prepareRecord(
1791                          final DuplicateValueBehavior duplicateValueBehavior,
1792                          final TrailingSpaceBehavior trailingSpaceBehavior,
1793                          final Schema schema, final String... ldifLines)
1794              throws LDIFException
1795      {
1796        ensureNotNull(ldifLines);
1797        ensureFalse(ldifLines.length == 0,
1798                    "LDIFReader.prepareRecord.ldifLines must not be empty.");
1799    
1800        boolean lastWasComment = false;
1801        final ArrayList<StringBuilder> lineList =
1802             new ArrayList<StringBuilder>(ldifLines.length);
1803        for (int i=0; i < ldifLines.length; i++)
1804        {
1805          final String line = ldifLines[i];
1806          if (line.length() == 0)
1807          {
1808            // This is only acceptable if there are no more non-empty lines in the
1809            // array.
1810            for (int j=i+1; j < ldifLines.length; j++)
1811            {
1812              if (ldifLines[j].length() > 0)
1813              {
1814                throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
1815                                        ldifLines, null);
1816              }
1817    
1818              // If we've gotten here, then we know that we're at the end of the
1819              // entry.  If we have read data, then we can decode it as an entry.
1820              // Otherwise, there was no real data in the provided LDIF lines.
1821              if (lineList.isEmpty())
1822              {
1823                throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
1824                                        ldifLines, null);
1825              }
1826              else
1827              {
1828                return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1829                     trailingSpaceBehavior, schema, 0);
1830              }
1831            }
1832          }
1833    
1834          if (line.charAt(0) == ' ')
1835          {
1836            if (i > 0)
1837            {
1838              if (! lastWasComment)
1839              {
1840                lineList.get(lineList.size() - 1).append(line.substring(1));
1841              }
1842            }
1843            else
1844            {
1845              throw new LDIFException(
1846                             ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
1847                             true, ldifLines, null);
1848            }
1849          }
1850          else if (line.charAt(0) == '#')
1851          {
1852            lastWasComment = true;
1853          }
1854          else
1855          {
1856            lineList.add(new StringBuilder(line));
1857            lastWasComment = false;
1858          }
1859        }
1860    
1861        if (lineList.isEmpty())
1862        {
1863          throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
1864        }
1865        else
1866        {
1867          return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1868               trailingSpaceBehavior, schema, 0);
1869        }
1870      }
1871    
1872    
1873    
1874      /**
1875       * Decodes the unparsed record that was read from the LDIF source.  It may be
1876       * either an entry or an LDIF change record.
1877       *
1878       * @param  unparsedRecord    The unparsed LDIF record that was read from the
1879       *                           input.  It must not be {@code null} or empty.
1880       * @param  relativeBasePath  The base path that will be prepended to relative
1881       *                           paths in order to obtain an absolute path.
1882       *
1883       * @return  The parsed record, or {@code null} if there are no more entries to
1884       *          be read.
1885       *
1886       * @throws  LDIFException  If the data read could not be parsed as an entry or
1887       *                         an LDIF change record.
1888       */
1889      private static LDIFRecord decodeRecord(
1890                                     final UnparsedLDIFRecord unparsedRecord,
1891                                     final String relativeBasePath)
1892           throws LDIFException
1893      {
1894        // If there was an error reading from the input, then we rethrow it here.
1895        final Exception readError = unparsedRecord.getFailureCause();
1896        if (readError != null)
1897        {
1898          if (readError instanceof LDIFException)
1899          {
1900            // If the error was an LDIFException, which will normally be the case,
1901            // then rethrow it with all of the same state.  We could just
1902            //   throw (LDIFException) readError;
1903            // but that's considered bad form.
1904            final LDIFException ldifEx = (LDIFException) readError;
1905            throw new LDIFException(ldifEx.getMessage(),
1906                                    ldifEx.getLineNumber(),
1907                                    ldifEx.mayContinueReading(),
1908                                    ldifEx.getDataLines(),
1909                                    ldifEx.getCause());
1910          }
1911          else
1912          {
1913            throw new LDIFException(getExceptionMessage(readError),
1914                                    -1, true, readError);
1915          }
1916        }
1917    
1918        if (unparsedRecord.isEOF())
1919        {
1920          return null;
1921        }
1922    
1923        final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
1924        if (unparsedRecord.getLineList() == null)
1925        {
1926          return null;  // We can get here if there was an error reading the lines.
1927        }
1928    
1929        final LDIFRecord r;
1930        if (lineList.size() == 1)
1931        {
1932          r = decodeEntry(unparsedRecord, relativeBasePath);
1933        }
1934        else
1935        {
1936          final String lowerSecondLine = toLowerCase(lineList.get(1).toString());
1937          if (lowerSecondLine.startsWith("control:") ||
1938              lowerSecondLine.startsWith("changetype:"))
1939          {
1940            r = decodeChangeRecord(unparsedRecord, relativeBasePath, true);
1941          }
1942          else
1943          {
1944            r = decodeEntry(unparsedRecord, relativeBasePath);
1945          }
1946        }
1947    
1948        debugLDIFRead(r);
1949        return r;
1950      }
1951    
1952    
1953    
1954      /**
1955       * Decodes the provided set of LDIF lines as an entry.  The provided list must
1956       * not contain any blank lines or comments, and lines are not allowed to be
1957       * wrapped.
1958       *
1959       * @param  unparsedRecord   The unparsed LDIF record that was read from the
1960       *                          input.  It must not be {@code null} or empty.
1961       * @param  relativeBasePath  The base path that will be prepended to relative
1962       *                           paths in order to obtain an absolute path.
1963       *
1964       * @return  The entry read from LDIF.
1965       *
1966       * @throws  LDIFException  If the provided LDIF data cannot be read as an
1967       *                         entry.
1968       */
1969      private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
1970                                       final String relativeBasePath)
1971              throws LDIFException
1972      {
1973        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
1974        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
1975    
1976        final Iterator<StringBuilder> iterator = ldifLines.iterator();
1977    
1978        // The first line must start with either "version:" or "dn:".  If the first
1979        // line starts with "version:" then the second must start with "dn:".
1980        StringBuilder line = iterator.next();
1981        handleTrailingSpaces(line, null, firstLineNumber,
1982             unparsedRecord.getTrailingSpaceBehavior());
1983        int colonPos = line.indexOf(":");
1984        if ((colonPos > 0) &&
1985            line.substring(0, colonPos).equalsIgnoreCase("version"))
1986        {
1987          // The first line is "version:".  Under most conditions, this will be
1988          // handled by the LDIF reader, but this can happen if you call
1989          // decodeEntry with a set of data that includes a version.  At any rate,
1990          // read the next line, which must specify the DN.
1991          line = iterator.next();
1992          handleTrailingSpaces(line, null, firstLineNumber,
1993               unparsedRecord.getTrailingSpaceBehavior());
1994        }
1995    
1996        colonPos = line.indexOf(":");
1997        if ((colonPos < 0) ||
1998             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
1999        {
2000          throw new LDIFException(
2001               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2002               firstLineNumber, true, ldifLines, null);
2003        }
2004    
2005        final String dn;
2006        final int length = line.length();
2007        if (length == (colonPos+1))
2008        {
2009          // The colon was the last character on the line.  This is acceptable and
2010          // indicates that the entry has the null DN.
2011          dn = "";
2012        }
2013        else if (line.charAt(colonPos+1) == ':')
2014        {
2015          // Skip over any spaces leading up to the value, and then the rest of the
2016          // string is the base64-encoded DN.
2017          int pos = colonPos+2;
2018          while ((pos < length) && (line.charAt(pos) == ' '))
2019          {
2020            pos++;
2021          }
2022    
2023          try
2024          {
2025            final byte[] dnBytes = Base64.decode(line.substring(pos));
2026            dn = new String(dnBytes, "UTF-8");
2027          }
2028          catch (final ParseException pe)
2029          {
2030            debugException(pe);
2031            throw new LDIFException(
2032                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2033                                                                pe.getMessage()),
2034                           firstLineNumber, true, ldifLines, pe);
2035          }
2036          catch (final Exception e)
2037          {
2038            debugException(e);
2039            throw new LDIFException(
2040                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2041                           firstLineNumber, true, ldifLines, e);
2042          }
2043        }
2044        else
2045        {
2046          // Skip over any spaces leading up to the value, and then the rest of the
2047          // string is the DN.
2048          int pos = colonPos+1;
2049          while ((pos < length) && (line.charAt(pos) == ' '))
2050          {
2051            pos++;
2052          }
2053    
2054          dn = line.substring(pos);
2055        }
2056    
2057    
2058        // The remaining lines must be the attributes for the entry.  However, we
2059        // will allow the case in which an entry does not have any attributes, to be
2060        // able to support reading search result entries in which no attributes were
2061        // returned.
2062        if (! iterator.hasNext())
2063        {
2064          return new Entry(dn, unparsedRecord.getSchema());
2065        }
2066    
2067        return new Entry(dn, unparsedRecord.getSchema(),
2068             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2069                  unparsedRecord.getTrailingSpaceBehavior(),
2070                  unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2071                  firstLineNumber));
2072      }
2073    
2074    
2075    
2076      /**
2077       * Decodes the provided set of LDIF lines as a change record.  The provided
2078       * list must not contain any blank lines or comments, and lines are not
2079       * allowed to be wrapped.
2080       *
2081       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2082       *                           input.  It must not be {@code null} or empty.
2083       * @param  relativeBasePath  The base path that will be prepended to relative
2084       *                           paths in order to obtain an absolute path.
2085       * @param  defaultAdd        Indicates whether an LDIF record not containing a
2086       *                           changetype should be retrieved as an add change
2087       *                           record.  If this is {@code false} and the record
2088       *                           read does not include a changetype, then an
2089       *                           {@code LDIFException} will be thrown.
2090       *
2091       * @return  The change record read from LDIF.
2092       *
2093       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2094       *                         change record.
2095       */
2096      private static LDIFChangeRecord decodeChangeRecord(
2097                                           final UnparsedLDIFRecord unparsedRecord,
2098                                           final String relativeBasePath,
2099                                           final boolean defaultAdd)
2100              throws LDIFException
2101      {
2102        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2103        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2104    
2105        Iterator<StringBuilder> iterator = ldifLines.iterator();
2106    
2107        // The first line must start with either "version:" or "dn:".  If the first
2108        // line starts with "version:" then the second must start with "dn:".
2109        StringBuilder line = iterator.next();
2110        handleTrailingSpaces(line, null, firstLineNumber,
2111             unparsedRecord.getTrailingSpaceBehavior());
2112        int colonPos = line.indexOf(":");
2113        int linesRead = 1;
2114        if ((colonPos > 0) &&
2115            line.substring(0, colonPos).equalsIgnoreCase("version"))
2116        {
2117          // The first line is "version:".  Under most conditions, this will be
2118          // handled by the LDIF reader, but this can happen if you call
2119          // decodeEntry with a set of data that includes a version.  At any rate,
2120          // read the next line, which must specify the DN.
2121          line = iterator.next();
2122          linesRead++;
2123          handleTrailingSpaces(line, null, firstLineNumber,
2124               unparsedRecord.getTrailingSpaceBehavior());
2125        }
2126    
2127        colonPos = line.indexOf(":");
2128        if ((colonPos < 0) ||
2129             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2130        {
2131          throw new LDIFException(
2132               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2133               firstLineNumber, true, ldifLines, null);
2134        }
2135    
2136        final String dn;
2137        int length = line.length();
2138        if (length == (colonPos+1))
2139        {
2140          // The colon was the last character on the line.  This is acceptable and
2141          // indicates that the entry has the null DN.
2142          dn = "";
2143        }
2144        else if (line.charAt(colonPos+1) == ':')
2145        {
2146          // Skip over any spaces leading up to the value, and then the rest of the
2147          // string is the base64-encoded DN.
2148          int pos = colonPos+2;
2149          while ((pos < length) && (line.charAt(pos) == ' '))
2150          {
2151            pos++;
2152          }
2153    
2154          try
2155          {
2156            final byte[] dnBytes = Base64.decode(line.substring(pos));
2157            dn = new String(dnBytes, "UTF-8");
2158          }
2159          catch (final ParseException pe)
2160          {
2161            debugException(pe);
2162            throw new LDIFException(
2163                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2164                                                                   pe.getMessage()),
2165                           firstLineNumber, true, ldifLines, pe);
2166          }
2167          catch (final Exception e)
2168          {
2169            debugException(e);
2170            throw new LDIFException(
2171                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2172                                                                   e),
2173                           firstLineNumber, true, ldifLines, e);
2174          }
2175        }
2176        else
2177        {
2178          // Skip over any spaces leading up to the value, and then the rest of the
2179          // string is the DN.
2180          int pos = colonPos+1;
2181          while ((pos < length) && (line.charAt(pos) == ' '))
2182          {
2183            pos++;
2184          }
2185    
2186          dn = line.substring(pos);
2187        }
2188    
2189    
2190        // An LDIF change record may contain zero or more controls, with the end of
2191        // the controls signified by the changetype.  The changetype element must be
2192        // present, unless defaultAdd is true in which case the first thing that is
2193        // neither control or changetype will trigger the start of add attribute
2194        // parsing.
2195        if (! iterator.hasNext())
2196        {
2197          throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2198                                  firstLineNumber, true, ldifLines, null);
2199        }
2200    
2201        String changeType = null;
2202        ArrayList<Control> controls = null;
2203        while (true)
2204        {
2205          line = iterator.next();
2206          handleTrailingSpaces(line, dn, firstLineNumber,
2207               unparsedRecord.getTrailingSpaceBehavior());
2208          colonPos = line.indexOf(":");
2209          if (colonPos < 0)
2210          {
2211            throw new LDIFException(
2212                 ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2213                 firstLineNumber, true, ldifLines, null);
2214          }
2215    
2216          final String token = toLowerCase(line.substring(0, colonPos));
2217          if (token.equals("control"))
2218          {
2219            if (controls == null)
2220            {
2221              controls = new ArrayList<Control>(5);
2222            }
2223    
2224            controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2225                 relativeBasePath));
2226          }
2227          else if (token.equals("changetype"))
2228          {
2229            changeType =
2230                 decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2231            break;
2232          }
2233          else if (defaultAdd)
2234          {
2235            // The line we read wasn't a control or changetype declaration, so we'll
2236            // assume it's an attribute in an add record.  However, we're not ready
2237            // for that yet, and since we can't rewind an iterator we'll create a
2238            // new one that hasn't yet gotten to this line.
2239            changeType = "add";
2240            iterator = ldifLines.iterator();
2241            for (int i=0; i < linesRead; i++)
2242            {
2243              iterator.next();
2244            }
2245            break;
2246          }
2247          else
2248          {
2249            throw new LDIFException(
2250                 ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2251                      firstLineNumber),
2252                 firstLineNumber, true, ldifLines, null);
2253          }
2254    
2255          linesRead++;
2256        }
2257    
2258    
2259        // Make sure that the change type is acceptable and then decode the rest of
2260        // the change record accordingly.
2261        final String lowerChangeType = toLowerCase(changeType);
2262        if (lowerChangeType.equals("add"))
2263        {
2264          // There must be at least one more line.  If not, then that's an error.
2265          // Otherwise, parse the rest of the data as attribute-value pairs.
2266          if (iterator.hasNext())
2267          {
2268            final Collection<Attribute> attrs =
2269                 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2270                      unparsedRecord.getTrailingSpaceBehavior(),
2271                      unparsedRecord.getSchema(), ldifLines, iterator,
2272                      relativeBasePath, firstLineNumber);
2273            final Attribute[] attributes = new Attribute[attrs.size()];
2274            final Iterator<Attribute> attrIterator = attrs.iterator();
2275            for (int i=0; i < attributes.length; i++)
2276            {
2277              attributes[i] = attrIterator.next();
2278            }
2279    
2280            return new LDIFAddChangeRecord(dn, attributes, controls);
2281          }
2282          else
2283          {
2284            throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2285                                    firstLineNumber, true, ldifLines, null);
2286          }
2287        }
2288        else if (lowerChangeType.equals("delete"))
2289        {
2290          // There shouldn't be any more data.  If there is, then that's an error.
2291          // Otherwise, we can just return the delete change record with what we
2292          // already know.
2293          if (iterator.hasNext())
2294          {
2295            throw new LDIFException(
2296                           ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2297                           firstLineNumber, true, ldifLines, null);
2298          }
2299          else
2300          {
2301            return new LDIFDeleteChangeRecord(dn, controls);
2302          }
2303        }
2304        else if (lowerChangeType.equals("modify"))
2305        {
2306          // There must be at least one more line.  If not, then that's an error.
2307          // Otherwise, parse the rest of the data as a set of modifications.
2308          if (iterator.hasNext())
2309          {
2310            final Modification[] mods = parseModifications(dn,
2311                 unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2312                 firstLineNumber);
2313            return new LDIFModifyChangeRecord(dn, mods, controls);
2314          }
2315          else
2316          {
2317            throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2318                                    firstLineNumber, true, ldifLines, null);
2319          }
2320        }
2321        else if (lowerChangeType.equals("moddn") ||
2322                 lowerChangeType.equals("modrdn"))
2323        {
2324          // There must be at least one more line.  If not, then that's an error.
2325          // Otherwise, parse the rest of the data as a set of modifications.
2326          if (iterator.hasNext())
2327          {
2328            return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2329                 unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2330          }
2331          else
2332          {
2333            throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2334                                    firstLineNumber, true, ldifLines, null);
2335          }
2336        }
2337        else
2338        {
2339          throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2340                                                             firstLineNumber),
2341                                  firstLineNumber, true, ldifLines, null);
2342        }
2343      }
2344    
2345    
2346    
2347      /**
2348       * Decodes information about a control from the provided line.
2349       *
2350       * @param  line              The line to process.
2351       * @param  colonPos          The position of the colon that separates the
2352       *                           control token string from tbe encoded control.
2353       * @param  firstLineNumber   The line number for the start of the record.
2354       * @param  ldifLines         The lines that comprise the LDIF representation
2355       *                           of the full record being parsed.
2356       * @param  relativeBasePath  The base path that will be prepended to relative
2357       *                           paths in order to obtain an absolute path.
2358       *
2359       * @return  The decoded control.
2360       *
2361       * @throws  LDIFException  If a problem is encountered while trying to decode
2362       *                         the changetype.
2363       */
2364      private static Control decodeControl(final StringBuilder line,
2365                                           final int colonPos,
2366                                           final long firstLineNumber,
2367                                           final ArrayList<StringBuilder> ldifLines,
2368                                           final String relativeBasePath)
2369              throws LDIFException
2370      {
2371        final String controlString;
2372        int length = line.length();
2373        if (length == (colonPos+1))
2374        {
2375          // The colon was the last character on the line.  This is not
2376          // acceptable.
2377          throw new LDIFException(
2378               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2379               firstLineNumber, true, ldifLines, null);
2380        }
2381        else if (line.charAt(colonPos+1) == ':')
2382        {
2383          // Skip over any spaces leading up to the value, and then the rest of
2384          // the string is the base64-encoded control representation.  This is
2385          // unusual and unnecessary, but is nevertheless acceptable.
2386          int pos = colonPos+2;
2387          while ((pos < length) && (line.charAt(pos) == ' '))
2388          {
2389            pos++;
2390          }
2391    
2392          try
2393          {
2394            final byte[] controlBytes = Base64.decode(line.substring(pos));
2395            controlString =  new String(controlBytes, "UTF-8");
2396          }
2397          catch (final ParseException pe)
2398          {
2399            debugException(pe);
2400            throw new LDIFException(
2401                           ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2402                                firstLineNumber, pe.getMessage()),
2403                           firstLineNumber, true, ldifLines, pe);
2404          }
2405          catch (final Exception e)
2406          {
2407            debugException(e);
2408            throw new LDIFException(
2409                 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2410                 firstLineNumber, true, ldifLines, e);
2411          }
2412        }
2413        else
2414        {
2415          // Skip over any spaces leading up to the value, and then the rest of
2416          // the string is the encoded control.
2417          int pos = colonPos+1;
2418          while ((pos < length) && (line.charAt(pos) == ' '))
2419          {
2420            pos++;
2421          }
2422    
2423          controlString = line.substring(pos);
2424        }
2425    
2426        // If the resulting control definition is empty, then that's invalid.
2427        if (controlString.length() == 0)
2428        {
2429          throw new LDIFException(
2430               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2431               firstLineNumber, true, ldifLines, null);
2432        }
2433    
2434    
2435        // The first element of the control must be the OID, and it must be followed
2436        // by a space (to separate it from the criticality), a colon (to separate it
2437        // from the value and indicate a default criticality of false), or the end
2438        // of the line (to indicate a default criticality of false and no value).
2439        String oid = null;
2440        boolean hasCriticality = false;
2441        boolean hasValue = false;
2442        int pos = 0;
2443        length = controlString.length();
2444        while (pos < length)
2445        {
2446          final char c = controlString.charAt(pos);
2447          if (c == ':')
2448          {
2449            // This indicates that there is no criticality and that the value
2450            // immediately follows the OID.
2451            oid = controlString.substring(0, pos++);
2452            hasValue = true;
2453            break;
2454          }
2455          else if (c == ' ')
2456          {
2457            // This indicates that there is a criticality.  We don't know anything
2458            // about the presence of a value yet.
2459            oid = controlString.substring(0, pos++);
2460            hasCriticality = true;
2461            break;
2462          }
2463          else
2464          {
2465            pos++;
2466          }
2467        }
2468    
2469        if (oid == null)
2470        {
2471          // This indicates that the string representation of the control is only
2472          // the OID.
2473          return new Control(controlString, false);
2474        }
2475    
2476    
2477        // See if we need to read the criticality.  If so, then do so now.
2478        // Otherwise, assume a default criticality of false.
2479        final boolean isCritical;
2480        if (hasCriticality)
2481        {
2482          // Skip over any spaces before the criticality.
2483          while (controlString.charAt(pos) == ' ')
2484          {
2485            pos++;
2486          }
2487    
2488          // Read until we find a colon or the end of the string.
2489          final int criticalityStartPos = pos;
2490          while (pos < length)
2491          {
2492            final char c = controlString.charAt(pos);
2493            if (c == ':')
2494            {
2495              hasValue = true;
2496              break;
2497            }
2498            else
2499            {
2500              pos++;
2501            }
2502          }
2503    
2504          final String criticalityString =
2505               toLowerCase(controlString.substring(criticalityStartPos, pos));
2506          if (criticalityString.equals("true"))
2507          {
2508            isCritical = true;
2509          }
2510          else if (criticalityString.equals("false"))
2511          {
2512            isCritical = false;
2513          }
2514          else
2515          {
2516            throw new LDIFException(
2517                 ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2518                      firstLineNumber),
2519                 firstLineNumber, true, ldifLines, null);
2520          }
2521    
2522          if (hasValue)
2523          {
2524            pos++;
2525          }
2526        }
2527        else
2528        {
2529          isCritical = false;
2530        }
2531    
2532        // See if we need to read the value.  If so, then do so now.  It may be
2533        // a string, or it may be base64-encoded.  It could conceivably even be read
2534        // from a URL.
2535        final ASN1OctetString value;
2536        if (hasValue)
2537        {
2538          // The character immediately after the colon that precedes the value may
2539          // be one of the following:
2540          // - A second colon (optionally followed by a single space) to indicate
2541          //   that the value is base64-encoded.
2542          // - A less-than symbol to indicate that the value should be read from a
2543          //   location specified by a URL.
2544          // - A single space that precedes the non-base64-encoded value.
2545          // - The first character of the non-base64-encoded value.
2546          switch (controlString.charAt(pos))
2547          {
2548            case ':':
2549              try
2550              {
2551                if (controlString.length() == (pos+1))
2552                {
2553                  value = new ASN1OctetString();
2554                }
2555                else if (controlString.charAt(pos+1) == ' ')
2556                {
2557                  value = new ASN1OctetString(
2558                       Base64.decode(controlString.substring(pos+2)));
2559                }
2560                else
2561                {
2562                  value = new ASN1OctetString(
2563                       Base64.decode(controlString.substring(pos+1)));
2564                }
2565              }
2566              catch (final Exception e)
2567              {
2568                debugException(e);
2569                throw new LDIFException(
2570                     ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2571                          firstLineNumber, getExceptionMessage(e)),
2572                     firstLineNumber, true, ldifLines, e);
2573              }
2574              break;
2575            case '<':
2576              try
2577              {
2578                final String urlString;
2579                if (controlString.charAt(pos+1) == ' ')
2580                {
2581                  urlString = controlString.substring(pos+2);
2582                }
2583                else
2584                {
2585                  urlString = controlString.substring(pos+1);
2586                }
2587                value = new ASN1OctetString(retrieveURLBytes(urlString,
2588                     relativeBasePath, firstLineNumber));
2589              }
2590              catch (final Exception e)
2591              {
2592                debugException(e);
2593                throw new LDIFException(
2594                     ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2595                          firstLineNumber, getExceptionMessage(e)),
2596                     firstLineNumber, true, ldifLines, e);
2597              }
2598              break;
2599            case ' ':
2600              value = new ASN1OctetString(controlString.substring(pos+1));
2601              break;
2602            default:
2603              value = new ASN1OctetString(controlString.substring(pos));
2604              break;
2605          }
2606        }
2607        else
2608        {
2609          value = null;
2610        }
2611    
2612        return new Control(oid, isCritical, value);
2613      }
2614    
2615    
2616    
2617      /**
2618       * Decodes the changetype element from the provided line.
2619       *
2620       * @param  line             The line to process.
2621       * @param  colonPos         The position of the colon that separates the
2622       *                          changetype string from its value.
2623       * @param  firstLineNumber  The line number for the start of the record.
2624       * @param  ldifLines        The lines that comprise the LDIF representation of
2625       *                          the full record being parsed.
2626       *
2627       * @return  The decoded changetype string.
2628       *
2629       * @throws  LDIFException  If a problem is encountered while trying to decode
2630       *                         the changetype.
2631       */
2632      private static String decodeChangeType(final StringBuilder line,
2633                                 final int colonPos, final long firstLineNumber,
2634                                 final ArrayList<StringBuilder> ldifLines)
2635              throws LDIFException
2636      {
2637        final int length = line.length();
2638        if (length == (colonPos+1))
2639        {
2640          // The colon was the last character on the line.  This is not
2641          // acceptable.
2642          throw new LDIFException(
2643               ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2644               true, ldifLines, null);
2645        }
2646        else if (line.charAt(colonPos+1) == ':')
2647        {
2648          // Skip over any spaces leading up to the value, and then the rest of
2649          // the string is the base64-encoded changetype.  This is unusual and
2650          // unnecessary, but is nevertheless acceptable.
2651          int pos = colonPos+2;
2652          while ((pos < length) && (line.charAt(pos) == ' '))
2653          {
2654            pos++;
2655          }
2656    
2657          try
2658          {
2659            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
2660            return new String(changeTypeBytes, "UTF-8");
2661          }
2662          catch (final ParseException pe)
2663          {
2664            debugException(pe);
2665            throw new LDIFException(
2666                           ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
2667                                                                pe.getMessage()),
2668                           firstLineNumber, true, ldifLines, pe);
2669          }
2670          catch (final Exception e)
2671          {
2672            debugException(e);
2673            throw new LDIFException(
2674                 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
2675                 firstLineNumber, true, ldifLines, e);
2676          }
2677        }
2678        else
2679        {
2680          // Skip over any spaces leading up to the value, and then the rest of
2681          // the string is the changetype.
2682          int pos = colonPos+1;
2683          while ((pos < length) && (line.charAt(pos) == ' '))
2684          {
2685            pos++;
2686          }
2687    
2688          return line.substring(pos);
2689        }
2690      }
2691    
2692    
2693    
2694      /**
2695       * Parses the data available through the provided iterator as a collection of
2696       * attributes suitable for use in an entry or an add change record.
2697       *
2698       * @param  dn                      The DN of the record being read.
2699       * @param  duplicateValueBehavior  The behavior that should be exhibited if
2700       *                                 the LDIF reader encounters an entry with
2701       *                                 duplicate values.
2702       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2703       *                                 encountering attribute values which are not
2704       *                                 base64-encoded but contain trailing spaces.
2705       * @param  schema                  The schema to use when parsing the
2706       *                                 attributes, or {@code null} if none is
2707       *                                 needed.
2708       * @param  ldifLines               The lines that comprise the LDIF
2709       *                                 representation of the full record being
2710       *                                 parsed.
2711       * @param  iterator                The iterator to use to access the attribute
2712       *                                 lines.
2713       * @param  relativeBasePath        The base path that will be prepended to
2714       *                                 relative paths in order to obtain an
2715       *                                 absolute path.
2716       * @param  firstLineNumber         The line number for the start of the
2717       *                                 record.
2718       *
2719       * @return  The collection of attributes that were read.
2720       *
2721       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2722       *                         set of attributes.
2723       */
2724      private static ArrayList<Attribute> parseAttributes(final String dn,
2725           final DuplicateValueBehavior duplicateValueBehavior,
2726           final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
2727           final ArrayList<StringBuilder> ldifLines,
2728           final Iterator<StringBuilder> iterator, final String relativeBasePath,
2729           final long firstLineNumber)
2730              throws LDIFException
2731      {
2732        final LinkedHashMap<String,Object> attributes =
2733             new LinkedHashMap<String,Object>(ldifLines.size());
2734        while (iterator.hasNext())
2735        {
2736          final StringBuilder line = iterator.next();
2737          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
2738          final int colonPos = line.indexOf(":");
2739          if (colonPos <= 0)
2740          {
2741            throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
2742                                    firstLineNumber, true, ldifLines, null);
2743          }
2744    
2745          final String attributeName = line.substring(0, colonPos);
2746          final String lowerName     = toLowerCase(attributeName);
2747    
2748          final MatchingRule matchingRule;
2749          if (schema == null)
2750          {
2751            matchingRule = CaseIgnoreStringMatchingRule.getInstance();
2752          }
2753          else
2754          {
2755            matchingRule =
2756                 MatchingRule.selectEqualityMatchingRule(attributeName, schema);
2757          }
2758    
2759          Attribute attr;
2760          final LDIFAttribute ldifAttr;
2761          final Object attrObject = attributes.get(lowerName);
2762          if (attrObject == null)
2763          {
2764            attr     = null;
2765            ldifAttr = null;
2766          }
2767          else
2768          {
2769            if (attrObject instanceof Attribute)
2770            {
2771              attr     = (Attribute) attrObject;
2772              ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
2773                                           attr.getRawValues()[0]);
2774              attributes.put(lowerName, ldifAttr);
2775            }
2776            else
2777            {
2778              attr     = null;
2779              ldifAttr = (LDIFAttribute) attrObject;
2780            }
2781          }
2782    
2783          final int length = line.length();
2784          if (length == (colonPos+1))
2785          {
2786            // This means that the attribute has a zero-length value, which is
2787            // acceptable.
2788            if (attrObject == null)
2789            {
2790              attr = new Attribute(attributeName, "");
2791              attributes.put(lowerName, attr);
2792            }
2793            else
2794            {
2795              try
2796              {
2797                if (! ldifAttr.addValue(new ASN1OctetString(),
2798                           duplicateValueBehavior))
2799                {
2800                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2801                  {
2802                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2803                         firstLineNumber, attributeName), firstLineNumber, true,
2804                         ldifLines, null);
2805                  }
2806                }
2807              }
2808              catch (LDAPException le)
2809              {
2810                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2811                     firstLineNumber, attributeName, getExceptionMessage(le)),
2812                     firstLineNumber, true, ldifLines, le);
2813              }
2814            }
2815          }
2816          else if (line.charAt(colonPos+1) == ':')
2817          {
2818            // Skip over any spaces leading up to the value, and then the rest of
2819            // the string is the base64-encoded attribute value.
2820            int pos = colonPos+2;
2821            while ((pos < length) && (line.charAt(pos) == ' '))
2822            {
2823              pos++;
2824            }
2825    
2826            try
2827            {
2828              final byte[] valueBytes = Base64.decode(line.substring(pos));
2829              if (attrObject == null)
2830              {
2831                attr = new Attribute(attributeName, valueBytes);
2832                attributes.put(lowerName, attr);
2833              }
2834              else
2835              {
2836                try
2837                {
2838                  if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
2839                             duplicateValueBehavior))
2840                  {
2841                    if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2842                    {
2843                      throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2844                           firstLineNumber, attributeName), firstLineNumber, true,
2845                           ldifLines, null);
2846                    }
2847                  }
2848                }
2849                catch (LDAPException le)
2850                {
2851                  throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2852                       firstLineNumber, attributeName, getExceptionMessage(le)),
2853                       firstLineNumber, true, ldifLines, le);
2854                }
2855              }
2856            }
2857            catch (final ParseException pe)
2858            {
2859              debugException(pe);
2860              throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
2861                                           attributeName,  firstLineNumber,
2862                                           pe.getMessage()),
2863                                      firstLineNumber, true, ldifLines, pe);
2864            }
2865          }
2866          else if (line.charAt(colonPos+1) == '<')
2867          {
2868            // Skip over any spaces leading up to the value, and then the rest of
2869            // the string is a URL that indicates where to get the real content.
2870            // At the present time, we'll only support the file URLs.
2871            int pos = colonPos+2;
2872            while ((pos < length) && (line.charAt(pos) == ' '))
2873            {
2874              pos++;
2875            }
2876    
2877            final byte[] urlBytes;
2878            final String urlString = line.substring(pos);
2879            try
2880            {
2881              urlBytes =
2882                   retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
2883            }
2884            catch (final Exception e)
2885            {
2886              debugException(e);
2887              throw new LDIFException(
2888                   ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2889                        firstLineNumber, e),
2890                   firstLineNumber, true, ldifLines, e);
2891            }
2892    
2893            if (attrObject == null)
2894            {
2895              attr = new Attribute(attributeName, urlBytes);
2896              attributes.put(lowerName, attr);
2897            }
2898            else
2899            {
2900              try
2901              {
2902                if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
2903                     duplicateValueBehavior))
2904                {
2905                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2906                  {
2907                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2908                         firstLineNumber, attributeName), firstLineNumber, true,
2909                         ldifLines, null);
2910                  }
2911                }
2912              }
2913              catch (final LDIFException le)
2914              {
2915                debugException(le);
2916                throw le;
2917              }
2918              catch (final Exception e)
2919              {
2920                debugException(e);
2921                throw new LDIFException(
2922                     ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
2923                          firstLineNumber, e),
2924                     firstLineNumber, true, ldifLines, e);
2925              }
2926            }
2927          }
2928          else
2929          {
2930            // Skip over any spaces leading up to the value, and then the rest of
2931            // the string is the value.
2932            int pos = colonPos+1;
2933            while ((pos < length) && (line.charAt(pos) == ' '))
2934            {
2935              pos++;
2936            }
2937    
2938            final String valueString = line.substring(pos);
2939            if (attrObject == null)
2940            {
2941              attr = new Attribute(attributeName, valueString);
2942              attributes.put(lowerName, attr);
2943            }
2944            else
2945            {
2946              try
2947              {
2948                if (! ldifAttr.addValue(new ASN1OctetString(valueString),
2949                           duplicateValueBehavior))
2950                {
2951                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
2952                  {
2953                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
2954                         firstLineNumber, attributeName), firstLineNumber, true,
2955                         ldifLines, null);
2956                  }
2957                }
2958              }
2959              catch (LDAPException le)
2960              {
2961                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
2962                     firstLineNumber, attributeName, getExceptionMessage(le)),
2963                     firstLineNumber, true, ldifLines, le);
2964              }
2965            }
2966          }
2967        }
2968    
2969        final ArrayList<Attribute> attrList =
2970             new ArrayList<Attribute>(attributes.size());
2971        for (final Object o : attributes.values())
2972        {
2973          if (o instanceof Attribute)
2974          {
2975            attrList.add((Attribute) o);
2976          }
2977          else
2978          {
2979            attrList.add(((LDIFAttribute) o).toAttribute());
2980          }
2981        }
2982    
2983        return attrList;
2984      }
2985    
2986    
2987    
2988      /**
2989       * Retrieves the bytes that make up the file referenced by the given URL.
2990       *
2991       * @param  urlString         The string representation of the URL to retrieve.
2992       * @param  relativeBasePath  The base path that will be prepended to relative
2993       *                           paths in order to obtain an absolute path.
2994       * @param  firstLineNumber   The line number for the start of the record.
2995       *
2996       * @return  The bytes contained in the specified file, or an empty array if
2997       *          the specified file is empty.
2998       *
2999       * @throws  LDIFException  If the provided URL is malformed or references a
3000       *                         nonexistent file.
3001       *
3002       * @throws  IOException  If a problem is encountered while attempting to read
3003       *                       from the target file.
3004       */
3005      private static byte[] retrieveURLBytes(final String urlString,
3006                                             final String relativeBasePath,
3007                                             final long firstLineNumber)
3008              throws LDIFException, IOException
3009      {
3010        int pos;
3011        String path;
3012        final String lowerURLString = toLowerCase(urlString);
3013        if (lowerURLString.startsWith("file:/"))
3014        {
3015          pos = 6;
3016          while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3017          {
3018            pos++;
3019          }
3020    
3021          path = urlString.substring(pos-1);
3022        }
3023        else if (lowerURLString.startsWith("file:"))
3024        {
3025          // A file: URL that doesn't include a slash will be interpreted as a
3026          // relative path.
3027          path = relativeBasePath + urlString.substring(5);
3028        }
3029        else
3030        {
3031          throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3032               firstLineNumber, true);
3033        }
3034    
3035        final File f = new File(path);
3036        if (! f.exists())
3037        {
3038          throw new LDIFException(
3039               ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3040               firstLineNumber, true);
3041        }
3042    
3043        // In order to conserve memory, we'll only allow values to be read from
3044        // files no larger than 10 megabytes.
3045        final long fileSize = f.length();
3046        if (fileSize > (10 * 1024 * 1024))
3047        {
3048          throw new LDIFException(
3049               ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3050                    (10*1024*1024)),
3051               firstLineNumber, true);
3052        }
3053    
3054        int fileBytesRemaining = (int) fileSize;
3055        final byte[] fileData = new byte[(int) fileSize];
3056        final FileInputStream fis = new FileInputStream(f);
3057        try
3058        {
3059          int fileBytesRead = 0;
3060          while (fileBytesRead < fileSize)
3061          {
3062            final int bytesRead =
3063                 fis.read(fileData, fileBytesRead, fileBytesRemaining);
3064            if (bytesRead < 0)
3065            {
3066              // We hit the end of the file before we expected to.  This shouldn't
3067              // happen unless the file size changed since we first looked at it,
3068              // which we won't allow.
3069              throw new LDIFException(
3070                   ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3071                        f.getAbsolutePath()),
3072                   firstLineNumber, true);
3073            }
3074    
3075            fileBytesRead      += bytesRead;
3076            fileBytesRemaining -= bytesRead;
3077          }
3078    
3079          if (fis.read() != -1)
3080          {
3081            // There is still more data to read.  This shouldn't happen unless the
3082            // file size changed since we first looked at it, which we won't allow.
3083            throw new LDIFException(
3084                 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3085                 firstLineNumber, true);
3086          }
3087        }
3088        finally
3089        {
3090          fis.close();
3091        }
3092    
3093        return fileData;
3094      }
3095    
3096    
3097    
3098      /**
3099       * Parses the data available through the provided iterator into an array of
3100       * modifications suitable for use in a modify change record.
3101       *
3102       * @param  dn                     The DN of the entry being parsed.
3103       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3104       *                                encountering attribute values which are not
3105       *                                base64-encoded but contain trailing spaces.
3106       * @param  ldifLines              The lines that comprise the LDIF
3107       *                                representation of the full record being
3108       *                                parsed.
3109       * @param  iterator               The iterator to use to access the
3110       *                                modification data.
3111       * @param  firstLineNumber        The line number for the start of the record.
3112       *
3113       * @return  An array containing the modifications that were read.
3114       *
3115       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3116       *                         set of modifications.
3117       */
3118      private static Modification[] parseModifications(final String dn,
3119           final TrailingSpaceBehavior trailingSpaceBehavior,
3120           final ArrayList<StringBuilder> ldifLines,
3121           final Iterator<StringBuilder> iterator, final long firstLineNumber)
3122           throws LDIFException
3123      {
3124        final ArrayList<Modification> modList =
3125             new ArrayList<Modification>(ldifLines.size());
3126    
3127        while (iterator.hasNext())
3128        {
3129          // The first line must start with "add:", "delete:", "replace:", or
3130          // "increment:" followed by an attribute name.
3131          StringBuilder line = iterator.next();
3132          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3133          int colonPos = line.indexOf(":");
3134          if (colonPos < 0)
3135          {
3136            throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3137                                    firstLineNumber, true, ldifLines, null);
3138          }
3139    
3140          final ModificationType modType;
3141          final String modTypeStr = toLowerCase(line.substring(0, colonPos));
3142          if (modTypeStr.equals("add"))
3143          {
3144            modType = ModificationType.ADD;
3145          }
3146          else if (modTypeStr.equals("delete"))
3147          {
3148            modType = ModificationType.DELETE;
3149          }
3150          else if (modTypeStr.equals("replace"))
3151          {
3152            modType = ModificationType.REPLACE;
3153          }
3154          else if (modTypeStr.equals("increment"))
3155          {
3156            modType = ModificationType.INCREMENT;
3157          }
3158          else
3159          {
3160            throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3161                                         firstLineNumber),
3162                                    firstLineNumber, true, ldifLines, null);
3163          }
3164    
3165          final String attributeName;
3166          int length = line.length();
3167          if (length == (colonPos+1))
3168          {
3169            // The colon was the last character on the line.  This is not
3170            // acceptable.
3171            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3172                                         firstLineNumber),
3173                                    firstLineNumber, true, ldifLines, null);
3174          }
3175          else if (line.charAt(colonPos+1) == ':')
3176          {
3177            // Skip over any spaces leading up to the value, and then the rest of
3178            // the string is the base64-encoded attribute name.
3179            int pos = colonPos+2;
3180            while ((pos < length) && (line.charAt(pos) == ' '))
3181            {
3182              pos++;
3183            }
3184    
3185            try
3186            {
3187              final byte[] dnBytes = Base64.decode(line.substring(pos));
3188              attributeName = new String(dnBytes, "UTF-8");
3189            }
3190            catch (final ParseException pe)
3191            {
3192              debugException(pe);
3193              throw new LDIFException(
3194                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3195                        firstLineNumber, pe.getMessage()),
3196                   firstLineNumber, true, ldifLines, pe);
3197            }
3198            catch (final Exception e)
3199            {
3200              debugException(e);
3201              throw new LDIFException(
3202                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3203                        firstLineNumber, e),
3204                   firstLineNumber, true, ldifLines, e);
3205            }
3206          }
3207          else
3208          {
3209            // Skip over any spaces leading up to the value, and then the rest of
3210            // the string is the attribute name.
3211            int pos = colonPos+1;
3212            while ((pos < length) && (line.charAt(pos) == ' '))
3213            {
3214              pos++;
3215            }
3216    
3217            attributeName = line.substring(pos);
3218          }
3219    
3220          if (attributeName.length() == 0)
3221          {
3222            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3223                                         firstLineNumber),
3224                                    firstLineNumber, true, ldifLines, null);
3225          }
3226    
3227    
3228          // The next zero or more lines may be the set of attribute values.  Keep
3229          // reading until we reach the end of the iterator or until we find a line
3230          // with just a "-".
3231          final ArrayList<ASN1OctetString> valueList =
3232               new ArrayList<ASN1OctetString>(ldifLines.size());
3233          while (iterator.hasNext())
3234          {
3235            line = iterator.next();
3236            handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3237            if (line.toString().equals("-"))
3238            {
3239              break;
3240            }
3241    
3242            colonPos = line.indexOf(":");
3243            if (colonPos < 0)
3244            {
3245              throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3246                                      firstLineNumber, true, ldifLines, null);
3247            }
3248            else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3249            {
3250              throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3251                                           firstLineNumber,
3252                                           line.substring(0, colonPos),
3253                                           attributeName),
3254                                      firstLineNumber, true, ldifLines, null);
3255            }
3256    
3257            final ASN1OctetString value;
3258            length = line.length();
3259            if (length == (colonPos+1))
3260            {
3261              // The colon was the last character on the line.  This is fine.
3262              value = new ASN1OctetString();
3263            }
3264            else if (line.charAt(colonPos+1) == ':')
3265            {
3266              // Skip over any spaces leading up to the value, and then the rest of
3267              // the string is the base64-encoded value.  This is unusual and
3268              // unnecessary, but is nevertheless acceptable.
3269              int pos = colonPos+2;
3270              while ((pos < length) && (line.charAt(pos) == ' '))
3271              {
3272                pos++;
3273              }
3274    
3275              try
3276              {
3277                value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3278              }
3279              catch (final ParseException pe)
3280              {
3281                debugException(pe);
3282                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3283                     attributeName, firstLineNumber, pe.getMessage()),
3284                     firstLineNumber, true, ldifLines, pe);
3285              }
3286              catch (final Exception e)
3287              {
3288                debugException(e);
3289                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3290                                             firstLineNumber, e),
3291                                        firstLineNumber, true, ldifLines, e);
3292              }
3293            }
3294            else
3295            {
3296              // Skip over any spaces leading up to the value, and then the rest of
3297              // the string is the value.
3298              int pos = colonPos+1;
3299              while ((pos < length) && (line.charAt(pos) == ' '))
3300              {
3301                pos++;
3302              }
3303    
3304              value = new ASN1OctetString(line.substring(pos));
3305            }
3306    
3307            valueList.add(value);
3308          }
3309    
3310          final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3311          valueList.toArray(values);
3312    
3313          // If it's an add modification type, then there must be at least one
3314          // value.
3315          if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3316              (values.length == 0))
3317          {
3318            throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3319                                         firstLineNumber),
3320                                    firstLineNumber, true, ldifLines, null);
3321          }
3322    
3323          // If it's an increment modification type, then there must be exactly one
3324          // value.
3325          if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3326              (values.length != 1))
3327          {
3328            throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3329                                         firstLineNumber, attributeName),
3330                                    firstLineNumber, true, ldifLines, null);
3331          }
3332    
3333          modList.add(new Modification(modType, attributeName, values));
3334        }
3335    
3336        final Modification[] mods = new Modification[modList.size()];
3337        modList.toArray(mods);
3338        return mods;
3339      }
3340    
3341    
3342    
3343      /**
3344       * Parses the data available through the provided iterator as the body of a
3345       * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3346       * newsuperior lines).
3347       *
3348       * @param  ldifLines              The lines that comprise the LDIF
3349       *                                representation of the full record being
3350       *                                parsed.
3351       * @param  iterator               The iterator to use to access the modify DN
3352       *                                data.
3353       * @param  dn                     The current DN of the entry.
3354       * @param  controls               The set of controls to include in the change
3355       *                                record.
3356       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3357       *                                encountering attribute values which are not
3358       *                                base64-encoded but contain trailing spaces.
3359       * @param  firstLineNumber        The line number for the start of the record.
3360       *
3361       * @return  The decoded modify DN change record.
3362       *
3363       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3364       *                         modify DN change record.
3365       */
3366      private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3367           final ArrayList<StringBuilder> ldifLines,
3368           final Iterator<StringBuilder> iterator, final String dn,
3369           final List<Control> controls,
3370           final TrailingSpaceBehavior trailingSpaceBehavior,
3371           final long firstLineNumber)
3372           throws LDIFException
3373      {
3374        // The next line must be the new RDN, and it must start with "newrdn:".
3375        StringBuilder line = iterator.next();
3376        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3377        int colonPos = line.indexOf(":");
3378        if ((colonPos < 0) ||
3379            (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3380        {
3381          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3382                                       firstLineNumber),
3383                                  firstLineNumber, true, ldifLines, null);
3384        }
3385    
3386        final String newRDN;
3387        int length = line.length();
3388        if (length == (colonPos+1))
3389        {
3390          // The colon was the last character on the line.  This is not acceptable.
3391          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3392                                       firstLineNumber),
3393                                  firstLineNumber, true, ldifLines, null);
3394        }
3395        else if (line.charAt(colonPos+1) == ':')
3396        {
3397          // Skip over any spaces leading up to the value, and then the rest of the
3398          // string is the base64-encoded new RDN.
3399          int pos = colonPos+2;
3400          while ((pos < length) && (line.charAt(pos) == ' '))
3401          {
3402            pos++;
3403          }
3404    
3405          try
3406          {
3407            final byte[] dnBytes = Base64.decode(line.substring(pos));
3408            newRDN = new String(dnBytes, "UTF-8");
3409          }
3410          catch (final ParseException pe)
3411          {
3412            debugException(pe);
3413            throw new LDIFException(
3414                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3415                                                                   pe.getMessage()),
3416                 firstLineNumber, true, ldifLines, pe);
3417          }
3418          catch (final Exception e)
3419          {
3420            debugException(e);
3421            throw new LDIFException(
3422                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3423                                                                   e),
3424                 firstLineNumber, true, ldifLines, e);
3425          }
3426        }
3427        else
3428        {
3429          // Skip over any spaces leading up to the value, and then the rest of the
3430          // string is the new RDN.
3431          int pos = colonPos+1;
3432          while ((pos < length) && (line.charAt(pos) == ' '))
3433          {
3434            pos++;
3435          }
3436    
3437          newRDN = line.substring(pos);
3438        }
3439    
3440        if (newRDN.length() == 0)
3441        {
3442          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3443                                       firstLineNumber),
3444                                  firstLineNumber, true, ldifLines, null);
3445        }
3446    
3447    
3448        // The next line must be the deleteOldRDN flag, and it must start with
3449        // 'deleteoldrdn:'.
3450        if (! iterator.hasNext())
3451        {
3452          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3453                                       firstLineNumber),
3454                                  firstLineNumber, true, ldifLines, null);
3455        }
3456    
3457        line = iterator.next();
3458        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3459        colonPos = line.indexOf(":");
3460        if ((colonPos < 0) ||
3461            (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3462        {
3463          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3464                                       firstLineNumber),
3465                                  firstLineNumber, true, ldifLines, null);
3466        }
3467    
3468        final String deleteOldRDNStr;
3469        length = line.length();
3470        if (length == (colonPos+1))
3471        {
3472          // The colon was the last character on the line.  This is not acceptable.
3473          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3474                                       firstLineNumber),
3475                                  firstLineNumber, true, ldifLines, null);
3476        }
3477        else if (line.charAt(colonPos+1) == ':')
3478        {
3479          // Skip over any spaces leading up to the value, and then the rest of the
3480          // string is the base64-encoded value.  This is unusual and
3481          // unnecessary, but is nevertheless acceptable.
3482          int pos = colonPos+2;
3483          while ((pos < length) && (line.charAt(pos) == ' '))
3484          {
3485            pos++;
3486          }
3487    
3488          try
3489          {
3490            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3491            deleteOldRDNStr = new String(changeTypeBytes, "UTF-8");
3492          }
3493          catch (final ParseException pe)
3494          {
3495            debugException(pe);
3496            throw new LDIFException(
3497                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3498                      firstLineNumber, pe.getMessage()),
3499                 firstLineNumber, true, ldifLines, pe);
3500          }
3501          catch (final Exception e)
3502          {
3503            debugException(e);
3504            throw new LDIFException(
3505                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3506                      firstLineNumber, e),
3507                 firstLineNumber, true, ldifLines, e);
3508          }
3509        }
3510        else
3511        {
3512          // Skip over any spaces leading up to the value, and then the rest of the
3513          // string is the value.
3514          int pos = colonPos+1;
3515          while ((pos < length) && (line.charAt(pos) == ' '))
3516          {
3517            pos++;
3518          }
3519    
3520          deleteOldRDNStr = line.substring(pos);
3521        }
3522    
3523        final boolean deleteOldRDN;
3524        if (deleteOldRDNStr.equals("0"))
3525        {
3526          deleteOldRDN = false;
3527        }
3528        else if (deleteOldRDNStr.equals("1"))
3529        {
3530          deleteOldRDN = true;
3531        }
3532        else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3533                 deleteOldRDNStr.equalsIgnoreCase("no"))
3534        {
3535          // This is technically illegal, but we'll allow it.
3536          deleteOldRDN = false;
3537        }
3538        else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3539                 deleteOldRDNStr.equalsIgnoreCase("yes"))
3540        {
3541          // This is also technically illegal, but we'll allow it.
3542          deleteOldRDN = false;
3543        }
3544        else
3545        {
3546          throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3547                                       deleteOldRDNStr, firstLineNumber),
3548                                  firstLineNumber, true, ldifLines, null);
3549        }
3550    
3551    
3552        // If there is another line, then it must be the new superior DN and it must
3553        // start with "newsuperior:".  If this is absent, then it's fine.
3554        final String newSuperiorDN;
3555        if (iterator.hasNext())
3556        {
3557          line = iterator.next();
3558          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3559          colonPos = line.indexOf(":");
3560          if ((colonPos < 0) ||
3561              (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3562          {
3563            throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3564                                         firstLineNumber),
3565                                    firstLineNumber, true, ldifLines, null);
3566          }
3567    
3568          length = line.length();
3569          if (length == (colonPos+1))
3570          {
3571            // The colon was the last character on the line.  This is fine.
3572            newSuperiorDN = "";
3573          }
3574          else if (line.charAt(colonPos+1) == ':')
3575          {
3576            // Skip over any spaces leading up to the value, and then the rest of
3577            // the string is the base64-encoded new superior DN.
3578            int pos = colonPos+2;
3579            while ((pos < length) && (line.charAt(pos) == ' '))
3580            {
3581              pos++;
3582            }
3583    
3584            try
3585            {
3586              final byte[] dnBytes = Base64.decode(line.substring(pos));
3587              newSuperiorDN = new String(dnBytes, "UTF-8");
3588            }
3589            catch (final ParseException pe)
3590            {
3591              debugException(pe);
3592              throw new LDIFException(
3593                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3594                        firstLineNumber, pe.getMessage()),
3595                   firstLineNumber, true, ldifLines, pe);
3596            }
3597            catch (final Exception e)
3598            {
3599              debugException(e);
3600              throw new LDIFException(
3601                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3602                        firstLineNumber, e),
3603                   firstLineNumber, true, ldifLines, e);
3604            }
3605          }
3606          else
3607          {
3608            // Skip over any spaces leading up to the value, and then the rest of
3609            // the string is the new superior DN.
3610            int pos = colonPos+1;
3611            while ((pos < length) && (line.charAt(pos) == ' '))
3612            {
3613              pos++;
3614            }
3615    
3616            newSuperiorDN = line.substring(pos);
3617          }
3618        }
3619        else
3620        {
3621          newSuperiorDN = null;
3622        }
3623    
3624    
3625        // There must not be any more lines.
3626        if (iterator.hasNext())
3627        {
3628          throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
3629                                  firstLineNumber, true, ldifLines, null);
3630        }
3631    
3632        return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
3633             newSuperiorDN, controls);
3634      }
3635    
3636    
3637    
3638      /**
3639       * Examines the line contained in the provided buffer to determine whether it
3640       * may contain one or more illegal trailing spaces.  If it does, then those
3641       * spaces will either be stripped out or an exception will be thrown to
3642       * indicate that they are illegal.
3643       *
3644       * @param  buffer                 The buffer to be examined.
3645       * @param  dn                     The DN of the LDIF record being parsed.  It
3646       *                                may be {@code null} if the DN is not yet
3647       *                                known (e.g., because the provided line is
3648       *                                expected to contain that DN).
3649       * @param  firstLineNumber        The approximate line number in the LDIF
3650       *                                source on which the LDIF record begins.
3651       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3652       *                                encountering attribute values which are not
3653       *                                base64-encoded but contain trailing spaces.
3654       *
3655       * @throws  LDIFException  If the line contained in the provided buffer ends
3656       *                         with one or more illegal trailing spaces and
3657       *                         {@code stripTrailingSpaces} was provided with a
3658       *                         value of {@code false}.
3659       */
3660      private static void handleTrailingSpaces(final StringBuilder buffer,
3661                               final String dn, final long firstLineNumber,
3662                               final TrailingSpaceBehavior trailingSpaceBehavior)
3663              throws LDIFException
3664      {
3665        int pos = buffer.length() - 1;
3666        boolean trailingFound = false;
3667        while ((pos >= 0) && (buffer.charAt(pos) == ' '))
3668        {
3669          trailingFound = true;
3670          pos--;
3671        }
3672    
3673        if (trailingFound && (buffer.charAt(pos) != ':'))
3674        {
3675          switch (trailingSpaceBehavior)
3676          {
3677            case STRIP:
3678              buffer.setLength(pos+1);
3679              break;
3680    
3681            case REJECT:
3682              if (dn == null)
3683              {
3684                throw new LDIFException(
3685                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
3686                          buffer.toString()),
3687                     firstLineNumber, true);
3688              }
3689              else
3690              {
3691                throw new LDIFException(
3692                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
3693                          firstLineNumber, buffer.toString()),
3694                     firstLineNumber, true);
3695              }
3696    
3697            case RETAIN:
3698            default:
3699              // No action will be taken.
3700              break;
3701          }
3702        }
3703      }
3704    
3705    
3706    
3707      /**
3708       * This represents an unparsed LDIFRecord.  It stores the line number of the
3709       * first line of the record and each line of the record.
3710       */
3711      private static final class UnparsedLDIFRecord
3712      {
3713        private final ArrayList<StringBuilder> lineList;
3714        private final long firstLineNumber;
3715        private final Exception failureCause;
3716        private final boolean isEOF;
3717        private final DuplicateValueBehavior duplicateValueBehavior;
3718        private final Schema schema;
3719        private final TrailingSpaceBehavior trailingSpaceBehavior;
3720    
3721    
3722    
3723        /**
3724         * Constructor.
3725         *
3726         * @param  lineList                The lines that comprise the LDIF record.
3727         * @param  duplicateValueBehavior  The behavior to exhibit if the entry
3728         *                                 contains duplicate attribute values.
3729         * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
3730         *                                 encountering trailing spaces in
3731         *                                 non-base64-encoded attribute values.
3732         * @param  schema                  The schema to use when parsing, if
3733         *                                 applicable.
3734         * @param  firstLineNumber         The first line number of the LDIF record.
3735         */
3736        private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
3737                     final DuplicateValueBehavior duplicateValueBehavior,
3738                     final TrailingSpaceBehavior trailingSpaceBehavior,
3739                     final Schema schema, final long firstLineNumber)
3740        {
3741          this.lineList               = lineList;
3742          this.firstLineNumber        = firstLineNumber;
3743          this.duplicateValueBehavior = duplicateValueBehavior;
3744          this.trailingSpaceBehavior  = trailingSpaceBehavior;
3745          this.schema                 = schema;
3746    
3747          failureCause = null;
3748          isEOF =
3749               (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
3750        }
3751    
3752    
3753    
3754        /**
3755         * Constructor.
3756         *
3757         * @param failureCause  The Exception thrown when reading from the input.
3758         */
3759        private UnparsedLDIFRecord(final Exception failureCause)
3760        {
3761          this.failureCause = failureCause;
3762    
3763          lineList               = null;
3764          firstLineNumber        = 0;
3765          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
3766          trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
3767          schema                 = null;
3768          isEOF                  = false;
3769        }
3770    
3771    
3772    
3773        /**
3774         * Return the lines that comprise the LDIF record.
3775         *
3776         * @return  The lines that comprise the LDIF record.
3777         */
3778        private ArrayList<StringBuilder> getLineList()
3779        {
3780          return lineList;
3781        }
3782    
3783    
3784    
3785        /**
3786         * Retrieves the behavior to exhibit when encountering duplicate attribute
3787         * values.
3788         *
3789         * @return  The behavior to exhibit when encountering duplicate attribute
3790         *          values.
3791         */
3792        private DuplicateValueBehavior getDuplicateValueBehavior()
3793        {
3794          return duplicateValueBehavior;
3795        }
3796    
3797    
3798    
3799        /**
3800         * Retrieves the behavior that should be exhibited when encountering
3801         * attribute values which are not base64-encoded but contain trailing
3802         * spaces.  The LDIF specification strongly recommends that any value which
3803         * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
3804         * LDIF parser may be configured to automatically strip these spaces, to
3805         * preserve them, or to reject any entry or change record containing them.
3806         *
3807         * @return  The behavior that should be exhibited when encountering
3808         *          attribute values which are not base64-encoded but contain
3809         *          trailing spaces.
3810         */
3811        private TrailingSpaceBehavior getTrailingSpaceBehavior()
3812        {
3813          return trailingSpaceBehavior;
3814        }
3815    
3816    
3817    
3818        /**
3819         * Retrieves the schema that should be used when parsing the record, if
3820         * applicable.
3821         *
3822         * @return  The schema that should be used when parsing the record, or
3823         *          {@code null} if none should be used.
3824         */
3825        private Schema getSchema()
3826        {
3827          return schema;
3828        }
3829    
3830    
3831    
3832        /**
3833         * Return the first line number of the LDIF record.
3834         *
3835         * @return  The first line number of the LDIF record.
3836         */
3837        private long getFirstLineNumber()
3838        {
3839          return firstLineNumber;
3840        }
3841    
3842    
3843    
3844        /**
3845         * Return {@code true} iff the end of the input was reached.
3846         *
3847         * @return  {@code true} iff the end of the input was reached.
3848         */
3849        private boolean isEOF()
3850        {
3851          return isEOF;
3852        }
3853    
3854    
3855    
3856        /**
3857         * Returns the reason that reading the record lines failed.  This normally
3858         * is only non-null if something bad happened to the input stream (like
3859         * a disk read error).
3860         *
3861         * @return  The reason that reading the record lines failed.
3862         */
3863        private Exception getFailureCause()
3864        {
3865          return failureCause;
3866        }
3867      }
3868    
3869    
3870      /**
3871       * When processing in asynchronous mode, this thread is responsible for
3872       * reading the raw unparsed records from the input and submitting them for
3873       * processing.
3874       */
3875      private final class LineReaderThread
3876           extends Thread
3877      {
3878        /**
3879         * Constructor.
3880         */
3881        private LineReaderThread()
3882        {
3883          super("Asynchronous LDIF line reader");
3884          setDaemon(true);
3885        }
3886    
3887    
3888    
3889        /**
3890         * Reads raw, unparsed records from the input and submits them for
3891         * processing until the input is finished or closed.
3892         */
3893        @Override()
3894        public void run()
3895        {
3896          try
3897          {
3898            boolean stopProcessing = false;
3899            while (!stopProcessing)
3900            {
3901              UnparsedLDIFRecord unparsedRecord = null;
3902              try
3903              {
3904                unparsedRecord = readUnparsedRecord();
3905              }
3906              catch (IOException e)
3907              {
3908                debugException(e);
3909                unparsedRecord = new UnparsedLDIFRecord(e);
3910                stopProcessing = true;
3911              }
3912              catch (Exception e)
3913              {
3914                debugException(e);
3915                unparsedRecord = new UnparsedLDIFRecord(e);
3916              }
3917    
3918              try
3919              {
3920                asyncParser.submit(unparsedRecord);
3921              }
3922              catch (InterruptedException e)
3923              {
3924                debugException(e);
3925                // If this thread is interrupted, then someone wants us to stop
3926                // processing, so that's what we'll do.
3927                stopProcessing = true;
3928              }
3929    
3930              if ((unparsedRecord == null) || (unparsedRecord.isEOF()))
3931              {
3932                stopProcessing = true;
3933              }
3934            }
3935          }
3936          finally
3937          {
3938            try
3939            {
3940              asyncParser.shutdown();
3941            }
3942            catch (InterruptedException e)
3943            {
3944              debugException(e);
3945            }
3946            finally
3947            {
3948              asyncParsingComplete.set(true);
3949            }
3950          }
3951        }
3952      }
3953    
3954    
3955    
3956      /**
3957       * Used to parse Records asynchronously.
3958       */
3959      private final class RecordParser implements Processor<UnparsedLDIFRecord,
3960                                                            LDIFRecord>
3961      {
3962        /**
3963         * {@inheritDoc}
3964         */
3965        public LDIFRecord process(final UnparsedLDIFRecord input)
3966               throws LDIFException
3967        {
3968          LDIFRecord record = decodeRecord(input, relativeBasePath);
3969    
3970          if ((record instanceof Entry) && (entryTranslator != null))
3971          {
3972            record = entryTranslator.translate((Entry) record,
3973                 input.getFirstLineNumber());
3974    
3975            if (record == null)
3976            {
3977              record = SKIP_ENTRY;
3978            }
3979          }
3980          return record;
3981        }
3982      }
3983    }