001    /*
002     * Copyright 2007-2016 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2016 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldif;
022    
023    
024    
025    import java.io.BufferedReader;
026    import java.io.BufferedWriter;
027    import java.io.Closeable;
028    import java.io.File;
029    import java.io.FileInputStream;
030    import java.io.FileWriter;
031    import java.io.InputStream;
032    import java.io.InputStreamReader;
033    import java.io.IOException;
034    import java.text.ParseException;
035    import java.util.ArrayList;
036    import java.util.Collection;
037    import java.util.Iterator;
038    import java.util.HashSet;
039    import java.util.LinkedHashMap;
040    import java.util.List;
041    import java.util.Set;
042    import java.util.concurrent.BlockingQueue;
043    import java.util.concurrent.ArrayBlockingQueue;
044    import java.util.concurrent.TimeUnit;
045    import java.util.concurrent.atomic.AtomicBoolean;
046    import java.nio.charset.Charset;
047    
048    import com.unboundid.asn1.ASN1OctetString;
049    import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
050    import com.unboundid.ldap.matchingrules.MatchingRule;
051    import com.unboundid.ldap.sdk.Attribute;
052    import com.unboundid.ldap.sdk.Control;
053    import com.unboundid.ldap.sdk.Entry;
054    import com.unboundid.ldap.sdk.Modification;
055    import com.unboundid.ldap.sdk.ModificationType;
056    import com.unboundid.ldap.sdk.LDAPException;
057    import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition;
058    import com.unboundid.ldap.sdk.schema.Schema;
059    import com.unboundid.util.AggregateInputStream;
060    import com.unboundid.util.Base64;
061    import com.unboundid.util.LDAPSDKThreadFactory;
062    import com.unboundid.util.parallel.AsynchronousParallelProcessor;
063    import com.unboundid.util.parallel.Result;
064    import com.unboundid.util.parallel.ParallelProcessor;
065    import com.unboundid.util.parallel.Processor;
066    
067    import static com.unboundid.ldif.LDIFMessages.*;
068    import static com.unboundid.util.Debug.*;
069    import static com.unboundid.util.StaticUtils.*;
070    import static com.unboundid.util.Validator.*;
071    
072    /**
073     * This class provides an LDIF reader, which can be used to read and decode
074     * entries and change records from a data source using the LDAP Data Interchange
075     * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
076     * <BR>
077     * This class is not synchronized.  If multiple threads read from the
078     * LDIFReader, they must be synchronized externally.
079     * <BR><BR>
080     * <H2>Example</H2>
081     * The following example iterates through all entries contained in an LDIF file
082     * and attempts to add them to a directory server:
083     * <PRE>
084     * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
085     *
086     * int entriesRead = 0;
087     * int entriesAdded = 0;
088     * int errorsEncountered = 0;
089     * while (true)
090     * {
091     *   Entry entry;
092     *   try
093     *   {
094     *     entry = ldifReader.readEntry();
095     *     if (entry == null)
096     *     {
097     *       // All entries have been read.
098     *       break;
099     *     }
100     *
101     *     entriesRead++;
102     *   }
103     *   catch (LDIFException le)
104     *   {
105     *     errorsEncountered++;
106     *     if (le.mayContinueReading())
107     *     {
108     *       // A recoverable error occurred while attempting to read a change
109     *       // record, at or near line number le.getLineNumber()
110     *       // The entry will be skipped, but we'll try to keep reading from the
111     *       // LDIF file.
112     *       continue;
113     *     }
114     *     else
115     *     {
116     *       // An unrecoverable error occurred while attempting to read an entry
117     *       // at or near line number le.getLineNumber()
118     *       // No further LDIF processing will be performed.
119     *       break;
120     *     }
121     *   }
122     *   catch (IOException ioe)
123     *   {
124     *     // An I/O error occurred while attempting to read from the LDIF file.
125     *     // No further LDIF processing will be performed.
126     *     errorsEncountered++;
127     *     break;
128     *   }
129     *
130     *   LDAPResult addResult;
131     *   try
132     *   {
133     *     addResult = connection.add(entry);
134     *     // If we got here, then the change should have been processed
135     *     // successfully.
136     *     entriesAdded++;
137     *   }
138     *   catch (LDAPException le)
139     *   {
140     *     // If we got here, then the change attempt failed.
141     *     addResult = le.toLDAPResult();
142     *     errorsEncountered++;
143     *   }
144     * }
145     *
146     * ldifReader.close();
147     * </PRE>
148     */
149    public final class LDIFReader
150           implements Closeable
151    {
152      /**
153       * The default buffer size (128KB) that will be used when reading from the
154       * data source.
155       */
156      public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
157    
158    
159    
160      /*
161       * When processing asynchronously, this determines how many of the allocated
162       * worker threads are used to parse each batch of read entries.
163       */
164      private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
165    
166    
167    
168      /**
169       * When processing asynchronously, this specifies the size of the pending and
170       * completed queues.
171       */
172      private static final int ASYNC_QUEUE_SIZE = 500;
173    
174    
175    
176      /**
177       * Special entry used internally to signal that the LDIFReaderEntryTranslator
178       * has signalled that a read Entry should be skipped by returning null,
179       * which normally implies EOF.
180       */
181      private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
182    
183    
184    
185      /**
186       * The default base path that will be prepended to relative paths.  It will
187       * end with a trailing slash.
188       */
189      private static final String DEFAULT_RELATIVE_BASE_PATH;
190      static
191      {
192        final File currentDir;
193        String currentDirString = System.getProperty("user.dir");
194        if (currentDirString == null)
195        {
196          currentDir = new File(".");
197        }
198        else
199        {
200          currentDir = new File(currentDirString);
201        }
202    
203        final String currentDirAbsolutePath = currentDir.getAbsolutePath();
204        if (currentDirAbsolutePath.endsWith(File.separator))
205        {
206          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
207        }
208        else
209        {
210          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
211        }
212      }
213    
214    
215    
216      // The buffered reader that will be used to read LDIF data.
217      private final BufferedReader reader;
218    
219      // The behavior that should be exhibited when encountering duplicate attribute
220      // values.
221      private volatile DuplicateValueBehavior duplicateValueBehavior;
222    
223      // A line number counter.
224      private long lineNumberCounter = 0;
225    
226      // The change record translator to use, if any.
227      private final LDIFReaderChangeRecordTranslator changeRecordTranslator;
228    
229      // The entry translator to use, if any.
230      private final LDIFReaderEntryTranslator entryTranslator;
231    
232      // The schema that will be used when processing, if applicable.
233      private Schema schema;
234    
235      // Specifies the base path that will be prepended to relative paths for file
236      // URLs.
237      private volatile String relativeBasePath;
238    
239      // The behavior that should be exhibited with regard to illegal trailing
240      // spaces in attribute values.
241      private volatile TrailingSpaceBehavior trailingSpaceBehavior;
242    
243      // True iff we are processing asynchronously.
244      private final boolean isAsync;
245    
246      //
247      // The following only apply to asynchronous processing.
248      //
249    
250      // Parses entries asynchronously.
251      private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord>
252           asyncParser;
253    
254      // Set to true when the end of the input is reached.
255      private final AtomicBoolean asyncParsingComplete;
256    
257      // The records that have been read and parsed.
258      private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>>
259           asyncParsedRecords;
260    
261    
262    
263      /**
264       * Creates a new LDIF reader that will read data from the specified file.
265       *
266       * @param  path  The path to the file from which the data is to be read.  It
267       *               must not be {@code null}.
268       *
269       * @throws  IOException  If a problem occurs while opening the file for
270       *                       reading.
271       */
272      public LDIFReader(final String path)
273             throws IOException
274      {
275        this(new FileInputStream(path));
276      }
277    
278    
279    
280      /**
281       * Creates a new LDIF reader that will read data from the specified file
282       * and parses the LDIF records asynchronously using the specified number of
283       * threads.
284       *
285       * @param  path  The path to the file from which the data is to be read.  It
286       *               must not be {@code null}.
287       * @param  numParseThreads  If this value is greater than zero, then the
288       *                          specified number of threads will be used to
289       *                          asynchronously read and parse the LDIF file.
290       *
291       * @throws  IOException  If a problem occurs while opening the file for
292       *                       reading.
293       *
294       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
295       *      constructor for more details about asynchronous processing.
296       */
297      public LDIFReader(final String path, final int numParseThreads)
298             throws IOException
299      {
300        this(new FileInputStream(path), numParseThreads);
301      }
302    
303    
304    
305      /**
306       * Creates a new LDIF reader that will read data from the specified file.
307       *
308       * @param  file  The file from which the data is to be read.  It must not be
309       *               {@code null}.
310       *
311       * @throws  IOException  If a problem occurs while opening the file for
312       *                       reading.
313       */
314      public LDIFReader(final File file)
315             throws IOException
316      {
317        this(new FileInputStream(file));
318      }
319    
320    
321    
322      /**
323       * Creates a new LDIF reader that will read data from the specified file
324       * and optionally parses the LDIF records asynchronously using the specified
325       * number of threads.
326       *
327       * @param  file             The file from which the data is to be read.  It
328       *                          must not be {@code null}.
329       * @param  numParseThreads  If this value is greater than zero, then the
330       *                          specified number of threads will be used to
331       *                          asynchronously read and parse the LDIF file.
332       *
333       * @throws  IOException  If a problem occurs while opening the file for
334       *                       reading.
335       */
336      public LDIFReader(final File file, final int numParseThreads)
337             throws IOException
338      {
339        this(new FileInputStream(file), numParseThreads);
340      }
341    
342    
343    
344      /**
345       * Creates a new LDIF reader that will read data from the specified files in
346       * the order in which they are provided and optionally parses the LDIF records
347       * asynchronously using the specified number of threads.
348       *
349       * @param  files            The files from which the data is to be read.  It
350       *                          must not be {@code null} or empty.
351       * @param  numParseThreads  If this value is greater than zero, then the
352       *                          specified number of threads will be used to
353       *                          asynchronously read and parse the LDIF file.
354       * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
355       *                          before they are returned.  This is normally
356       *                          {@code null}, which causes entries to be returned
357       *                          unaltered. This is particularly useful when
358       *                          parsing the input file in parallel because the
359       *                          entry translation is also done in parallel.
360       *
361       * @throws  IOException  If a problem occurs while opening the file for
362       *                       reading.
363       */
364      public LDIFReader(final File[] files, final int numParseThreads,
365                        final LDIFReaderEntryTranslator entryTranslator)
366             throws IOException
367      {
368        this(files, numParseThreads, entryTranslator, null);
369      }
370    
371    
372    
373      /**
374       * Creates a new LDIF reader that will read data from the specified files in
375       * the order in which they are provided and optionally parses the LDIF records
376       * asynchronously using the specified number of threads.
377       *
378       * @param  files                   The files from which the data is to be
379       *                                 read.  It must not be {@code null} or
380       *                                 empty.
381       * @param  numParseThreads         If this value is greater than zero, then
382       *                                 the specified number of threads will be
383       *                                 used to asynchronously read and parse the
384       *                                 LDIF file.
385       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
386       *                                 entries before they are returned.  This is
387       *                                 normally {@code null}, which causes entries
388       *                                 to be returned unaltered.  This is
389       *                                 particularly useful when parsing the input
390       *                                 file in parallel because the entry
391       *                                 translation is also done in parallel.
392       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
393       *                                 apply to change records before they are
394       *                                 returned.  This is normally {@code null},
395       *                                 which causes change records to be returned
396       *                                 unaltered.  This is particularly useful
397       *                                 when parsing the input file in parallel
398       *                                 because the change record translation is
399       *                                 also done in parallel.
400       *
401       * @throws  IOException  If a problem occurs while opening the file for
402       *                       reading.
403       */
404      public LDIFReader(final File[] files, final int numParseThreads,
405                  final LDIFReaderEntryTranslator entryTranslator,
406                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
407             throws IOException
408      {
409        this(files, numParseThreads, entryTranslator, changeRecordTranslator,
410             "UTF-8");
411      }
412    
413    
414    
415      /**
416       * Creates a new LDIF reader that will read data from the specified files in
417       * the order in which they are provided and optionally parses the LDIF records
418       * asynchronously using the specified number of threads.
419       *
420       * @param  files                   The files from which the data is to be
421       *                                 read.  It must not be {@code null} or
422       *                                 empty.
423       * @param  numParseThreads         If this value is greater than zero, then
424       *                                 the specified number of threads will be
425       *                                 used to asynchronously read and parse the
426       *                                 LDIF file.
427       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
428       *                                 entries before they are returned.  This is
429       *                                 normally {@code null}, which causes entries
430       *                                 to be returned unaltered.  This is
431       *                                 particularly useful when parsing the input
432       *                                 file in parallel because the entry
433       *                                 translation is also done in parallel.
434       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
435       *                                 apply to change records before they are
436       *                                 returned.  This is normally {@code null},
437       *                                 which causes change records to be returned
438       *                                 unaltered.  This is particularly useful
439       *                                 when parsing the input file in parallel
440       *                                 because the change record translation is
441       *                                 also done in parallel.
442       * @param  characterSet            The character set to use when reading from
443       *                                 the input stream.  It must not be
444       *                                 {@code null}.
445       *
446       * @throws  IOException  If a problem occurs while opening the file for
447       *                       reading.
448       */
449      public LDIFReader(final File[] files, final int numParseThreads,
450                  final LDIFReaderEntryTranslator entryTranslator,
451                  final LDIFReaderChangeRecordTranslator changeRecordTranslator,
452                  final String characterSet)
453             throws IOException
454      {
455        this(createAggregateInputStream(files), numParseThreads, entryTranslator,
456             changeRecordTranslator, characterSet);
457      }
458    
459    
460    
461      /**
462       * Creates a new aggregate input stream that will read data from the specified
463       * files.  If there are multiple files, then a "padding" file will be inserted
464       * between them to ensure that there is at least one blank line between the
465       * end of one file and the beginning of another.
466       *
467       * @param  files  The files from which the data is to be read.  It must not be
468       *                {@code null} or empty.
469       *
470       * @return  The input stream to use to read data from the provided files.
471       *
472       * @throws  IOException  If a problem is encountered while attempting to
473       *                       create the input stream.
474       */
475      private static InputStream createAggregateInputStream(final File... files)
476              throws IOException
477      {
478        if (files.length == 0)
479        {
480          throw new IOException(ERR_READ_NO_LDIF_FILES.get());
481        }
482        else if (files.length == 1)
483        {
484          return new FileInputStream(files[0]);
485        }
486        else
487        {
488          final File spacerFile =
489               File.createTempFile("ldif-reader-spacer", ".ldif");
490          spacerFile.deleteOnExit();
491    
492          final BufferedWriter spacerWriter =
493               new BufferedWriter(new FileWriter(spacerFile));
494          try
495          {
496            spacerWriter.newLine();
497            spacerWriter.newLine();
498          }
499          finally
500          {
501            spacerWriter.close();
502          }
503    
504          final File[] returnArray = new File[(files.length * 2) - 1];
505          returnArray[0] = files[0];
506    
507          int pos = 1;
508          for (int i=1; i < files.length; i++)
509          {
510            returnArray[pos++] = spacerFile;
511            returnArray[pos++] = files[i];
512          }
513    
514          return new AggregateInputStream(returnArray);
515        }
516      }
517    
518    
519    
520      /**
521       * Creates a new LDIF reader that will read data from the provided input
522       * stream.
523       *
524       * @param  inputStream  The input stream from which the data is to be read.
525       *                      It must not be {@code null}.
526       */
527      public LDIFReader(final InputStream inputStream)
528      {
529        this(inputStream, 0);
530      }
531    
532    
533    
534      /**
535       * Creates a new LDIF reader that will read data from the specified stream
536       * and parses the LDIF records asynchronously using the specified number of
537       * threads.
538       *
539       * @param  inputStream  The input stream from which the data is to be read.
540       *                      It must not be {@code null}.
541       * @param  numParseThreads  If this value is greater than zero, then the
542       *                          specified number of threads will be used to
543       *                          asynchronously read and parse the LDIF file.
544       *
545       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
546       *      constructor for more details about asynchronous processing.
547       */
548      public LDIFReader(final InputStream inputStream, final int numParseThreads)
549      {
550        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
551        this(new BufferedReader(new InputStreamReader(inputStream,
552                                                      Charset.forName("UTF-8")),
553                                DEFAULT_BUFFER_SIZE),
554             numParseThreads);
555      }
556    
557    
558    
559      /**
560       * Creates a new LDIF reader that will read data from the specified stream
561       * and parses the LDIF records asynchronously using the specified number of
562       * threads.
563       *
564       * @param  inputStream  The input stream from which the data is to be read.
565       *                      It must not be {@code null}.
566       * @param  numParseThreads  If this value is greater than zero, then the
567       *                          specified number of threads will be used to
568       *                          asynchronously read and parse the LDIF file.
569       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
570       *                         entries before they are returned.  This is normally
571       *                         {@code null}, which causes entries to be returned
572       *                         unaltered. This is particularly useful when parsing
573       *                         the input file in parallel because the entry
574       *                         translation is also done in parallel.
575       *
576       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
577       *      constructor for more details about asynchronous processing.
578       */
579      public LDIFReader(final InputStream inputStream, final int numParseThreads,
580                        final LDIFReaderEntryTranslator entryTranslator)
581      {
582        this(inputStream, numParseThreads, entryTranslator, null);
583      }
584    
585    
586    
587      /**
588       * Creates a new LDIF reader that will read data from the specified stream
589       * and parses the LDIF records asynchronously using the specified number of
590       * threads.
591       *
592       * @param  inputStream             The input stream from which the data is to
593       *                                 be read.  It must not be {@code null}.
594       * @param  numParseThreads         If this value is greater than zero, then
595       *                                 the specified number of threads will be
596       *                                 used to asynchronously read and parse the
597       *                                 LDIF file.
598       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
599       *                                 entries before they are returned.  This is
600       *                                 normally {@code null}, which causes entries
601       *                                 to be returned unaltered.  This is
602       *                                 particularly useful when parsing the input
603       *                                 file in parallel because the entry
604       *                                 translation is also done in parallel.
605       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
606       *                                 apply to change records before they are
607       *                                 returned.  This is normally {@code null},
608       *                                 which causes change records to be returned
609       *                                 unaltered.  This is particularly useful
610       *                                 when parsing the input file in parallel
611       *                                 because the change record translation is
612       *                                 also done in parallel.
613       *
614       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
615       *      constructor for more details about asynchronous processing.
616       */
617      public LDIFReader(final InputStream inputStream, final int numParseThreads,
618                  final LDIFReaderEntryTranslator entryTranslator,
619                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
620      {
621        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
622        this(inputStream, numParseThreads, entryTranslator, changeRecordTranslator,
623             "UTF-8");
624      }
625    
626    
627    
628      /**
629       * Creates a new LDIF reader that will read data from the specified stream
630       * and parses the LDIF records asynchronously using the specified number of
631       * threads.
632       *
633       * @param  inputStream             The input stream from which the data is to
634       *                                 be read.  It must not be {@code null}.
635       * @param  numParseThreads         If this value is greater than zero, then
636       *                                 the specified number of threads will be
637       *                                 used to asynchronously read and parse the
638       *                                 LDIF file.
639       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
640       *                                 entries before they are returned.  This is
641       *                                 normally {@code null}, which causes entries
642       *                                 to be returned unaltered.  This is
643       *                                 particularly useful when parsing the input
644       *                                 file in parallel because the entry
645       *                                 translation is also done in parallel.
646       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
647       *                                 apply to change records before they are
648       *                                 returned.  This is normally {@code null},
649       *                                 which causes change records to be returned
650       *                                 unaltered.  This is particularly useful
651       *                                 when parsing the input file in parallel
652       *                                 because the change record translation is
653       *                                 also done in parallel.
654       * @param  characterSet            The character set to use when reading from
655       *                                 the input stream.  It must not be
656       *                                 {@code null}.
657       *
658       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
659       *      constructor for more details about asynchronous processing.
660       */
661      public LDIFReader(final InputStream inputStream, final int numParseThreads,
662                  final LDIFReaderEntryTranslator entryTranslator,
663                  final LDIFReaderChangeRecordTranslator changeRecordTranslator,
664                  final String characterSet)
665      {
666        this(new BufferedReader(
667                  new InputStreamReader(inputStream, Charset.forName(characterSet)),
668                  DEFAULT_BUFFER_SIZE),
669             numParseThreads, entryTranslator, changeRecordTranslator);
670      }
671    
672    
673    
674      /**
675       * Creates a new LDIF reader that will use the provided buffered reader to
676       * read the LDIF data.  The encoding of the underlying Reader must be set to
677       * "UTF-8" as required by RFC 2849.
678       *
679       * @param  reader  The buffered reader that will be used to read the LDIF
680       *                 data.  It must not be {@code null}.
681       */
682      public LDIFReader(final BufferedReader reader)
683      {
684        this(reader, 0);
685      }
686    
687    
688    
689      /**
690       * Creates a new LDIF reader that will read data from the specified buffered
691       * reader and parses the LDIF records asynchronously using the specified
692       * number of threads.  The encoding of the underlying Reader must be set to
693       * "UTF-8" as required by RFC 2849.
694       *
695       * @param reader The buffered reader that will be used to read the LDIF data.
696       *               It must not be {@code null}.
697       * @param  numParseThreads  If this value is greater than zero, then the
698       *                          specified number of threads will be used to
699       *                          asynchronously read and parse the LDIF file.
700       *
701       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
702       *      constructor for more details about asynchronous processing.
703       */
704      public LDIFReader(final BufferedReader reader, final int numParseThreads)
705      {
706        this(reader, numParseThreads, null);
707      }
708    
709    
710    
711      /**
712       * Creates a new LDIF reader that will read data from the specified buffered
713       * reader and parses the LDIF records asynchronously using the specified
714       * number of threads.  The encoding of the underlying Reader must be set to
715       * "UTF-8" as required by RFC 2849.
716       *
717       * @param reader The buffered reader that will be used to read the LDIF data.
718       *               It must not be {@code null}.
719       * @param  numParseThreads  If this value is greater than zero, then the
720       *                          specified number of threads will be used to
721       *                          asynchronously read and parse the LDIF file.
722       *                          This should only be set to greater than zero when
723       *                          performance analysis has demonstrated that reading
724       *                          and parsing the LDIF is a bottleneck.  The default
725       *                          synchronous processing is normally fast enough.
726       *                          There is little benefit in passing in a value
727       *                          greater than four (unless there is an
728       *                          LDIFReaderEntryTranslator that does time-consuming
729       *                          processing).  A value of zero implies the
730       *                          default behavior of reading and parsing LDIF
731       *                          records synchronously when one of the read
732       *                          methods is called.
733       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
734       *                         entries before they are returned.  This is normally
735       *                         {@code null}, which causes entries to be returned
736       *                         unaltered. This is particularly useful when parsing
737       *                         the input file in parallel because the entry
738       *                         translation is also done in parallel.
739       */
740      public LDIFReader(final BufferedReader reader,
741                        final int numParseThreads,
742                        final LDIFReaderEntryTranslator entryTranslator)
743      {
744        this(reader, numParseThreads, entryTranslator, null);
745      }
746    
747    
748    
749      /**
750       * Creates a new LDIF reader that will read data from the specified buffered
751       * reader and parses the LDIF records asynchronously using the specified
752       * number of threads.  The encoding of the underlying Reader must be set to
753       * "UTF-8" as required by RFC 2849.
754       *
755       * @param reader                   The buffered reader that will be used to
756       *                                 read the LDIF data.  It must not be
757       *                                 {@code null}.
758       * @param  numParseThreads         If this value is greater than zero, then
759       *                                 the specified number of threads will be
760       *                                 used to asynchronously read and parse the
761       *                                 LDIF file.
762       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
763       *                                 entries before they are returned.  This is
764       *                                 normally {@code null}, which causes entries
765       *                                 to be returned unaltered.  This is
766       *                                 particularly useful when parsing the input
767       *                                 file in parallel because the entry
768       *                                 translation is also done in parallel.
769       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
770       *                                 apply to change records before they are
771       *                                 returned.  This is normally {@code null},
772       *                                 which causes change records to be returned
773       *                                 unaltered.  This is particularly useful
774       *                                 when parsing the input file in parallel
775       *                                 because the change record translation is
776       *                                 also done in parallel.
777       */
778      public LDIFReader(final BufferedReader reader, final int numParseThreads,
779                  final LDIFReaderEntryTranslator entryTranslator,
780                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
781      {
782        ensureNotNull(reader);
783        ensureTrue(numParseThreads >= 0,
784                   "LDIFReader.numParseThreads must not be negative.");
785    
786        this.reader = reader;
787        this.entryTranslator = entryTranslator;
788        this.changeRecordTranslator = changeRecordTranslator;
789    
790        duplicateValueBehavior = DuplicateValueBehavior.STRIP;
791        trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
792    
793        relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
794    
795        if (numParseThreads == 0)
796        {
797          isAsync = false;
798          asyncParser = null;
799          asyncParsingComplete = null;
800          asyncParsedRecords = null;
801        }
802        else
803        {
804          isAsync = true;
805          asyncParsingComplete = new AtomicBoolean(false);
806    
807          // Decodes entries in parallel.
808          final LDAPSDKThreadFactory threadFactory =
809               new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
810          final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
811               new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>(
812                    new RecordParser(), threadFactory, numParseThreads,
813                    ASYNC_MIN_PER_PARSING_THREAD);
814    
815          final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
816               ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE);
817    
818          // The output queue must be a little more than twice as big as the input
819          // queue to more easily handle being shutdown in the middle of processing
820          // when the queues are full and threads are blocked.
821          asyncParsedRecords = new ArrayBlockingQueue
822               <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100);
823    
824          asyncParser = new AsynchronousParallelProcessor
825               <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser,
826                                                asyncParsedRecords);
827    
828          final LineReaderThread lineReaderThread = new LineReaderThread();
829          lineReaderThread.start();
830        }
831      }
832    
833    
834    
835      /**
836       * Reads entries from the LDIF file with the specified path and returns them
837       * as a {@code List}.  This is a convenience method that should only be used
838       * for data sets that are small enough so that running out of memory isn't a
839       * concern.
840       *
841       * @param  path  The path to the LDIF file containing the entries to be read.
842       *
843       * @return  A list of the entries read from the given LDIF file.
844       *
845       * @throws  IOException  If a problem occurs while attempting to read data
846       *                       from the specified file.
847       *
848       * @throws  LDIFException  If a problem is encountered while attempting to
849       *                         decode data read as LDIF.
850       */
851      public static List<Entry> readEntries(final String path)
852             throws IOException, LDIFException
853      {
854        return readEntries(new LDIFReader(path));
855      }
856    
857    
858    
859      /**
860       * Reads entries from the specified LDIF file and returns them as a
861       * {@code List}.  This is a convenience method that should only be used for
862       * data sets that are small enough so that running out of memory isn't a
863       * concern.
864       *
865       * @param  file  A reference to the LDIF file containing the entries to be
866       *               read.
867       *
868       * @return  A list of the entries read from the given LDIF file.
869       *
870       * @throws  IOException  If a problem occurs while attempting to read data
871       *                       from the specified file.
872       *
873       * @throws  LDIFException  If a problem is encountered while attempting to
874       *                         decode data read as LDIF.
875       */
876      public static List<Entry> readEntries(final File file)
877             throws IOException, LDIFException
878      {
879        return readEntries(new LDIFReader(file));
880      }
881    
882    
883    
884      /**
885       * Reads and decodes LDIF entries from the provided input stream and
886       * returns them as a {@code List}.  This is a convenience method that should
887       * only be used for data sets that are small enough so that running out of
888       * memory isn't a concern.
889       *
890       * @param  inputStream  The input stream from which the entries should be
891       *                      read.  The input stream will be closed before
892       *                      returning.
893       *
894       * @return  A list of the entries read from the given input stream.
895       *
896       * @throws  IOException  If a problem occurs while attempting to read data
897       *                       from the input stream.
898       *
899       * @throws  LDIFException  If a problem is encountered while attempting to
900       *                         decode data read as LDIF.
901       */
902      public static List<Entry> readEntries(final InputStream inputStream)
903             throws IOException, LDIFException
904      {
905        return readEntries(new LDIFReader(inputStream));
906      }
907    
908    
909    
910      /**
911       * Reads entries from the provided LDIF reader and returns them as a list.
912       *
913       * @param  reader  The reader from which the entries should be read.  It will
914       *                 be closed before returning.
915       *
916       * @return  A list of the entries read from the provided reader.
917       *
918       * @throws  IOException  If a problem was encountered while attempting to read
919       *                       data from the LDIF data source.
920       *
921       * @throws  LDIFException  If a problem is encountered while attempting to
922       *                         decode data read as LDIF.
923       */
924      private static List<Entry> readEntries(final LDIFReader reader)
925              throws IOException, LDIFException
926      {
927        try
928        {
929          final ArrayList<Entry> entries = new ArrayList<Entry>(10);
930          while (true)
931          {
932            final Entry e = reader.readEntry();
933            if (e == null)
934            {
935              break;
936            }
937    
938            entries.add(e);
939          }
940    
941          return entries;
942        }
943        finally
944        {
945          reader.close();
946        }
947      }
948    
949    
950    
951      /**
952       * Closes this LDIF reader and the underlying LDIF source.
953       *
954       * @throws  IOException  If a problem occurs while closing the underlying LDIF
955       *                       source.
956       */
957      public void close()
958             throws IOException
959      {
960        reader.close();
961    
962        if (isAsync())
963        {
964          // Closing the reader will trigger the LineReaderThread to complete, but
965          // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
966          // this, we clear out the completed output queue, which is larger than
967          // the input queue, so the LineReaderThread will stop reading and
968          // shutdown the asyncParser.
969          asyncParsedRecords.clear();
970        }
971      }
972    
973    
974    
975      /**
976       * Indicates whether to ignore any duplicate values encountered while reading
977       * LDIF records.
978       *
979       * @return  {@code true} if duplicate values should be ignored, or
980       *          {@code false} if any LDIF records containing duplicate values
981       *          should be rejected.
982       *
983       * @deprecated  Use the {@code getDuplicateValueBehavior} method instead.
984       */
985      @Deprecated()
986      public boolean ignoreDuplicateValues()
987      {
988        return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
989      }
990    
991    
992    
993      /**
994       * Specifies whether to ignore any duplicate values encountered while reading
995       * LDIF records.
996       *
997       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
998       *                                attribute values encountered while reading
999       *                                LDIF records.
1000       *
1001       * @deprecated  Use the {@code setDuplicateValueBehavior} method instead.
1002       */
1003      @Deprecated()
1004      public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
1005      {
1006        if (ignoreDuplicateValues)
1007        {
1008          duplicateValueBehavior = DuplicateValueBehavior.STRIP;
1009        }
1010        else
1011        {
1012          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
1013        }
1014      }
1015    
1016    
1017    
1018      /**
1019       * Retrieves the behavior that should be exhibited if the LDIF reader
1020       * encounters an entry with duplicate values.
1021       *
1022       * @return  The behavior that should be exhibited if the LDIF reader
1023       *          encounters an entry with duplicate values.
1024       */
1025      public DuplicateValueBehavior getDuplicateValueBehavior()
1026      {
1027        return duplicateValueBehavior;
1028      }
1029    
1030    
1031    
1032      /**
1033       * Specifies the behavior that should be exhibited if the LDIF reader
1034       * encounters an entry with duplicate values.
1035       *
1036       * @param  duplicateValueBehavior  The behavior that should be exhibited if
1037       *                                 the LDIF reader encounters an entry with
1038       *                                 duplicate values.
1039       */
1040      public void setDuplicateValueBehavior(
1041                       final DuplicateValueBehavior duplicateValueBehavior)
1042      {
1043        this.duplicateValueBehavior = duplicateValueBehavior;
1044      }
1045    
1046    
1047    
1048      /**
1049       * Indicates whether to strip off any illegal trailing spaces that may appear
1050       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1051       * specification strongly recommends that any value which legitimately
1052       * contains trailing spaces be base64-encoded, and any spaces which appear
1053       * after the end of non-base64-encoded values may therefore be considered
1054       * invalid.  If any such trailing spaces are encountered in an LDIF record and
1055       * they are not to be stripped, then an {@code LDIFException} will be thrown
1056       * for that record.
1057       * <BR><BR>
1058       * Note that this applies only to spaces after the end of a value, and not to
1059       * spaces which may appear at the end of a line for a value that is wrapped
1060       * and continued on the next line.
1061       *
1062       * @return  {@code true} if illegal trailing spaces should be stripped off, or
1063       *          {@code false} if LDIF records containing illegal trailing spaces
1064       *          should be rejected.
1065       *
1066       * @deprecated  Use the {@code getTrailingSpaceBehavior} method instead.
1067       */
1068      @Deprecated()
1069      public boolean stripTrailingSpaces()
1070      {
1071        return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
1072      }
1073    
1074    
1075    
1076      /**
1077       * Specifies whether to strip off any illegal trailing spaces that may appear
1078       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
1079       * specification strongly recommends that any value which legitimately
1080       * contains trailing spaces be base64-encoded, and any spaces which appear
1081       * after the end of non-base64-encoded values may therefore be considered
1082       * invalid.  If any such trailing spaces are encountered in an LDIF record and
1083       * they are not to be stripped, then an {@code LDIFException} will be thrown
1084       * for that record.
1085       * <BR><BR>
1086       * Note that this applies only to spaces after the end of a value, and not to
1087       * spaces which may appear at the end of a line for a value that is wrapped
1088       * and continued on the next line.
1089       *
1090       * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
1091       *                              trailing spaces, or {@code false} if LDIF
1092       *                              records containing them should be rejected.
1093       *
1094       * @deprecated  Use the {@code setTrailingSpaceBehavior} method instead.
1095       */
1096      @Deprecated()
1097      public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
1098      {
1099        trailingSpaceBehavior = stripTrailingSpaces
1100             ? TrailingSpaceBehavior.STRIP
1101             : TrailingSpaceBehavior.REJECT;
1102      }
1103    
1104    
1105    
1106      /**
1107       * Retrieves the behavior that should be exhibited when encountering attribute
1108       * values which are not base64-encoded but contain trailing spaces.  The LDIF
1109       * specification strongly recommends that any value which legitimately
1110       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1111       * may be configured to automatically strip these spaces, to preserve them, or
1112       * to reject any entry or change record containing them.
1113       *
1114       * @return  The behavior that should be exhibited when encountering attribute
1115       *          values which are not base64-encoded but contain trailing spaces.
1116       */
1117      public TrailingSpaceBehavior getTrailingSpaceBehavior()
1118      {
1119        return trailingSpaceBehavior;
1120      }
1121    
1122    
1123    
1124      /**
1125       * Specifies the behavior that should be exhibited when encountering attribute
1126       * values which are not base64-encoded but contain trailing spaces.  The LDIF
1127       * specification strongly recommends that any value which legitimately
1128       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1129       * may be configured to automatically strip these spaces, to preserve them, or
1130       * to reject any entry or change record containing them.
1131       *
1132       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1133       *                                encountering attribute values which are not
1134       *                                base64-encoded but contain trailing spaces.
1135       */
1136      public void setTrailingSpaceBehavior(
1137                       final TrailingSpaceBehavior trailingSpaceBehavior)
1138      {
1139        this.trailingSpaceBehavior = trailingSpaceBehavior;
1140      }
1141    
1142    
1143    
1144      /**
1145       * Retrieves the base path that will be prepended to relative paths in order
1146       * to obtain an absolute path.  This will only be used for "file:" URLs that
1147       * have paths which do not begin with a slash.
1148       *
1149       * @return  The base path that will be prepended to relative paths in order to
1150       *          obtain an absolute path.
1151       */
1152      public String getRelativeBasePath()
1153      {
1154        return relativeBasePath;
1155      }
1156    
1157    
1158    
1159      /**
1160       * Specifies the base path that will be prepended to relative paths in order
1161       * to obtain an absolute path.  This will only be used for "file:" URLs that
1162       * have paths which do not begin with a space.
1163       *
1164       * @param  relativeBasePath  The base path that will be prepended to relative
1165       *                           paths in order to obtain an absolute path.
1166       */
1167      public void setRelativeBasePath(final String relativeBasePath)
1168      {
1169        setRelativeBasePath(new File(relativeBasePath));
1170      }
1171    
1172    
1173    
1174      /**
1175       * Specifies the base path that will be prepended to relative paths in order
1176       * to obtain an absolute path.  This will only be used for "file:" URLs that
1177       * have paths which do not begin with a space.
1178       *
1179       * @param  relativeBasePath  The base path that will be prepended to relative
1180       *                           paths in order to obtain an absolute path.
1181       */
1182      public void setRelativeBasePath(final File relativeBasePath)
1183      {
1184        final String path = relativeBasePath.getAbsolutePath();
1185        if (path.endsWith(File.separator))
1186        {
1187          this.relativeBasePath = path;
1188        }
1189        else
1190        {
1191          this.relativeBasePath = path + File.separator;
1192        }
1193      }
1194    
1195    
1196    
1197      /**
1198       * Retrieves the schema that will be used when reading LDIF records, if
1199       * defined.
1200       *
1201       * @return  The schema that will be used when reading LDIF records, or
1202       *          {@code null} if no schema should be used and all attributes should
1203       *          be treated as case-insensitive strings.
1204       */
1205      public Schema getSchema()
1206      {
1207        return schema;
1208      }
1209    
1210    
1211    
1212      /**
1213       * Specifies the schema that should be used when reading LDIF records.
1214       *
1215       * @param  schema  The schema that should be used when reading LDIF records,
1216       *                 or {@code null} if no schema should be used and all
1217       *                 attributes should be treated as case-insensitive strings.
1218       */
1219      public void setSchema(final Schema schema)
1220      {
1221        this.schema = schema;
1222      }
1223    
1224    
1225    
1226      /**
1227       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1228       * change record.
1229       *
1230       * @return  The record read from the LDIF source, or {@code null} if there are
1231       *          no more entries to be read.
1232       *
1233       * @throws  IOException  If a problem occurs while trying to read from the
1234       *                       LDIF source.
1235       *
1236       * @throws  LDIFException  If the data read could not be parsed as an entry or
1237       *                         an LDIF change record.
1238       */
1239      public LDIFRecord readLDIFRecord()
1240             throws IOException, LDIFException
1241      {
1242        if (isAsync())
1243        {
1244          return readLDIFRecordAsync();
1245        }
1246        else
1247        {
1248          return readLDIFRecordInternal();
1249        }
1250      }
1251    
1252    
1253    
1254      /**
1255       * Reads an entry from the LDIF source.
1256       *
1257       * @return  The entry read from the LDIF source, or {@code null} if there are
1258       *          no more entries to be read.
1259       *
1260       * @throws  IOException  If a problem occurs while attempting to read from the
1261       *                       LDIF source.
1262       *
1263       * @throws  LDIFException  If the data read could not be parsed as an entry.
1264       */
1265      public Entry readEntry()
1266             throws IOException, LDIFException
1267      {
1268        if (isAsync())
1269        {
1270          return readEntryAsync();
1271        }
1272        else
1273        {
1274          return readEntryInternal();
1275        }
1276      }
1277    
1278    
1279    
1280      /**
1281       * Reads an LDIF change record from the LDIF source.  The LDIF record must
1282       * have a changetype.
1283       *
1284       * @return  The change record read from the LDIF source, or {@code null} if
1285       *          there are no more records to be read.
1286       *
1287       * @throws  IOException  If a problem occurs while attempting to read from the
1288       *                       LDIF source.
1289       *
1290       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1291       *                         change record.
1292       */
1293      public LDIFChangeRecord readChangeRecord()
1294             throws IOException, LDIFException
1295      {
1296        return readChangeRecord(false);
1297      }
1298    
1299    
1300    
1301      /**
1302       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1303       * record does not have a changetype, then it may be assumed to be an add
1304       * change record.
1305       *
1306       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1307       *                     changetype should be retrieved as an add change record.
1308       *                     If this is {@code false} and the record read does not
1309       *                     include a changetype, then an {@code LDIFException}
1310       *                     will be thrown.
1311       *
1312       * @return  The change record read from the LDIF source, or {@code null} if
1313       *          there are no more records to be read.
1314       *
1315       * @throws  IOException  If a problem occurs while attempting to read from the
1316       *                       LDIF source.
1317       *
1318       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1319       *                         change record.
1320       */
1321      public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1322             throws IOException, LDIFException
1323      {
1324        if (isAsync())
1325        {
1326          return readChangeRecordAsync(defaultAdd);
1327        }
1328        else
1329        {
1330          return readChangeRecordInternal(defaultAdd);
1331        }
1332      }
1333    
1334    
1335    
1336      /**
1337       * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1338       * thread.
1339       *
1340       * @return  The next parsed record or {@code null} if there are no more
1341       *          records to read.
1342       *
1343       * @throws IOException  If IOException was thrown when reading or parsing
1344       *                      the record.
1345       *
1346       * @throws LDIFException If LDIFException was thrown parsing the record.
1347       */
1348      private LDIFRecord readLDIFRecordAsync()
1349              throws IOException, LDIFException
1350      {
1351        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1352        LDIFRecord record = null;
1353        while (record == null)
1354        {
1355          result = readLDIFRecordResultAsync();
1356          if (result == null)
1357          {
1358            return null;
1359          }
1360    
1361          record = result.getOutput();
1362    
1363          // This is a special value that means we should skip this Entry.  We have
1364          // to use something different than null because null means EOF.
1365          if (record == SKIP_ENTRY)
1366          {
1367            record = null;
1368          }
1369        }
1370        return record;
1371      }
1372    
1373    
1374    
1375      /**
1376       * Reads an entry asynchronously from the LDIF source.
1377       *
1378       * @return The entry read from the LDIF source, or {@code null} if there are
1379       *         no more entries to be read.
1380       *
1381       * @throws IOException   If a problem occurs while attempting to read from the
1382       *                       LDIF source.
1383       * @throws LDIFException If the data read could not be parsed as an entry.
1384       */
1385      private Entry readEntryAsync()
1386              throws IOException, LDIFException
1387      {
1388        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1389        LDIFRecord record = null;
1390        while (record == null)
1391        {
1392          result = readLDIFRecordResultAsync();
1393          if (result == null)
1394          {
1395            return null;
1396          }
1397    
1398          record = result.getOutput();
1399    
1400          // This is a special value that means we should skip this Entry.  We have
1401          // to use something different than null because null means EOF.
1402          if (record == SKIP_ENTRY)
1403          {
1404            record = null;
1405          }
1406        }
1407    
1408        if (record instanceof Entry)
1409        {
1410          return (Entry) record;
1411        }
1412        else if (record instanceof LDIFChangeRecord)
1413        {
1414          try
1415          {
1416            // Some LDIFChangeRecord can be converted to an Entry.  This is really
1417            // an edge case though.
1418            return ((LDIFChangeRecord)record).toEntry();
1419          }
1420          catch (LDIFException e)
1421          {
1422            debugException(e);
1423            final long firstLineNumber = result.getInput().getFirstLineNumber();
1424            throw new LDIFException(e.getExceptionMessage(),
1425                                    firstLineNumber, true, e);
1426          }
1427        }
1428    
1429        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1430                                 "LDIFChangeRecord");
1431      }
1432    
1433    
1434    
1435      /**
1436       * Reads an LDIF change record from the LDIF source asynchronously.
1437       * Optionally, if the LDIF record does not have a changetype, then it may be
1438       * assumed to be an add change record.
1439       *
1440       * @param defaultAdd Indicates whether an LDIF record not containing a
1441       *                   changetype should be retrieved as an add change record.
1442       *                   If this is {@code false} and the record read does not
1443       *                   include a changetype, then an {@code LDIFException} will
1444       *                   be thrown.
1445       *
1446       * @return The change record read from the LDIF source, or {@code null} if
1447       *         there are no more records to be read.
1448       *
1449       * @throws IOException   If a problem occurs while attempting to read from the
1450       *                       LDIF source.
1451       * @throws LDIFException If the data read could not be parsed as an LDIF
1452       *                       change record.
1453       */
1454      private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1455              throws IOException, LDIFException
1456      {
1457        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1458        LDIFRecord record = null;
1459        while (record == null)
1460        {
1461          result = readLDIFRecordResultAsync();
1462          if (result == null)
1463          {
1464            return null;
1465          }
1466    
1467          record = result.getOutput();
1468    
1469          // This is a special value that means we should skip this Entry.  We have
1470          // to use something different than null because null means EOF.
1471          if (record == SKIP_ENTRY)
1472          {
1473            record = null;
1474          }
1475        }
1476    
1477        if (record instanceof LDIFChangeRecord)
1478        {
1479          return (LDIFChangeRecord) record;
1480        }
1481        else if (record instanceof Entry)
1482        {
1483          if (defaultAdd)
1484          {
1485            return new LDIFAddChangeRecord((Entry) record);
1486          }
1487          else
1488          {
1489            final long firstLineNumber = result.getInput().getFirstLineNumber();
1490            throw new LDIFException(
1491                 ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1492                 true);
1493          }
1494        }
1495    
1496        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1497                                 "LDIFChangeRecord");
1498      }
1499    
1500    
1501    
1502      /**
1503       * Reads the next LDIF record, which was read and parsed asynchronously by
1504       * separate threads.
1505       *
1506       * @return  The next LDIF record or {@code null} if there are no more records.
1507       *
1508       * @throws  IOException  If a problem occurs while attempting to read from the
1509       *                       LDIF source.
1510       *
1511       * @throws  LDIFException  If the data read could not be parsed as an entry.
1512       */
1513      private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1514              throws IOException, LDIFException
1515      {
1516        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1517    
1518        // If the asynchronous reading and parsing is complete, then we don't have
1519        // to block waiting for the next record to show up on the queue.  If there
1520        // isn't a record there, then return null (EOF) right away.
1521        if (asyncParsingComplete.get())
1522        {
1523          result = asyncParsedRecords.poll();
1524        }
1525        else
1526        {
1527          try
1528          {
1529            // We probably could just do a asyncParsedRecords.take() here, but
1530            // there are some edge case error scenarios where
1531            // asyncParsingComplete might be set without a special EOF sentinel
1532            // Result enqueued.  So to guard against this, we have a very cautious
1533            // polling interval of 1 second.  During normal processing, we never
1534            // have to wait for this to expire, when there is something to do
1535            // (like shutdown).
1536            while ((result == null) && (!asyncParsingComplete.get()))
1537            {
1538              result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1539            }
1540    
1541            // There's a very small chance that we missed the value, so double-check
1542            if (result == null)
1543            {
1544              result = asyncParsedRecords.poll();
1545            }
1546          }
1547          catch (InterruptedException e)
1548          {
1549            debugException(e);
1550            throw createIOExceptionWithCause(null, e);
1551          }
1552        }
1553        if (result == null)
1554        {
1555          return null;
1556        }
1557    
1558        rethrow(result.getFailureCause());
1559    
1560        // Check if we reached the end of the input
1561        final UnparsedLDIFRecord unparsedRecord = result.getInput();
1562        if (unparsedRecord.isEOF())
1563        {
1564          // This might have been set already by the LineReaderThread, but
1565          // just in case it hasn't gotten to it yet, do so here.
1566          asyncParsingComplete.set(true);
1567    
1568          // Enqueue this EOF result again for any other thread that might be
1569          // blocked in asyncParsedRecords.take() even though having multiple
1570          // threads call this method concurrently breaks the contract of this
1571          // class.
1572          try
1573          {
1574            asyncParsedRecords.put(result);
1575          }
1576          catch (InterruptedException e)
1577          {
1578            // We shouldn't ever get interrupted because the put won't ever block.
1579            // Once we are done reading, this is the only item left in the queue,
1580            // so we should always be able to re-enqueue it.
1581            debugException(e);
1582          }
1583          return null;
1584        }
1585    
1586        return result;
1587      }
1588    
1589    
1590    
1591      /**
1592       * Indicates whether this LDIF reader was constructed to perform asynchronous
1593       * processing.
1594       *
1595       * @return  {@code true} if this LDIFReader was constructed to perform
1596       *          asynchronous processing, or {@code false} if not.
1597       */
1598      private boolean isAsync()
1599      {
1600        return isAsync;
1601      }
1602    
1603    
1604    
1605      /**
1606       * If not {@code null}, rethrows the specified Throwable as either an
1607       * IOException or LDIFException.
1608       *
1609       * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1610       *           is thrown.
1611       *
1612       * @throws IOException   If t is an IOException or a checked Exception that
1613       *                       is not an LDIFException.
1614       * @throws LDIFException  If t is an LDIFException.
1615       */
1616      static void rethrow(final Throwable t)
1617             throws IOException, LDIFException
1618      {
1619        if (t == null)
1620        {
1621          return;
1622        }
1623    
1624        if (t instanceof IOException)
1625        {
1626          throw (IOException) t;
1627        }
1628        else if (t instanceof LDIFException)
1629        {
1630          throw (LDIFException) t;
1631        }
1632        else if (t instanceof RuntimeException)
1633        {
1634          throw (RuntimeException) t;
1635        }
1636        else if (t instanceof Error)
1637        {
1638          throw (Error) t;
1639        }
1640        else
1641        {
1642          throw createIOExceptionWithCause(null, t);
1643        }
1644      }
1645    
1646    
1647    
1648      /**
1649       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1650       * change record.
1651       *
1652       * @return The record read from the LDIF source, or {@code null} if there are
1653       *         no more entries to be read.
1654       *
1655       * @throws IOException   If a problem occurs while trying to read from the
1656       *                       LDIF source.
1657       * @throws LDIFException If the data read could not be parsed as an entry or
1658       *                       an LDIF change record.
1659       */
1660      private LDIFRecord readLDIFRecordInternal()
1661           throws IOException, LDIFException
1662      {
1663        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1664        return decodeRecord(unparsedRecord, relativeBasePath, schema);
1665      }
1666    
1667    
1668    
1669      /**
1670       * Reads an entry from the LDIF source.
1671       *
1672       * @return The entry read from the LDIF source, or {@code null} if there are
1673       *         no more entries to be read.
1674       *
1675       * @throws IOException   If a problem occurs while attempting to read from the
1676       *                       LDIF source.
1677       * @throws LDIFException If the data read could not be parsed as an entry.
1678       */
1679      private Entry readEntryInternal()
1680           throws IOException, LDIFException
1681      {
1682        Entry e = null;
1683        while (e == null)
1684        {
1685          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1686          if (unparsedRecord.isEOF())
1687          {
1688            return null;
1689          }
1690    
1691          e = decodeEntry(unparsedRecord, relativeBasePath);
1692          debugLDIFRead(e);
1693    
1694          if (entryTranslator != null)
1695          {
1696            e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1697          }
1698        }
1699        return e;
1700      }
1701    
1702    
1703    
1704      /**
1705       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1706       * record does not have a changetype, then it may be assumed to be an add
1707       * change record.
1708       *
1709       * @param defaultAdd Indicates whether an LDIF record not containing a
1710       *                   changetype should be retrieved as an add change record.
1711       *                   If this is {@code false} and the record read does not
1712       *                   include a changetype, then an {@code LDIFException} will
1713       *                   be thrown.
1714       *
1715       * @return The change record read from the LDIF source, or {@code null} if
1716       *         there are no more records to be read.
1717       *
1718       * @throws IOException   If a problem occurs while attempting to read from the
1719       *                       LDIF source.
1720       * @throws LDIFException If the data read could not be parsed as an LDIF
1721       *                       change record.
1722       */
1723      private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1724           throws IOException, LDIFException
1725      {
1726        LDIFChangeRecord r = null;
1727        while (r == null)
1728        {
1729          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1730          if (unparsedRecord.isEOF())
1731          {
1732            return null;
1733          }
1734    
1735          r = decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd,
1736               schema);
1737          debugLDIFRead(r);
1738    
1739          if (changeRecordTranslator != null)
1740          {
1741            r = changeRecordTranslator.translate(r,
1742                 unparsedRecord.getFirstLineNumber());
1743          }
1744        }
1745        return r;
1746      }
1747    
1748    
1749    
1750      /**
1751       * Reads a record (either an entry or a change record) from the LDIF source
1752       * and places it in the line list.
1753       *
1754       * @return  The line number for the first line of the entry that was read.
1755       *
1756       * @throws  IOException  If a problem occurs while attempting to read from the
1757       *                       LDIF source.
1758       *
1759       * @throws  LDIFException  If the data read could not be parsed as a valid
1760       *                         LDIF record.
1761       */
1762      private UnparsedLDIFRecord readUnparsedRecord()
1763             throws IOException, LDIFException
1764      {
1765        final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20);
1766        boolean lastWasComment = false;
1767        long firstLineNumber = lineNumberCounter + 1;
1768        while (true)
1769        {
1770          final String line = reader.readLine();
1771          lineNumberCounter++;
1772    
1773          if (line == null)
1774          {
1775            // We've hit the end of the LDIF source.  If we haven't read any entry
1776            // data, then return null.  Otherwise, the last entry wasn't followed by
1777            // a blank line, which is OK, and we should decode that entry.
1778            if (lineList.isEmpty())
1779            {
1780              return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1781                   duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1782            }
1783            else
1784            {
1785              break;
1786            }
1787          }
1788    
1789          if (line.length() == 0)
1790          {
1791            // It's a blank line.  If we have read entry data, then this signals the
1792            // end of the entry.  Otherwise, it's an extra space between entries,
1793            // which is OK.
1794            lastWasComment = false;
1795            if (lineList.isEmpty())
1796            {
1797              firstLineNumber++;
1798              continue;
1799            }
1800            else
1801            {
1802              break;
1803            }
1804          }
1805    
1806          if (line.charAt(0) == ' ')
1807          {
1808            // The line starts with a space, which means that it must be a
1809            // continuation of the previous line.  This is true even if the last
1810            // line was a comment.
1811            if (lastWasComment)
1812            {
1813              // What we've read is part of a comment, so we don't care about its
1814              // content.
1815            }
1816            else if (lineList.isEmpty())
1817            {
1818              throw new LDIFException(
1819                             ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1820                             lineNumberCounter, false);
1821            }
1822            else
1823            {
1824              lineList.get(lineList.size() - 1).append(line.substring(1));
1825              lastWasComment = false;
1826            }
1827          }
1828          else if (line.charAt(0) == '#')
1829          {
1830            lastWasComment = true;
1831          }
1832          else
1833          {
1834            // We want to make sure that we skip over the "version:" line if it
1835            // exists, but that should only occur at the beginning of an entry where
1836            // it can't be confused with a possible "version" attribute.
1837            if (lineList.isEmpty() && line.startsWith("version:"))
1838            {
1839              lastWasComment = true;
1840            }
1841            else
1842            {
1843              lineList.add(new StringBuilder(line));
1844              lastWasComment = false;
1845            }
1846          }
1847        }
1848    
1849        return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1850             trailingSpaceBehavior, schema, firstLineNumber);
1851      }
1852    
1853    
1854    
1855      /**
1856       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1857       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1858       * LDIF specification, and it is acceptable to have one or more blank lines
1859       * following the entry. A default trailing space behavior of
1860       * {@code TrailingSpaceBehavior#REJECT} will be used.
1861       *
1862       * @param  ldifLines  The set of lines that comprise the LDIF representation
1863       *                    of the entry.  It must not be {@code null} or empty.
1864       *
1865       * @return  The entry read from LDIF.
1866       *
1867       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1868       *                         entry.
1869       */
1870      public static Entry decodeEntry(final String... ldifLines)
1871             throws LDIFException
1872      {
1873        final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1874             TrailingSpaceBehavior.REJECT, null, ldifLines),
1875             DEFAULT_RELATIVE_BASE_PATH);
1876        debugLDIFRead(e);
1877        return e;
1878      }
1879    
1880    
1881    
1882      /**
1883       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1884       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1885       * LDIF specification, and it is acceptable to have one or more blank lines
1886       * following the entry. A default trailing space behavior of
1887       * {@code TrailingSpaceBehavior#REJECT} will be used.
1888       *
1889       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1890       *                                attribute values encountered while parsing.
1891       * @param  schema                 The schema to use when parsing the record,
1892       *                                if applicable.
1893       * @param  ldifLines              The set of lines that comprise the LDIF
1894       *                                representation of the entry.  It must not be
1895       *                                {@code null} or empty.
1896       *
1897       * @return  The entry read from LDIF.
1898       *
1899       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1900       *                         entry.
1901       */
1902      public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1903                                      final Schema schema,
1904                                      final String... ldifLines)
1905             throws LDIFException
1906      {
1907        return decodeEntry(ignoreDuplicateValues, TrailingSpaceBehavior.REJECT,
1908             schema, ldifLines);
1909      }
1910    
1911    
1912    
1913      /**
1914       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1915       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1916       * LDIF specification, and it is acceptable to have one or more blank lines
1917       * following the entry.
1918       *
1919       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1920       *                                attribute values encountered while parsing.
1921       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1922       *                                encountering attribute values which are not
1923       *                                base64-encoded but contain trailing spaces.
1924       *                                It must not be {@code null}.
1925       * @param  schema                 The schema to use when parsing the record,
1926       *                                if applicable.
1927       * @param  ldifLines              The set of lines that comprise the LDIF
1928       *                                representation of the entry.  It must not be
1929       *                                {@code null} or empty.
1930       *
1931       * @return  The entry read from LDIF.
1932       *
1933       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1934       *                         entry.
1935       */
1936      public static Entry decodeEntry(
1937             final boolean ignoreDuplicateValues,
1938             final TrailingSpaceBehavior trailingSpaceBehavior,
1939             final Schema schema,
1940             final String... ldifLines) throws LDIFException
1941      {
1942        final Entry e = decodeEntry(prepareRecord(
1943                  (ignoreDuplicateValues
1944                       ? DuplicateValueBehavior.STRIP
1945                       : DuplicateValueBehavior.REJECT),
1946             trailingSpaceBehavior, schema, ldifLines),
1947             DEFAULT_RELATIVE_BASE_PATH);
1948        debugLDIFRead(e);
1949        return e;
1950      }
1951    
1952    
1953    
1954      /**
1955       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1956       * provided set of lines must contain exactly one change record and it must
1957       * include a changetype.  Long lines may be wrapped as per the LDIF
1958       * specification, and it is acceptable to have one or more blank lines
1959       * following the entry.
1960       *
1961       * @param  ldifLines  The set of lines that comprise the LDIF representation
1962       *                    of the change record.  It must not be {@code null} or
1963       *                    empty.
1964       *
1965       * @return  The change record read from LDIF.
1966       *
1967       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1968       *                         change record.
1969       */
1970      public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1971             throws LDIFException
1972      {
1973        return decodeChangeRecord(false, ldifLines);
1974      }
1975    
1976    
1977    
1978      /**
1979       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1980       * provided set of lines must contain exactly one change record.  Long lines
1981       * may be wrapped as per the LDIF specification, and it is acceptable to have
1982       * one or more blank lines following the entry.
1983       *
1984       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1985       *                     changetype should be retrieved as an add change record.
1986       *                     If this is {@code false} and the record read does not
1987       *                     include a changetype, then an {@code LDIFException}
1988       *                     will be thrown.
1989       * @param  ldifLines  The set of lines that comprise the LDIF representation
1990       *                    of the change record.  It must not be {@code null} or
1991       *                    empty.
1992       *
1993       * @return  The change record read from LDIF.
1994       *
1995       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1996       *                         change record.
1997       */
1998      public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1999                                                        final String... ldifLines)
2000             throws LDIFException
2001      {
2002        final LDIFChangeRecord r =
2003             decodeChangeRecord(
2004                  prepareRecord(DuplicateValueBehavior.STRIP,
2005                       TrailingSpaceBehavior.REJECT, null, ldifLines),
2006                  DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
2007        debugLDIFRead(r);
2008        return r;
2009      }
2010    
2011    
2012    
2013      /**
2014       * Decodes the provided set of LDIF lines as an LDIF change record.  The
2015       * provided set of lines must contain exactly one change record.  Long lines
2016       * may be wrapped as per the LDIF specification, and it is acceptable to have
2017       * one or more blank lines following the entry.
2018       *
2019       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2020       *                                attribute values encountered while parsing.
2021       * @param  schema                 The schema to use when processing the change
2022       *                                record, or {@code null} if no schema should
2023       *                                be used and all values should be treated as
2024       *                                case-insensitive strings.
2025       * @param  defaultAdd             Indicates whether an LDIF record not
2026       *                                containing a changetype should be retrieved
2027       *                                as an add change record.  If this is
2028       *                                {@code false} and the record read does not
2029       *                                include a changetype, then an
2030       *                                {@code LDIFException} will be thrown.
2031       * @param  ldifLines              The set of lines that comprise the LDIF
2032       *                                representation of the change record.  It
2033       *                                must not be {@code null} or empty.
2034       *
2035       * @return  The change record read from LDIF.
2036       *
2037       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2038       *                         change record.
2039       */
2040      public static LDIFChangeRecord decodeChangeRecord(
2041                                          final boolean ignoreDuplicateValues,
2042                                          final Schema schema,
2043                                          final boolean defaultAdd,
2044                                          final String... ldifLines)
2045             throws LDIFException
2046      {
2047        return decodeChangeRecord(ignoreDuplicateValues,
2048             TrailingSpaceBehavior.REJECT, schema, defaultAdd, ldifLines);
2049      }
2050    
2051    
2052    
2053      /**
2054       * Decodes the provided set of LDIF lines as an LDIF change record.  The
2055       * provided set of lines must contain exactly one change record.  Long lines
2056       * may be wrapped as per the LDIF specification, and it is acceptable to have
2057       * one or more blank lines following the entry.
2058       *
2059       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
2060       *                                attribute values encountered while parsing.
2061       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
2062       *                                encountering attribute values which are not
2063       *                                base64-encoded but contain trailing spaces.
2064       *                                It must not be {@code null}.
2065       * @param  schema                 The schema to use when processing the change
2066       *                                record, or {@code null} if no schema should
2067       *                                be used and all values should be treated as
2068       *                                case-insensitive strings.
2069       * @param  defaultAdd             Indicates whether an LDIF record not
2070       *                                containing a changetype should be retrieved
2071       *                                as an add change record.  If this is
2072       *                                {@code false} and the record read does not
2073       *                                include a changetype, then an
2074       *                                {@code LDIFException} will be thrown.
2075       * @param  ldifLines              The set of lines that comprise the LDIF
2076       *                                representation of the change record.  It
2077       *                                must not be {@code null} or empty.
2078       *
2079       * @return  The change record read from LDIF.
2080       *
2081       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2082       *                         change record.
2083       */
2084      public static LDIFChangeRecord decodeChangeRecord(
2085                         final boolean ignoreDuplicateValues,
2086                         final TrailingSpaceBehavior trailingSpaceBehavior,
2087                         final Schema schema,
2088                         final boolean defaultAdd,
2089                         final String... ldifLines)
2090             throws LDIFException
2091      {
2092        final LDIFChangeRecord r = decodeChangeRecord(
2093             prepareRecord(
2094                  (ignoreDuplicateValues
2095                       ? DuplicateValueBehavior.STRIP
2096                       : DuplicateValueBehavior.REJECT),
2097                  trailingSpaceBehavior, schema, ldifLines),
2098             DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
2099        debugLDIFRead(r);
2100        return r;
2101      }
2102    
2103    
2104    
2105      /**
2106       * Parses the provided set of lines into a list of {@code StringBuilder}
2107       * objects suitable for decoding into an entry or LDIF change record.
2108       * Comments will be ignored and wrapped lines will be unwrapped.
2109       *
2110       * @param  duplicateValueBehavior  The behavior that should be exhibited if
2111       *                                 the LDIF reader encounters an entry with
2112       *                                 duplicate values.
2113       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2114       *                                 encountering attribute values which are not
2115       *                                 base64-encoded but contain trailing spaces.
2116       * @param  schema                  The schema to use when parsing the record,
2117       *                                 if applicable.
2118       * @param  ldifLines               The set of lines that comprise the record
2119       *                                 to decode.  It must not be {@code null} or
2120       *                                 empty.
2121       *
2122       * @return  The prepared list of {@code StringBuilder} objects ready to be
2123       *          decoded.
2124       *
2125       * @throws  LDIFException  If the provided lines do not contain valid LDIF
2126       *                         content.
2127       */
2128      private static UnparsedLDIFRecord prepareRecord(
2129                          final DuplicateValueBehavior duplicateValueBehavior,
2130                          final TrailingSpaceBehavior trailingSpaceBehavior,
2131                          final Schema schema, final String... ldifLines)
2132              throws LDIFException
2133      {
2134        ensureNotNull(ldifLines);
2135        ensureFalse(ldifLines.length == 0,
2136                    "LDIFReader.prepareRecord.ldifLines must not be empty.");
2137    
2138        boolean lastWasComment = false;
2139        final ArrayList<StringBuilder> lineList =
2140             new ArrayList<StringBuilder>(ldifLines.length);
2141        for (int i=0; i < ldifLines.length; i++)
2142        {
2143          final String line = ldifLines[i];
2144          if (line.length() == 0)
2145          {
2146            // This is only acceptable if there are no more non-empty lines in the
2147            // array.
2148            for (int j=i+1; j < ldifLines.length; j++)
2149            {
2150              if (ldifLines[j].length() > 0)
2151              {
2152                throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
2153                                        ldifLines, null);
2154              }
2155    
2156              // If we've gotten here, then we know that we're at the end of the
2157              // entry.  If we have read data, then we can decode it as an entry.
2158              // Otherwise, there was no real data in the provided LDIF lines.
2159              if (lineList.isEmpty())
2160              {
2161                throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
2162                                        ldifLines, null);
2163              }
2164              else
2165              {
2166                return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2167                     trailingSpaceBehavior, schema, 0);
2168              }
2169            }
2170          }
2171    
2172          if (line.charAt(0) == ' ')
2173          {
2174            if (i > 0)
2175            {
2176              if (! lastWasComment)
2177              {
2178                lineList.get(lineList.size() - 1).append(line.substring(1));
2179              }
2180            }
2181            else
2182            {
2183              throw new LDIFException(
2184                             ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
2185                             true, ldifLines, null);
2186            }
2187          }
2188          else if (line.charAt(0) == '#')
2189          {
2190            lastWasComment = true;
2191          }
2192          else
2193          {
2194            lineList.add(new StringBuilder(line));
2195            lastWasComment = false;
2196          }
2197        }
2198    
2199        if (lineList.isEmpty())
2200        {
2201          throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
2202        }
2203        else
2204        {
2205          return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2206               trailingSpaceBehavior, schema, 0);
2207        }
2208      }
2209    
2210    
2211    
2212      /**
2213       * Decodes the unparsed record that was read from the LDIF source.  It may be
2214       * either an entry or an LDIF change record.
2215       *
2216       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2217       *                           input.  It must not be {@code null} or empty.
2218       * @param  relativeBasePath  The base path that will be prepended to relative
2219       *                           paths in order to obtain an absolute path.
2220       * @param  schema            The schema to use when parsing.
2221       *
2222       * @return  The parsed record, or {@code null} if there are no more entries to
2223       *          be read.
2224       *
2225       * @throws  LDIFException  If the data read could not be parsed as an entry or
2226       *                         an LDIF change record.
2227       */
2228      private static LDIFRecord decodeRecord(
2229                                     final UnparsedLDIFRecord unparsedRecord,
2230                                     final String relativeBasePath,
2231                                     final Schema schema)
2232           throws LDIFException
2233      {
2234        // If there was an error reading from the input, then we rethrow it here.
2235        final Exception readError = unparsedRecord.getFailureCause();
2236        if (readError != null)
2237        {
2238          if (readError instanceof LDIFException)
2239          {
2240            // If the error was an LDIFException, which will normally be the case,
2241            // then rethrow it with all of the same state.  We could just
2242            //   throw (LDIFException) readError;
2243            // but that's considered bad form.
2244            final LDIFException ldifEx = (LDIFException) readError;
2245            throw new LDIFException(ldifEx.getMessage(),
2246                                    ldifEx.getLineNumber(),
2247                                    ldifEx.mayContinueReading(),
2248                                    ldifEx.getDataLines(),
2249                                    ldifEx.getCause());
2250          }
2251          else
2252          {
2253            throw new LDIFException(getExceptionMessage(readError),
2254                                    -1, true, readError);
2255          }
2256        }
2257    
2258        if (unparsedRecord.isEOF())
2259        {
2260          return null;
2261        }
2262    
2263        final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
2264        if (unparsedRecord.getLineList() == null)
2265        {
2266          return null;  // We can get here if there was an error reading the lines.
2267        }
2268    
2269        final LDIFRecord r;
2270        if (lineList.size() == 1)
2271        {
2272          r = decodeEntry(unparsedRecord, relativeBasePath);
2273        }
2274        else
2275        {
2276          final String lowerSecondLine = toLowerCase(lineList.get(1).toString());
2277          if (lowerSecondLine.startsWith("control:") ||
2278              lowerSecondLine.startsWith("changetype:"))
2279          {
2280            r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema);
2281          }
2282          else
2283          {
2284            r = decodeEntry(unparsedRecord, relativeBasePath);
2285          }
2286        }
2287    
2288        debugLDIFRead(r);
2289        return r;
2290      }
2291    
2292    
2293    
2294      /**
2295       * Decodes the provided set of LDIF lines as an entry.  The provided list must
2296       * not contain any blank lines or comments, and lines are not allowed to be
2297       * wrapped.
2298       *
2299       * @param  unparsedRecord   The unparsed LDIF record that was read from the
2300       *                          input.  It must not be {@code null} or empty.
2301       * @param  relativeBasePath  The base path that will be prepended to relative
2302       *                           paths in order to obtain an absolute path.
2303       *
2304       * @return  The entry read from LDIF.
2305       *
2306       * @throws  LDIFException  If the provided LDIF data cannot be read as an
2307       *                         entry.
2308       */
2309      private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
2310                                       final String relativeBasePath)
2311              throws LDIFException
2312      {
2313        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2314        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2315    
2316        final Iterator<StringBuilder> iterator = ldifLines.iterator();
2317    
2318        // The first line must start with either "version:" or "dn:".  If the first
2319        // line starts with "version:" then the second must start with "dn:".
2320        StringBuilder line = iterator.next();
2321        handleTrailingSpaces(line, null, firstLineNumber,
2322             unparsedRecord.getTrailingSpaceBehavior());
2323        int colonPos = line.indexOf(":");
2324        if ((colonPos > 0) &&
2325            line.substring(0, colonPos).equalsIgnoreCase("version"))
2326        {
2327          // The first line is "version:".  Under most conditions, this will be
2328          // handled by the LDIF reader, but this can happen if you call
2329          // decodeEntry with a set of data that includes a version.  At any rate,
2330          // read the next line, which must specify the DN.
2331          line = iterator.next();
2332          handleTrailingSpaces(line, null, firstLineNumber,
2333               unparsedRecord.getTrailingSpaceBehavior());
2334        }
2335    
2336        colonPos = line.indexOf(":");
2337        if ((colonPos < 0) ||
2338             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2339        {
2340          throw new LDIFException(
2341               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2342               firstLineNumber, true, ldifLines, null);
2343        }
2344    
2345        final String dn;
2346        final int length = line.length();
2347        if (length == (colonPos+1))
2348        {
2349          // The colon was the last character on the line.  This is acceptable and
2350          // indicates that the entry has the null DN.
2351          dn = "";
2352        }
2353        else if (line.charAt(colonPos+1) == ':')
2354        {
2355          // Skip over any spaces leading up to the value, and then the rest of the
2356          // string is the base64-encoded DN.
2357          int pos = colonPos+2;
2358          while ((pos < length) && (line.charAt(pos) == ' '))
2359          {
2360            pos++;
2361          }
2362    
2363          try
2364          {
2365            final byte[] dnBytes = Base64.decode(line.substring(pos));
2366            dn = new String(dnBytes, "UTF-8");
2367          }
2368          catch (final ParseException pe)
2369          {
2370            debugException(pe);
2371            throw new LDIFException(
2372                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2373                                                                pe.getMessage()),
2374                           firstLineNumber, true, ldifLines, pe);
2375          }
2376          catch (final Exception e)
2377          {
2378            debugException(e);
2379            throw new LDIFException(
2380                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2381                           firstLineNumber, true, ldifLines, e);
2382          }
2383        }
2384        else
2385        {
2386          // Skip over any spaces leading up to the value, and then the rest of the
2387          // string is the DN.
2388          int pos = colonPos+1;
2389          while ((pos < length) && (line.charAt(pos) == ' '))
2390          {
2391            pos++;
2392          }
2393    
2394          dn = line.substring(pos);
2395        }
2396    
2397    
2398        // The remaining lines must be the attributes for the entry.  However, we
2399        // will allow the case in which an entry does not have any attributes, to be
2400        // able to support reading search result entries in which no attributes were
2401        // returned.
2402        if (! iterator.hasNext())
2403        {
2404          return new Entry(dn, unparsedRecord.getSchema());
2405        }
2406    
2407        return new Entry(dn, unparsedRecord.getSchema(),
2408             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2409                  unparsedRecord.getTrailingSpaceBehavior(),
2410                  unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2411                  firstLineNumber));
2412      }
2413    
2414    
2415    
2416      /**
2417       * Decodes the provided set of LDIF lines as a change record.  The provided
2418       * list must not contain any blank lines or comments, and lines are not
2419       * allowed to be wrapped.
2420       *
2421       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2422       *                           input.  It must not be {@code null} or empty.
2423       * @param  relativeBasePath  The base path that will be prepended to relative
2424       *                           paths in order to obtain an absolute path.
2425       * @param  defaultAdd        Indicates whether an LDIF record not containing a
2426       *                           changetype should be retrieved as an add change
2427       *                           record.  If this is {@code false} and the record
2428       *                           read does not include a changetype, then an
2429       *                           {@code LDIFException} will be thrown.
2430       * @param  schema            The schema to use in parsing.
2431       *
2432       * @return  The change record read from LDIF.
2433       *
2434       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2435       *                         change record.
2436       */
2437      private static LDIFChangeRecord decodeChangeRecord(
2438                                           final UnparsedLDIFRecord unparsedRecord,
2439                                           final String relativeBasePath,
2440                                           final boolean defaultAdd,
2441                                           final Schema schema)
2442              throws LDIFException
2443      {
2444        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2445        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2446    
2447        Iterator<StringBuilder> iterator = ldifLines.iterator();
2448    
2449        // The first line must start with either "version:" or "dn:".  If the first
2450        // line starts with "version:" then the second must start with "dn:".
2451        StringBuilder line = iterator.next();
2452        handleTrailingSpaces(line, null, firstLineNumber,
2453             unparsedRecord.getTrailingSpaceBehavior());
2454        int colonPos = line.indexOf(":");
2455        int linesRead = 1;
2456        if ((colonPos > 0) &&
2457            line.substring(0, colonPos).equalsIgnoreCase("version"))
2458        {
2459          // The first line is "version:".  Under most conditions, this will be
2460          // handled by the LDIF reader, but this can happen if you call
2461          // decodeEntry with a set of data that includes a version.  At any rate,
2462          // read the next line, which must specify the DN.
2463          line = iterator.next();
2464          linesRead++;
2465          handleTrailingSpaces(line, null, firstLineNumber,
2466               unparsedRecord.getTrailingSpaceBehavior());
2467        }
2468    
2469        colonPos = line.indexOf(":");
2470        if ((colonPos < 0) ||
2471             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2472        {
2473          throw new LDIFException(
2474               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2475               firstLineNumber, true, ldifLines, null);
2476        }
2477    
2478        final String dn;
2479        int length = line.length();
2480        if (length == (colonPos+1))
2481        {
2482          // The colon was the last character on the line.  This is acceptable and
2483          // indicates that the entry has the null DN.
2484          dn = "";
2485        }
2486        else if (line.charAt(colonPos+1) == ':')
2487        {
2488          // Skip over any spaces leading up to the value, and then the rest of the
2489          // string is the base64-encoded DN.
2490          int pos = colonPos+2;
2491          while ((pos < length) && (line.charAt(pos) == ' '))
2492          {
2493            pos++;
2494          }
2495    
2496          try
2497          {
2498            final byte[] dnBytes = Base64.decode(line.substring(pos));
2499            dn = new String(dnBytes, "UTF-8");
2500          }
2501          catch (final ParseException pe)
2502          {
2503            debugException(pe);
2504            throw new LDIFException(
2505                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2506                                                                   pe.getMessage()),
2507                           firstLineNumber, true, ldifLines, pe);
2508          }
2509          catch (final Exception e)
2510          {
2511            debugException(e);
2512            throw new LDIFException(
2513                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2514                                                                   e),
2515                           firstLineNumber, true, ldifLines, e);
2516          }
2517        }
2518        else
2519        {
2520          // Skip over any spaces leading up to the value, and then the rest of the
2521          // string is the DN.
2522          int pos = colonPos+1;
2523          while ((pos < length) && (line.charAt(pos) == ' '))
2524          {
2525            pos++;
2526          }
2527    
2528          dn = line.substring(pos);
2529        }
2530    
2531    
2532        // An LDIF change record may contain zero or more controls, with the end of
2533        // the controls signified by the changetype.  The changetype element must be
2534        // present, unless defaultAdd is true in which case the first thing that is
2535        // neither control or changetype will trigger the start of add attribute
2536        // parsing.
2537        if (! iterator.hasNext())
2538        {
2539          throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2540                                  firstLineNumber, true, ldifLines, null);
2541        }
2542    
2543        String changeType = null;
2544        ArrayList<Control> controls = null;
2545        while (true)
2546        {
2547          line = iterator.next();
2548          handleTrailingSpaces(line, dn, firstLineNumber,
2549               unparsedRecord.getTrailingSpaceBehavior());
2550          colonPos = line.indexOf(":");
2551          if (colonPos < 0)
2552          {
2553            throw new LDIFException(
2554                 ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2555                 firstLineNumber, true, ldifLines, null);
2556          }
2557    
2558          final String token = toLowerCase(line.substring(0, colonPos));
2559          if (token.equals("control"))
2560          {
2561            if (controls == null)
2562            {
2563              controls = new ArrayList<Control>(5);
2564            }
2565    
2566            controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2567                 relativeBasePath));
2568          }
2569          else if (token.equals("changetype"))
2570          {
2571            changeType =
2572                 decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2573            break;
2574          }
2575          else if (defaultAdd)
2576          {
2577            // The line we read wasn't a control or changetype declaration, so we'll
2578            // assume it's an attribute in an add record.  However, we're not ready
2579            // for that yet, and since we can't rewind an iterator we'll create a
2580            // new one that hasn't yet gotten to this line.
2581            changeType = "add";
2582            iterator = ldifLines.iterator();
2583            for (int i=0; i < linesRead; i++)
2584            {
2585              iterator.next();
2586            }
2587            break;
2588          }
2589          else
2590          {
2591            throw new LDIFException(
2592                 ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2593                      firstLineNumber),
2594                 firstLineNumber, true, ldifLines, null);
2595          }
2596    
2597          linesRead++;
2598        }
2599    
2600    
2601        // Make sure that the change type is acceptable and then decode the rest of
2602        // the change record accordingly.
2603        final String lowerChangeType = toLowerCase(changeType);
2604        if (lowerChangeType.equals("add"))
2605        {
2606          // There must be at least one more line.  If not, then that's an error.
2607          // Otherwise, parse the rest of the data as attribute-value pairs.
2608          if (iterator.hasNext())
2609          {
2610            final Collection<Attribute> attrs =
2611                 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2612                      unparsedRecord.getTrailingSpaceBehavior(),
2613                      unparsedRecord.getSchema(), ldifLines, iterator,
2614                      relativeBasePath, firstLineNumber);
2615            final Attribute[] attributes = new Attribute[attrs.size()];
2616            final Iterator<Attribute> attrIterator = attrs.iterator();
2617            for (int i=0; i < attributes.length; i++)
2618            {
2619              attributes[i] = attrIterator.next();
2620            }
2621    
2622            return new LDIFAddChangeRecord(dn, attributes, controls);
2623          }
2624          else
2625          {
2626            throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2627                                    firstLineNumber, true, ldifLines, null);
2628          }
2629        }
2630        else if (lowerChangeType.equals("delete"))
2631        {
2632          // There shouldn't be any more data.  If there is, then that's an error.
2633          // Otherwise, we can just return the delete change record with what we
2634          // already know.
2635          if (iterator.hasNext())
2636          {
2637            throw new LDIFException(
2638                           ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2639                           firstLineNumber, true, ldifLines, null);
2640          }
2641          else
2642          {
2643            return new LDIFDeleteChangeRecord(dn, controls);
2644          }
2645        }
2646        else if (lowerChangeType.equals("modify"))
2647        {
2648          // There must be at least one more line.  If not, then that's an error.
2649          // Otherwise, parse the rest of the data as a set of modifications.
2650          if (iterator.hasNext())
2651          {
2652            final Modification[] mods = parseModifications(dn,
2653                 unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2654                 firstLineNumber, schema);
2655            return new LDIFModifyChangeRecord(dn, mods, controls);
2656          }
2657          else
2658          {
2659            throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2660                                    firstLineNumber, true, ldifLines, null);
2661          }
2662        }
2663        else if (lowerChangeType.equals("moddn") ||
2664                 lowerChangeType.equals("modrdn"))
2665        {
2666          // There must be at least one more line.  If not, then that's an error.
2667          // Otherwise, parse the rest of the data as a set of modifications.
2668          if (iterator.hasNext())
2669          {
2670            return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2671                 unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2672          }
2673          else
2674          {
2675            throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2676                                    firstLineNumber, true, ldifLines, null);
2677          }
2678        }
2679        else
2680        {
2681          throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2682                                                             firstLineNumber),
2683                                  firstLineNumber, true, ldifLines, null);
2684        }
2685      }
2686    
2687    
2688    
2689      /**
2690       * Decodes information about a control from the provided line.
2691       *
2692       * @param  line              The line to process.
2693       * @param  colonPos          The position of the colon that separates the
2694       *                           control token string from tbe encoded control.
2695       * @param  firstLineNumber   The line number for the start of the record.
2696       * @param  ldifLines         The lines that comprise the LDIF representation
2697       *                           of the full record being parsed.
2698       * @param  relativeBasePath  The base path that will be prepended to relative
2699       *                           paths in order to obtain an absolute path.
2700       *
2701       * @return  The decoded control.
2702       *
2703       * @throws  LDIFException  If a problem is encountered while trying to decode
2704       *                         the changetype.
2705       */
2706      private static Control decodeControl(final StringBuilder line,
2707                                           final int colonPos,
2708                                           final long firstLineNumber,
2709                                           final ArrayList<StringBuilder> ldifLines,
2710                                           final String relativeBasePath)
2711              throws LDIFException
2712      {
2713        final String controlString;
2714        int length = line.length();
2715        if (length == (colonPos+1))
2716        {
2717          // The colon was the last character on the line.  This is not
2718          // acceptable.
2719          throw new LDIFException(
2720               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2721               firstLineNumber, true, ldifLines, null);
2722        }
2723        else if (line.charAt(colonPos+1) == ':')
2724        {
2725          // Skip over any spaces leading up to the value, and then the rest of
2726          // the string is the base64-encoded control representation.  This is
2727          // unusual and unnecessary, but is nevertheless acceptable.
2728          int pos = colonPos+2;
2729          while ((pos < length) && (line.charAt(pos) == ' '))
2730          {
2731            pos++;
2732          }
2733    
2734          try
2735          {
2736            final byte[] controlBytes = Base64.decode(line.substring(pos));
2737            controlString =  new String(controlBytes, "UTF-8");
2738          }
2739          catch (final ParseException pe)
2740          {
2741            debugException(pe);
2742            throw new LDIFException(
2743                           ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2744                                firstLineNumber, pe.getMessage()),
2745                           firstLineNumber, true, ldifLines, pe);
2746          }
2747          catch (final Exception e)
2748          {
2749            debugException(e);
2750            throw new LDIFException(
2751                 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2752                 firstLineNumber, true, ldifLines, e);
2753          }
2754        }
2755        else
2756        {
2757          // Skip over any spaces leading up to the value, and then the rest of
2758          // the string is the encoded control.
2759          int pos = colonPos+1;
2760          while ((pos < length) && (line.charAt(pos) == ' '))
2761          {
2762            pos++;
2763          }
2764    
2765          controlString = line.substring(pos);
2766        }
2767    
2768        // If the resulting control definition is empty, then that's invalid.
2769        if (controlString.length() == 0)
2770        {
2771          throw new LDIFException(
2772               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2773               firstLineNumber, true, ldifLines, null);
2774        }
2775    
2776    
2777        // The first element of the control must be the OID, and it must be followed
2778        // by a space (to separate it from the criticality), a colon (to separate it
2779        // from the value and indicate a default criticality of false), or the end
2780        // of the line (to indicate a default criticality of false and no value).
2781        String oid = null;
2782        boolean hasCriticality = false;
2783        boolean hasValue = false;
2784        int pos = 0;
2785        length = controlString.length();
2786        while (pos < length)
2787        {
2788          final char c = controlString.charAt(pos);
2789          if (c == ':')
2790          {
2791            // This indicates that there is no criticality and that the value
2792            // immediately follows the OID.
2793            oid = controlString.substring(0, pos++);
2794            hasValue = true;
2795            break;
2796          }
2797          else if (c == ' ')
2798          {
2799            // This indicates that there is a criticality.  We don't know anything
2800            // about the presence of a value yet.
2801            oid = controlString.substring(0, pos++);
2802            hasCriticality = true;
2803            break;
2804          }
2805          else
2806          {
2807            pos++;
2808          }
2809        }
2810    
2811        if (oid == null)
2812        {
2813          // This indicates that the string representation of the control is only
2814          // the OID.
2815          return new Control(controlString, false);
2816        }
2817    
2818    
2819        // See if we need to read the criticality.  If so, then do so now.
2820        // Otherwise, assume a default criticality of false.
2821        final boolean isCritical;
2822        if (hasCriticality)
2823        {
2824          // Skip over any spaces before the criticality.
2825          while (controlString.charAt(pos) == ' ')
2826          {
2827            pos++;
2828          }
2829    
2830          // Read until we find a colon or the end of the string.
2831          final int criticalityStartPos = pos;
2832          while (pos < length)
2833          {
2834            final char c = controlString.charAt(pos);
2835            if (c == ':')
2836            {
2837              hasValue = true;
2838              break;
2839            }
2840            else
2841            {
2842              pos++;
2843            }
2844          }
2845    
2846          final String criticalityString =
2847               toLowerCase(controlString.substring(criticalityStartPos, pos));
2848          if (criticalityString.equals("true"))
2849          {
2850            isCritical = true;
2851          }
2852          else if (criticalityString.equals("false"))
2853          {
2854            isCritical = false;
2855          }
2856          else
2857          {
2858            throw new LDIFException(
2859                 ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2860                      firstLineNumber),
2861                 firstLineNumber, true, ldifLines, null);
2862          }
2863    
2864          if (hasValue)
2865          {
2866            pos++;
2867          }
2868        }
2869        else
2870        {
2871          isCritical = false;
2872        }
2873    
2874        // See if we need to read the value.  If so, then do so now.  It may be
2875        // a string, or it may be base64-encoded.  It could conceivably even be read
2876        // from a URL.
2877        final ASN1OctetString value;
2878        if (hasValue)
2879        {
2880          // The character immediately after the colon that precedes the value may
2881          // be one of the following:
2882          // - A second colon (optionally followed by a single space) to indicate
2883          //   that the value is base64-encoded.
2884          // - A less-than symbol to indicate that the value should be read from a
2885          //   location specified by a URL.
2886          // - A single space that precedes the non-base64-encoded value.
2887          // - The first character of the non-base64-encoded value.
2888          switch (controlString.charAt(pos))
2889          {
2890            case ':':
2891              try
2892              {
2893                if (controlString.length() == (pos+1))
2894                {
2895                  value = new ASN1OctetString();
2896                }
2897                else if (controlString.charAt(pos+1) == ' ')
2898                {
2899                  value = new ASN1OctetString(
2900                       Base64.decode(controlString.substring(pos+2)));
2901                }
2902                else
2903                {
2904                  value = new ASN1OctetString(
2905                       Base64.decode(controlString.substring(pos+1)));
2906                }
2907              }
2908              catch (final Exception e)
2909              {
2910                debugException(e);
2911                throw new LDIFException(
2912                     ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2913                          firstLineNumber, getExceptionMessage(e)),
2914                     firstLineNumber, true, ldifLines, e);
2915              }
2916              break;
2917            case '<':
2918              try
2919              {
2920                final String urlString;
2921                if (controlString.charAt(pos+1) == ' ')
2922                {
2923                  urlString = controlString.substring(pos+2);
2924                }
2925                else
2926                {
2927                  urlString = controlString.substring(pos+1);
2928                }
2929                value = new ASN1OctetString(retrieveURLBytes(urlString,
2930                     relativeBasePath, firstLineNumber));
2931              }
2932              catch (final Exception e)
2933              {
2934                debugException(e);
2935                throw new LDIFException(
2936                     ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2937                          firstLineNumber, getExceptionMessage(e)),
2938                     firstLineNumber, true, ldifLines, e);
2939              }
2940              break;
2941            case ' ':
2942              value = new ASN1OctetString(controlString.substring(pos+1));
2943              break;
2944            default:
2945              value = new ASN1OctetString(controlString.substring(pos));
2946              break;
2947          }
2948        }
2949        else
2950        {
2951          value = null;
2952        }
2953    
2954        return new Control(oid, isCritical, value);
2955      }
2956    
2957    
2958    
2959      /**
2960       * Decodes the changetype element from the provided line.
2961       *
2962       * @param  line             The line to process.
2963       * @param  colonPos         The position of the colon that separates the
2964       *                          changetype string from its value.
2965       * @param  firstLineNumber  The line number for the start of the record.
2966       * @param  ldifLines        The lines that comprise the LDIF representation of
2967       *                          the full record being parsed.
2968       *
2969       * @return  The decoded changetype string.
2970       *
2971       * @throws  LDIFException  If a problem is encountered while trying to decode
2972       *                         the changetype.
2973       */
2974      private static String decodeChangeType(final StringBuilder line,
2975                                 final int colonPos, final long firstLineNumber,
2976                                 final ArrayList<StringBuilder> ldifLines)
2977              throws LDIFException
2978      {
2979        final int length = line.length();
2980        if (length == (colonPos+1))
2981        {
2982          // The colon was the last character on the line.  This is not
2983          // acceptable.
2984          throw new LDIFException(
2985               ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2986               true, ldifLines, null);
2987        }
2988        else if (line.charAt(colonPos+1) == ':')
2989        {
2990          // Skip over any spaces leading up to the value, and then the rest of
2991          // the string is the base64-encoded changetype.  This is unusual and
2992          // unnecessary, but is nevertheless acceptable.
2993          int pos = colonPos+2;
2994          while ((pos < length) && (line.charAt(pos) == ' '))
2995          {
2996            pos++;
2997          }
2998    
2999          try
3000          {
3001            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3002            return new String(changeTypeBytes, "UTF-8");
3003          }
3004          catch (final ParseException pe)
3005          {
3006            debugException(pe);
3007            throw new LDIFException(
3008                           ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
3009                                                                pe.getMessage()),
3010                           firstLineNumber, true, ldifLines, pe);
3011          }
3012          catch (final Exception e)
3013          {
3014            debugException(e);
3015            throw new LDIFException(
3016                 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
3017                 firstLineNumber, true, ldifLines, e);
3018          }
3019        }
3020        else
3021        {
3022          // Skip over any spaces leading up to the value, and then the rest of
3023          // the string is the changetype.
3024          int pos = colonPos+1;
3025          while ((pos < length) && (line.charAt(pos) == ' '))
3026          {
3027            pos++;
3028          }
3029    
3030          return line.substring(pos);
3031        }
3032      }
3033    
3034    
3035    
3036      /**
3037       * Parses the data available through the provided iterator as a collection of
3038       * attributes suitable for use in an entry or an add change record.
3039       *
3040       * @param  dn                      The DN of the record being read.
3041       * @param  duplicateValueBehavior  The behavior that should be exhibited if
3042       *                                 the LDIF reader encounters an entry with
3043       *                                 duplicate values.
3044       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
3045       *                                 encountering attribute values which are not
3046       *                                 base64-encoded but contain trailing spaces.
3047       * @param  schema                  The schema to use when parsing the
3048       *                                 attributes, or {@code null} if none is
3049       *                                 needed.
3050       * @param  ldifLines               The lines that comprise the LDIF
3051       *                                 representation of the full record being
3052       *                                 parsed.
3053       * @param  iterator                The iterator to use to access the attribute
3054       *                                 lines.
3055       * @param  relativeBasePath        The base path that will be prepended to
3056       *                                 relative paths in order to obtain an
3057       *                                 absolute path.
3058       * @param  firstLineNumber         The line number for the start of the
3059       *                                 record.
3060       *
3061       * @return  The collection of attributes that were read.
3062       *
3063       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3064       *                         set of attributes.
3065       */
3066      private static ArrayList<Attribute> parseAttributes(final String dn,
3067           final DuplicateValueBehavior duplicateValueBehavior,
3068           final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
3069           final ArrayList<StringBuilder> ldifLines,
3070           final Iterator<StringBuilder> iterator, final String relativeBasePath,
3071           final long firstLineNumber)
3072              throws LDIFException
3073      {
3074        final LinkedHashMap<String,Object> attributes =
3075             new LinkedHashMap<String,Object>(ldifLines.size());
3076        while (iterator.hasNext())
3077        {
3078          final StringBuilder line = iterator.next();
3079          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3080          final int colonPos = line.indexOf(":");
3081          if (colonPos <= 0)
3082          {
3083            throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3084                                    firstLineNumber, true, ldifLines, null);
3085          }
3086    
3087          final String attributeName = line.substring(0, colonPos);
3088          final String lowerName     = toLowerCase(attributeName);
3089    
3090          final MatchingRule matchingRule;
3091          if (schema == null)
3092          {
3093            matchingRule = CaseIgnoreStringMatchingRule.getInstance();
3094          }
3095          else
3096          {
3097            matchingRule =
3098                 MatchingRule.selectEqualityMatchingRule(attributeName, schema);
3099          }
3100    
3101          Attribute attr;
3102          final LDIFAttribute ldifAttr;
3103          final Object attrObject = attributes.get(lowerName);
3104          if (attrObject == null)
3105          {
3106            attr     = null;
3107            ldifAttr = null;
3108          }
3109          else
3110          {
3111            if (attrObject instanceof Attribute)
3112            {
3113              attr     = (Attribute) attrObject;
3114              ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
3115                                           attr.getRawValues()[0]);
3116              attributes.put(lowerName, ldifAttr);
3117            }
3118            else
3119            {
3120              attr     = null;
3121              ldifAttr = (LDIFAttribute) attrObject;
3122            }
3123          }
3124    
3125          final int length = line.length();
3126          if (length == (colonPos+1))
3127          {
3128            // This means that the attribute has a zero-length value, which is
3129            // acceptable.
3130            if (attrObject == null)
3131            {
3132              attr = new Attribute(attributeName, matchingRule, "");
3133              attributes.put(lowerName, attr);
3134            }
3135            else
3136            {
3137              try
3138              {
3139                if (! ldifAttr.addValue(new ASN1OctetString(),
3140                           duplicateValueBehavior))
3141                {
3142                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3143                  {
3144                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3145                         firstLineNumber, attributeName), firstLineNumber, true,
3146                         ldifLines, null);
3147                  }
3148                }
3149              }
3150              catch (LDAPException le)
3151              {
3152                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3153                     firstLineNumber, attributeName, getExceptionMessage(le)),
3154                     firstLineNumber, true, ldifLines, le);
3155              }
3156            }
3157          }
3158          else if (line.charAt(colonPos+1) == ':')
3159          {
3160            // Skip over any spaces leading up to the value, and then the rest of
3161            // the string is the base64-encoded attribute value.
3162            int pos = colonPos+2;
3163            while ((pos < length) && (line.charAt(pos) == ' '))
3164            {
3165              pos++;
3166            }
3167    
3168            try
3169            {
3170              final byte[] valueBytes = Base64.decode(line.substring(pos));
3171              if (attrObject == null)
3172              {
3173                attr = new Attribute(attributeName, matchingRule, valueBytes);
3174                attributes.put(lowerName, attr);
3175              }
3176              else
3177              {
3178                try
3179                {
3180                  if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
3181                             duplicateValueBehavior))
3182                  {
3183                    if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3184                    {
3185                      throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3186                           firstLineNumber, attributeName), firstLineNumber, true,
3187                           ldifLines, null);
3188                    }
3189                  }
3190                }
3191                catch (LDAPException le)
3192                {
3193                  throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3194                       firstLineNumber, attributeName, getExceptionMessage(le)),
3195                       firstLineNumber, true, ldifLines, le);
3196                }
3197              }
3198            }
3199            catch (final ParseException pe)
3200            {
3201              debugException(pe);
3202              throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3203                                           attributeName,  firstLineNumber,
3204                                           pe.getMessage()),
3205                                      firstLineNumber, true, ldifLines, pe);
3206            }
3207          }
3208          else if (line.charAt(colonPos+1) == '<')
3209          {
3210            // Skip over any spaces leading up to the value, and then the rest of
3211            // the string is a URL that indicates where to get the real content.
3212            // At the present time, we'll only support the file URLs.
3213            int pos = colonPos+2;
3214            while ((pos < length) && (line.charAt(pos) == ' '))
3215            {
3216              pos++;
3217            }
3218    
3219            final byte[] urlBytes;
3220            final String urlString = line.substring(pos);
3221            try
3222            {
3223              urlBytes =
3224                   retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
3225            }
3226            catch (final Exception e)
3227            {
3228              debugException(e);
3229              throw new LDIFException(
3230                   ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3231                        firstLineNumber, e),
3232                   firstLineNumber, true, ldifLines, e);
3233            }
3234    
3235            if (attrObject == null)
3236            {
3237              attr = new Attribute(attributeName, matchingRule, urlBytes);
3238              attributes.put(lowerName, attr);
3239            }
3240            else
3241            {
3242              try
3243              {
3244                if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
3245                     duplicateValueBehavior))
3246                {
3247                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3248                  {
3249                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3250                         firstLineNumber, attributeName), firstLineNumber, true,
3251                         ldifLines, null);
3252                  }
3253                }
3254              }
3255              catch (final LDIFException le)
3256              {
3257                debugException(le);
3258                throw le;
3259              }
3260              catch (final Exception e)
3261              {
3262                debugException(e);
3263                throw new LDIFException(
3264                     ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3265                          firstLineNumber, e),
3266                     firstLineNumber, true, ldifLines, e);
3267              }
3268            }
3269          }
3270          else
3271          {
3272            // Skip over any spaces leading up to the value, and then the rest of
3273            // the string is the value.
3274            int pos = colonPos+1;
3275            while ((pos < length) && (line.charAt(pos) == ' '))
3276            {
3277              pos++;
3278            }
3279    
3280            final String valueString = line.substring(pos);
3281            if (attrObject == null)
3282            {
3283              attr = new Attribute(attributeName, matchingRule, valueString);
3284              attributes.put(lowerName, attr);
3285            }
3286            else
3287            {
3288              try
3289              {
3290                if (! ldifAttr.addValue(new ASN1OctetString(valueString),
3291                           duplicateValueBehavior))
3292                {
3293                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3294                  {
3295                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3296                         firstLineNumber, attributeName), firstLineNumber, true,
3297                         ldifLines, null);
3298                  }
3299                }
3300              }
3301              catch (LDAPException le)
3302              {
3303                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3304                     firstLineNumber, attributeName, getExceptionMessage(le)),
3305                     firstLineNumber, true, ldifLines, le);
3306              }
3307            }
3308          }
3309        }
3310    
3311        final ArrayList<Attribute> attrList =
3312             new ArrayList<Attribute>(attributes.size());
3313        for (final Object o : attributes.values())
3314        {
3315          if (o instanceof Attribute)
3316          {
3317            attrList.add((Attribute) o);
3318          }
3319          else
3320          {
3321            attrList.add(((LDIFAttribute) o).toAttribute());
3322          }
3323        }
3324    
3325        return attrList;
3326      }
3327    
3328    
3329    
3330      /**
3331       * Retrieves the bytes that make up the file referenced by the given URL.
3332       *
3333       * @param  urlString         The string representation of the URL to retrieve.
3334       * @param  relativeBasePath  The base path that will be prepended to relative
3335       *                           paths in order to obtain an absolute path.
3336       * @param  firstLineNumber   The line number for the start of the record.
3337       *
3338       * @return  The bytes contained in the specified file, or an empty array if
3339       *          the specified file is empty.
3340       *
3341       * @throws  LDIFException  If the provided URL is malformed or references a
3342       *                         nonexistent file.
3343       *
3344       * @throws  IOException  If a problem is encountered while attempting to read
3345       *                       from the target file.
3346       */
3347      private static byte[] retrieveURLBytes(final String urlString,
3348                                             final String relativeBasePath,
3349                                             final long firstLineNumber)
3350              throws LDIFException, IOException
3351      {
3352        int pos;
3353        String path;
3354        final String lowerURLString = toLowerCase(urlString);
3355        if (lowerURLString.startsWith("file:/"))
3356        {
3357          pos = 6;
3358          while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3359          {
3360            pos++;
3361          }
3362    
3363          path = urlString.substring(pos-1);
3364        }
3365        else if (lowerURLString.startsWith("file:"))
3366        {
3367          // A file: URL that doesn't include a slash will be interpreted as a
3368          // relative path.
3369          path = relativeBasePath + urlString.substring(5);
3370        }
3371        else
3372        {
3373          throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3374               firstLineNumber, true);
3375        }
3376    
3377        final File f = new File(path);
3378        if (! f.exists())
3379        {
3380          throw new LDIFException(
3381               ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3382               firstLineNumber, true);
3383        }
3384    
3385        // In order to conserve memory, we'll only allow values to be read from
3386        // files no larger than 10 megabytes.
3387        final long fileSize = f.length();
3388        if (fileSize > (10 * 1024 * 1024))
3389        {
3390          throw new LDIFException(
3391               ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3392                    (10*1024*1024)),
3393               firstLineNumber, true);
3394        }
3395    
3396        int fileBytesRemaining = (int) fileSize;
3397        final byte[] fileData = new byte[(int) fileSize];
3398        final FileInputStream fis = new FileInputStream(f);
3399        try
3400        {
3401          int fileBytesRead = 0;
3402          while (fileBytesRead < fileSize)
3403          {
3404            final int bytesRead =
3405                 fis.read(fileData, fileBytesRead, fileBytesRemaining);
3406            if (bytesRead < 0)
3407            {
3408              // We hit the end of the file before we expected to.  This shouldn't
3409              // happen unless the file size changed since we first looked at it,
3410              // which we won't allow.
3411              throw new LDIFException(
3412                   ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3413                        f.getAbsolutePath()),
3414                   firstLineNumber, true);
3415            }
3416    
3417            fileBytesRead      += bytesRead;
3418            fileBytesRemaining -= bytesRead;
3419          }
3420    
3421          if (fis.read() != -1)
3422          {
3423            // There is still more data to read.  This shouldn't happen unless the
3424            // file size changed since we first looked at it, which we won't allow.
3425            throw new LDIFException(
3426                 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3427                 firstLineNumber, true);
3428          }
3429        }
3430        finally
3431        {
3432          fis.close();
3433        }
3434    
3435        return fileData;
3436      }
3437    
3438    
3439    
3440      /**
3441       * Parses the data available through the provided iterator into an array of
3442       * modifications suitable for use in a modify change record.
3443       *
3444       * @param  dn                     The DN of the entry being parsed.
3445       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3446       *                                encountering attribute values which are not
3447       *                                base64-encoded but contain trailing spaces.
3448       * @param  ldifLines              The lines that comprise the LDIF
3449       *                                representation of the full record being
3450       *                                parsed.
3451       * @param  iterator               The iterator to use to access the
3452       *                                modification data.
3453       * @param  firstLineNumber        The line number for the start of the record.
3454       * @param  schema                 The schema to use in processing.
3455       *
3456       * @return  An array containing the modifications that were read.
3457       *
3458       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3459       *                         set of modifications.
3460       */
3461      private static Modification[] parseModifications(final String dn,
3462                          final TrailingSpaceBehavior trailingSpaceBehavior,
3463                          final ArrayList<StringBuilder> ldifLines,
3464                          final Iterator<StringBuilder> iterator,
3465                          final long firstLineNumber, final Schema schema)
3466              throws LDIFException
3467      {
3468        final ArrayList<Modification> modList =
3469             new ArrayList<Modification>(ldifLines.size());
3470    
3471        while (iterator.hasNext())
3472        {
3473          // The first line must start with "add:", "delete:", "replace:", or
3474          // "increment:" followed by an attribute name.
3475          StringBuilder line = iterator.next();
3476          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3477          int colonPos = line.indexOf(":");
3478          if (colonPos < 0)
3479          {
3480            throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3481                                    firstLineNumber, true, ldifLines, null);
3482          }
3483    
3484          final ModificationType modType;
3485          final String modTypeStr = toLowerCase(line.substring(0, colonPos));
3486          if (modTypeStr.equals("add"))
3487          {
3488            modType = ModificationType.ADD;
3489          }
3490          else if (modTypeStr.equals("delete"))
3491          {
3492            modType = ModificationType.DELETE;
3493          }
3494          else if (modTypeStr.equals("replace"))
3495          {
3496            modType = ModificationType.REPLACE;
3497          }
3498          else if (modTypeStr.equals("increment"))
3499          {
3500            modType = ModificationType.INCREMENT;
3501          }
3502          else
3503          {
3504            throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3505                                         firstLineNumber),
3506                                    firstLineNumber, true, ldifLines, null);
3507          }
3508    
3509          String attributeName;
3510          int length = line.length();
3511          if (length == (colonPos+1))
3512          {
3513            // The colon was the last character on the line.  This is not
3514            // acceptable.
3515            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3516                                         firstLineNumber),
3517                                    firstLineNumber, true, ldifLines, null);
3518          }
3519          else if (line.charAt(colonPos+1) == ':')
3520          {
3521            // Skip over any spaces leading up to the value, and then the rest of
3522            // the string is the base64-encoded attribute name.
3523            int pos = colonPos+2;
3524            while ((pos < length) && (line.charAt(pos) == ' '))
3525            {
3526              pos++;
3527            }
3528    
3529            try
3530            {
3531              final byte[] dnBytes = Base64.decode(line.substring(pos));
3532              attributeName = new String(dnBytes, "UTF-8");
3533            }
3534            catch (final ParseException pe)
3535            {
3536              debugException(pe);
3537              throw new LDIFException(
3538                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3539                        firstLineNumber, pe.getMessage()),
3540                   firstLineNumber, true, ldifLines, pe);
3541            }
3542            catch (final Exception e)
3543            {
3544              debugException(e);
3545              throw new LDIFException(
3546                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3547                        firstLineNumber, e),
3548                   firstLineNumber, true, ldifLines, e);
3549            }
3550          }
3551          else
3552          {
3553            // Skip over any spaces leading up to the value, and then the rest of
3554            // the string is the attribute name.
3555            int pos = colonPos+1;
3556            while ((pos < length) && (line.charAt(pos) == ' '))
3557            {
3558              pos++;
3559            }
3560    
3561            attributeName = line.substring(pos);
3562          }
3563    
3564          if (attributeName.length() == 0)
3565          {
3566            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3567                                         firstLineNumber),
3568                                    firstLineNumber, true, ldifLines, null);
3569          }
3570    
3571    
3572          // The next zero or more lines may be the set of attribute values.  Keep
3573          // reading until we reach the end of the iterator or until we find a line
3574          // with just a "-".
3575          final ArrayList<ASN1OctetString> valueList =
3576               new ArrayList<ASN1OctetString>(ldifLines.size());
3577          while (iterator.hasNext())
3578          {
3579            line = iterator.next();
3580            handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3581            if (line.toString().equals("-"))
3582            {
3583              break;
3584            }
3585    
3586            colonPos = line.indexOf(":");
3587            if (colonPos < 0)
3588            {
3589              throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3590                                      firstLineNumber, true, ldifLines, null);
3591            }
3592            else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3593            {
3594              // There are a couple of cases in which this might be acceptable:
3595              // - If the two names are logically equivalent, but have an alternate
3596              //   name (or OID) for the target attribute type, or if there are
3597              //   attribute options and the options are just in a different order.
3598              // - If this is the first value for the target attribute and the
3599              //   alternate name includes a "binary" option that the original
3600              //   attribute name did not have.  In this case, all subsequent values
3601              //   will also be required to have the binary option.
3602              final String alternateName = line.substring(0, colonPos);
3603    
3604    
3605              // Check to see if the base names are equivalent.
3606              boolean baseNameEquivalent = false;
3607              final String expectedBaseName = Attribute.getBaseName(attributeName);
3608              final String alternateBaseName = Attribute.getBaseName(alternateName);
3609              if (alternateBaseName.equalsIgnoreCase(expectedBaseName))
3610              {
3611                baseNameEquivalent = true;
3612              }
3613              else
3614              {
3615                if (schema != null)
3616                {
3617                  final AttributeTypeDefinition expectedAT =
3618                       schema.getAttributeType(expectedBaseName);
3619                  final AttributeTypeDefinition alternateAT =
3620                       schema.getAttributeType(alternateBaseName);
3621                  if ((expectedAT != null) && (alternateAT != null) &&
3622                      expectedAT.equals(alternateAT))
3623                  {
3624                    baseNameEquivalent = true;
3625                  }
3626                }
3627              }
3628    
3629    
3630              // Check to see if the attribute options are equivalent.
3631              final Set<String> expectedOptions =
3632                   Attribute.getOptions(attributeName);
3633              final Set<String> lowerExpectedOptions =
3634                   new HashSet<String>(expectedOptions.size());
3635              for (final String s : expectedOptions)
3636              {
3637                lowerExpectedOptions.add(toLowerCase(s));
3638              }
3639    
3640              final Set<String> alternateOptions =
3641                   Attribute.getOptions(alternateName);
3642              final Set<String> lowerAlternateOptions =
3643                   new HashSet<String>(alternateOptions.size());
3644              for (final String s : alternateOptions)
3645              {
3646                lowerAlternateOptions.add(toLowerCase(s));
3647              }
3648    
3649              final boolean optionsEquivalent =
3650                   lowerAlternateOptions.equals(lowerExpectedOptions);
3651    
3652    
3653              if (baseNameEquivalent && optionsEquivalent)
3654              {
3655                // This is fine.  The two attribute descriptions are logically
3656                // equivalent.  We'll continue using the attribute description that
3657                // was provided first.
3658              }
3659              else if (valueList.isEmpty() && baseNameEquivalent &&
3660                       lowerAlternateOptions.remove("binary") &&
3661                       lowerAlternateOptions.equals(lowerExpectedOptions))
3662              {
3663                // This means that the provided value is the first value for the
3664                // attribute, and that the only significant difference is that the
3665                // provided attribute description included an unexpected "binary"
3666                // option.  We'll accept this, but will require any additional
3667                // values for this modification to also include the binary option,
3668                // and we'll use the binary option in the attribute that is
3669                // eventually created.
3670                attributeName = alternateName;
3671              }
3672              else
3673              {
3674                // This means that either the base names are different or the sets
3675                // of options are incompatible.  This is not acceptable.
3676                throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3677                                             firstLineNumber,
3678                                             line.substring(0, colonPos),
3679                                             attributeName),
3680                                        firstLineNumber, true, ldifLines, null);
3681              }
3682            }
3683    
3684            length = line.length();
3685            final ASN1OctetString value;
3686            if (length == (colonPos+1))
3687            {
3688              // The colon was the last character on the line.  This is fine.
3689              value = new ASN1OctetString();
3690            }
3691            else if (line.charAt(colonPos+1) == ':')
3692            {
3693              // Skip over any spaces leading up to the value, and then the rest of
3694              // the string is the base64-encoded value.  This is unusual and
3695              // unnecessary, but is nevertheless acceptable.
3696              int pos = colonPos+2;
3697              while ((pos < length) && (line.charAt(pos) == ' '))
3698              {
3699                pos++;
3700              }
3701    
3702              try
3703              {
3704                value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3705              }
3706              catch (final ParseException pe)
3707              {
3708                debugException(pe);
3709                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3710                     attributeName, firstLineNumber, pe.getMessage()),
3711                     firstLineNumber, true, ldifLines, pe);
3712              }
3713              catch (final Exception e)
3714              {
3715                debugException(e);
3716                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3717                                             firstLineNumber, e),
3718                                        firstLineNumber, true, ldifLines, e);
3719              }
3720            }
3721            else
3722            {
3723              // Skip over any spaces leading up to the value, and then the rest of
3724              // the string is the value.
3725              int pos = colonPos+1;
3726              while ((pos < length) && (line.charAt(pos) == ' '))
3727              {
3728                pos++;
3729              }
3730    
3731              value = new ASN1OctetString(line.substring(pos));
3732            }
3733    
3734            valueList.add(value);
3735          }
3736    
3737          final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3738          valueList.toArray(values);
3739    
3740          // If it's an add modification type, then there must be at least one
3741          // value.
3742          if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3743              (values.length == 0))
3744          {
3745            throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3746                                         firstLineNumber),
3747                                    firstLineNumber, true, ldifLines, null);
3748          }
3749    
3750          // If it's an increment modification type, then there must be exactly one
3751          // value.
3752          if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3753              (values.length != 1))
3754          {
3755            throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3756                                         firstLineNumber, attributeName),
3757                                    firstLineNumber, true, ldifLines, null);
3758          }
3759    
3760          modList.add(new Modification(modType, attributeName, values));
3761        }
3762    
3763        final Modification[] mods = new Modification[modList.size()];
3764        modList.toArray(mods);
3765        return mods;
3766      }
3767    
3768    
3769    
3770      /**
3771       * Parses the data available through the provided iterator as the body of a
3772       * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3773       * newsuperior lines).
3774       *
3775       * @param  ldifLines              The lines that comprise the LDIF
3776       *                                representation of the full record being
3777       *                                parsed.
3778       * @param  iterator               The iterator to use to access the modify DN
3779       *                                data.
3780       * @param  dn                     The current DN of the entry.
3781       * @param  controls               The set of controls to include in the change
3782       *                                record.
3783       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3784       *                                encountering attribute values which are not
3785       *                                base64-encoded but contain trailing spaces.
3786       * @param  firstLineNumber        The line number for the start of the record.
3787       *
3788       * @return  The decoded modify DN change record.
3789       *
3790       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3791       *                         modify DN change record.
3792       */
3793      private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3794           final ArrayList<StringBuilder> ldifLines,
3795           final Iterator<StringBuilder> iterator, final String dn,
3796           final List<Control> controls,
3797           final TrailingSpaceBehavior trailingSpaceBehavior,
3798           final long firstLineNumber)
3799           throws LDIFException
3800      {
3801        // The next line must be the new RDN, and it must start with "newrdn:".
3802        StringBuilder line = iterator.next();
3803        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3804        int colonPos = line.indexOf(":");
3805        if ((colonPos < 0) ||
3806            (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3807        {
3808          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3809                                       firstLineNumber),
3810                                  firstLineNumber, true, ldifLines, null);
3811        }
3812    
3813        final String newRDN;
3814        int length = line.length();
3815        if (length == (colonPos+1))
3816        {
3817          // The colon was the last character on the line.  This is not acceptable.
3818          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3819                                       firstLineNumber),
3820                                  firstLineNumber, true, ldifLines, null);
3821        }
3822        else if (line.charAt(colonPos+1) == ':')
3823        {
3824          // Skip over any spaces leading up to the value, and then the rest of the
3825          // string is the base64-encoded new RDN.
3826          int pos = colonPos+2;
3827          while ((pos < length) && (line.charAt(pos) == ' '))
3828          {
3829            pos++;
3830          }
3831    
3832          try
3833          {
3834            final byte[] dnBytes = Base64.decode(line.substring(pos));
3835            newRDN = new String(dnBytes, "UTF-8");
3836          }
3837          catch (final ParseException pe)
3838          {
3839            debugException(pe);
3840            throw new LDIFException(
3841                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3842                                                                   pe.getMessage()),
3843                 firstLineNumber, true, ldifLines, pe);
3844          }
3845          catch (final Exception e)
3846          {
3847            debugException(e);
3848            throw new LDIFException(
3849                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3850                                                                   e),
3851                 firstLineNumber, true, ldifLines, e);
3852          }
3853        }
3854        else
3855        {
3856          // Skip over any spaces leading up to the value, and then the rest of the
3857          // string is the new RDN.
3858          int pos = colonPos+1;
3859          while ((pos < length) && (line.charAt(pos) == ' '))
3860          {
3861            pos++;
3862          }
3863    
3864          newRDN = line.substring(pos);
3865        }
3866    
3867        if (newRDN.length() == 0)
3868        {
3869          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3870                                       firstLineNumber),
3871                                  firstLineNumber, true, ldifLines, null);
3872        }
3873    
3874    
3875        // The next line must be the deleteOldRDN flag, and it must start with
3876        // 'deleteoldrdn:'.
3877        if (! iterator.hasNext())
3878        {
3879          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3880                                       firstLineNumber),
3881                                  firstLineNumber, true, ldifLines, null);
3882        }
3883    
3884        line = iterator.next();
3885        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3886        colonPos = line.indexOf(":");
3887        if ((colonPos < 0) ||
3888            (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3889        {
3890          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3891                                       firstLineNumber),
3892                                  firstLineNumber, true, ldifLines, null);
3893        }
3894    
3895        final String deleteOldRDNStr;
3896        length = line.length();
3897        if (length == (colonPos+1))
3898        {
3899          // The colon was the last character on the line.  This is not acceptable.
3900          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3901                                       firstLineNumber),
3902                                  firstLineNumber, true, ldifLines, null);
3903        }
3904        else if (line.charAt(colonPos+1) == ':')
3905        {
3906          // Skip over any spaces leading up to the value, and then the rest of the
3907          // string is the base64-encoded value.  This is unusual and
3908          // unnecessary, but is nevertheless acceptable.
3909          int pos = colonPos+2;
3910          while ((pos < length) && (line.charAt(pos) == ' '))
3911          {
3912            pos++;
3913          }
3914    
3915          try
3916          {
3917            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3918            deleteOldRDNStr = new String(changeTypeBytes, "UTF-8");
3919          }
3920          catch (final ParseException pe)
3921          {
3922            debugException(pe);
3923            throw new LDIFException(
3924                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3925                      firstLineNumber, pe.getMessage()),
3926                 firstLineNumber, true, ldifLines, pe);
3927          }
3928          catch (final Exception e)
3929          {
3930            debugException(e);
3931            throw new LDIFException(
3932                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3933                      firstLineNumber, e),
3934                 firstLineNumber, true, ldifLines, e);
3935          }
3936        }
3937        else
3938        {
3939          // Skip over any spaces leading up to the value, and then the rest of the
3940          // string is the value.
3941          int pos = colonPos+1;
3942          while ((pos < length) && (line.charAt(pos) == ' '))
3943          {
3944            pos++;
3945          }
3946    
3947          deleteOldRDNStr = line.substring(pos);
3948        }
3949    
3950        final boolean deleteOldRDN;
3951        if (deleteOldRDNStr.equals("0"))
3952        {
3953          deleteOldRDN = false;
3954        }
3955        else if (deleteOldRDNStr.equals("1"))
3956        {
3957          deleteOldRDN = true;
3958        }
3959        else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3960                 deleteOldRDNStr.equalsIgnoreCase("no"))
3961        {
3962          // This is technically illegal, but we'll allow it.
3963          deleteOldRDN = false;
3964        }
3965        else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3966                 deleteOldRDNStr.equalsIgnoreCase("yes"))
3967        {
3968          // This is also technically illegal, but we'll allow it.
3969          deleteOldRDN = false;
3970        }
3971        else
3972        {
3973          throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3974                                       deleteOldRDNStr, firstLineNumber),
3975                                  firstLineNumber, true, ldifLines, null);
3976        }
3977    
3978    
3979        // If there is another line, then it must be the new superior DN and it must
3980        // start with "newsuperior:".  If this is absent, then it's fine.
3981        final String newSuperiorDN;
3982        if (iterator.hasNext())
3983        {
3984          line = iterator.next();
3985          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3986          colonPos = line.indexOf(":");
3987          if ((colonPos < 0) ||
3988              (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3989          {
3990            throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3991                                         firstLineNumber),
3992                                    firstLineNumber, true, ldifLines, null);
3993          }
3994    
3995          length = line.length();
3996          if (length == (colonPos+1))
3997          {
3998            // The colon was the last character on the line.  This is fine.
3999            newSuperiorDN = "";
4000          }
4001          else if (line.charAt(colonPos+1) == ':')
4002          {
4003            // Skip over any spaces leading up to the value, and then the rest of
4004            // the string is the base64-encoded new superior DN.
4005            int pos = colonPos+2;
4006            while ((pos < length) && (line.charAt(pos) == ' '))
4007            {
4008              pos++;
4009            }
4010    
4011            try
4012            {
4013              final byte[] dnBytes = Base64.decode(line.substring(pos));
4014              newSuperiorDN = new String(dnBytes, "UTF-8");
4015            }
4016            catch (final ParseException pe)
4017            {
4018              debugException(pe);
4019              throw new LDIFException(
4020                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4021                        firstLineNumber, pe.getMessage()),
4022                   firstLineNumber, true, ldifLines, pe);
4023            }
4024            catch (final Exception e)
4025            {
4026              debugException(e);
4027              throw new LDIFException(
4028                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
4029                        firstLineNumber, e),
4030                   firstLineNumber, true, ldifLines, e);
4031            }
4032          }
4033          else
4034          {
4035            // Skip over any spaces leading up to the value, and then the rest of
4036            // the string is the new superior DN.
4037            int pos = colonPos+1;
4038            while ((pos < length) && (line.charAt(pos) == ' '))
4039            {
4040              pos++;
4041            }
4042    
4043            newSuperiorDN = line.substring(pos);
4044          }
4045        }
4046        else
4047        {
4048          newSuperiorDN = null;
4049        }
4050    
4051    
4052        // There must not be any more lines.
4053        if (iterator.hasNext())
4054        {
4055          throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
4056                                  firstLineNumber, true, ldifLines, null);
4057        }
4058    
4059        return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
4060             newSuperiorDN, controls);
4061      }
4062    
4063    
4064    
4065      /**
4066       * Examines the line contained in the provided buffer to determine whether it
4067       * may contain one or more illegal trailing spaces.  If it does, then those
4068       * spaces will either be stripped out or an exception will be thrown to
4069       * indicate that they are illegal.
4070       *
4071       * @param  buffer                 The buffer to be examined.
4072       * @param  dn                     The DN of the LDIF record being parsed.  It
4073       *                                may be {@code null} if the DN is not yet
4074       *                                known (e.g., because the provided line is
4075       *                                expected to contain that DN).
4076       * @param  firstLineNumber        The approximate line number in the LDIF
4077       *                                source on which the LDIF record begins.
4078       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
4079       *                                encountering attribute values which are not
4080       *                                base64-encoded but contain trailing spaces.
4081       *
4082       * @throws  LDIFException  If the line contained in the provided buffer ends
4083       *                         with one or more illegal trailing spaces and
4084       *                         {@code stripTrailingSpaces} was provided with a
4085       *                         value of {@code false}.
4086       */
4087      private static void handleTrailingSpaces(final StringBuilder buffer,
4088                               final String dn, final long firstLineNumber,
4089                               final TrailingSpaceBehavior trailingSpaceBehavior)
4090              throws LDIFException
4091      {
4092        int pos = buffer.length() - 1;
4093        boolean trailingFound = false;
4094        while ((pos >= 0) && (buffer.charAt(pos) == ' '))
4095        {
4096          trailingFound = true;
4097          pos--;
4098        }
4099    
4100        if (trailingFound && (buffer.charAt(pos) != ':'))
4101        {
4102          switch (trailingSpaceBehavior)
4103          {
4104            case STRIP:
4105              buffer.setLength(pos+1);
4106              break;
4107    
4108            case REJECT:
4109              if (dn == null)
4110              {
4111                throw new LDIFException(
4112                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
4113                          buffer.toString()),
4114                     firstLineNumber, true);
4115              }
4116              else
4117              {
4118                throw new LDIFException(
4119                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
4120                          firstLineNumber, buffer.toString()),
4121                     firstLineNumber, true);
4122              }
4123    
4124            case RETAIN:
4125            default:
4126              // No action will be taken.
4127              break;
4128          }
4129        }
4130      }
4131    
4132    
4133    
4134      /**
4135       * This represents an unparsed LDIFRecord.  It stores the line number of the
4136       * first line of the record and each line of the record.
4137       */
4138      private static final class UnparsedLDIFRecord
4139      {
4140        private final ArrayList<StringBuilder> lineList;
4141        private final long firstLineNumber;
4142        private final Exception failureCause;
4143        private final boolean isEOF;
4144        private final DuplicateValueBehavior duplicateValueBehavior;
4145        private final Schema schema;
4146        private final TrailingSpaceBehavior trailingSpaceBehavior;
4147    
4148    
4149    
4150        /**
4151         * Constructor.
4152         *
4153         * @param  lineList                The lines that comprise the LDIF record.
4154         * @param  duplicateValueBehavior  The behavior to exhibit if the entry
4155         *                                 contains duplicate attribute values.
4156         * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
4157         *                                 encountering trailing spaces in
4158         *                                 non-base64-encoded attribute values.
4159         * @param  schema                  The schema to use when parsing, if
4160         *                                 applicable.
4161         * @param  firstLineNumber         The first line number of the LDIF record.
4162         */
4163        private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
4164                     final DuplicateValueBehavior duplicateValueBehavior,
4165                     final TrailingSpaceBehavior trailingSpaceBehavior,
4166                     final Schema schema, final long firstLineNumber)
4167        {
4168          this.lineList               = lineList;
4169          this.firstLineNumber        = firstLineNumber;
4170          this.duplicateValueBehavior = duplicateValueBehavior;
4171          this.trailingSpaceBehavior  = trailingSpaceBehavior;
4172          this.schema                 = schema;
4173    
4174          failureCause = null;
4175          isEOF =
4176               (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
4177        }
4178    
4179    
4180    
4181        /**
4182         * Constructor.
4183         *
4184         * @param failureCause  The Exception thrown when reading from the input.
4185         */
4186        private UnparsedLDIFRecord(final Exception failureCause)
4187        {
4188          this.failureCause = failureCause;
4189    
4190          lineList               = null;
4191          firstLineNumber        = 0;
4192          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
4193          trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
4194          schema                 = null;
4195          isEOF                  = false;
4196        }
4197    
4198    
4199    
4200        /**
4201         * Return the lines that comprise the LDIF record.
4202         *
4203         * @return  The lines that comprise the LDIF record.
4204         */
4205        private ArrayList<StringBuilder> getLineList()
4206        {
4207          return lineList;
4208        }
4209    
4210    
4211    
4212        /**
4213         * Retrieves the behavior to exhibit when encountering duplicate attribute
4214         * values.
4215         *
4216         * @return  The behavior to exhibit when encountering duplicate attribute
4217         *          values.
4218         */
4219        private DuplicateValueBehavior getDuplicateValueBehavior()
4220        {
4221          return duplicateValueBehavior;
4222        }
4223    
4224    
4225    
4226        /**
4227         * Retrieves the behavior that should be exhibited when encountering
4228         * attribute values which are not base64-encoded but contain trailing
4229         * spaces.  The LDIF specification strongly recommends that any value which
4230         * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
4231         * LDIF parser may be configured to automatically strip these spaces, to
4232         * preserve them, or to reject any entry or change record containing them.
4233         *
4234         * @return  The behavior that should be exhibited when encountering
4235         *          attribute values which are not base64-encoded but contain
4236         *          trailing spaces.
4237         */
4238        private TrailingSpaceBehavior getTrailingSpaceBehavior()
4239        {
4240          return trailingSpaceBehavior;
4241        }
4242    
4243    
4244    
4245        /**
4246         * Retrieves the schema that should be used when parsing the record, if
4247         * applicable.
4248         *
4249         * @return  The schema that should be used when parsing the record, or
4250         *          {@code null} if none should be used.
4251         */
4252        private Schema getSchema()
4253        {
4254          return schema;
4255        }
4256    
4257    
4258    
4259        /**
4260         * Return the first line number of the LDIF record.
4261         *
4262         * @return  The first line number of the LDIF record.
4263         */
4264        private long getFirstLineNumber()
4265        {
4266          return firstLineNumber;
4267        }
4268    
4269    
4270    
4271        /**
4272         * Return {@code true} iff the end of the input was reached.
4273         *
4274         * @return  {@code true} iff the end of the input was reached.
4275         */
4276        private boolean isEOF()
4277        {
4278          return isEOF;
4279        }
4280    
4281    
4282    
4283        /**
4284         * Returns the reason that reading the record lines failed.  This normally
4285         * is only non-null if something bad happened to the input stream (like
4286         * a disk read error).
4287         *
4288         * @return  The reason that reading the record lines failed.
4289         */
4290        private Exception getFailureCause()
4291        {
4292          return failureCause;
4293        }
4294      }
4295    
4296    
4297      /**
4298       * When processing in asynchronous mode, this thread is responsible for
4299       * reading the raw unparsed records from the input and submitting them for
4300       * processing.
4301       */
4302      private final class LineReaderThread
4303           extends Thread
4304      {
4305        /**
4306         * Constructor.
4307         */
4308        private LineReaderThread()
4309        {
4310          super("Asynchronous LDIF line reader");
4311          setDaemon(true);
4312        }
4313    
4314    
4315    
4316        /**
4317         * Reads raw, unparsed records from the input and submits them for
4318         * processing until the input is finished or closed.
4319         */
4320        @Override()
4321        public void run()
4322        {
4323          try
4324          {
4325            boolean stopProcessing = false;
4326            while (!stopProcessing)
4327            {
4328              UnparsedLDIFRecord unparsedRecord = null;
4329              try
4330              {
4331                unparsedRecord = readUnparsedRecord();
4332              }
4333              catch (IOException e)
4334              {
4335                debugException(e);
4336                unparsedRecord = new UnparsedLDIFRecord(e);
4337                stopProcessing = true;
4338              }
4339              catch (Exception e)
4340              {
4341                debugException(e);
4342                unparsedRecord = new UnparsedLDIFRecord(e);
4343              }
4344    
4345              try
4346              {
4347                asyncParser.submit(unparsedRecord);
4348              }
4349              catch (InterruptedException e)
4350              {
4351                debugException(e);
4352                // If this thread is interrupted, then someone wants us to stop
4353                // processing, so that's what we'll do.
4354                stopProcessing = true;
4355              }
4356    
4357              if ((unparsedRecord == null) || (unparsedRecord.isEOF()))
4358              {
4359                stopProcessing = true;
4360              }
4361            }
4362          }
4363          finally
4364          {
4365            try
4366            {
4367              asyncParser.shutdown();
4368            }
4369            catch (InterruptedException e)
4370            {
4371              debugException(e);
4372            }
4373            finally
4374            {
4375              asyncParsingComplete.set(true);
4376            }
4377          }
4378        }
4379      }
4380    
4381    
4382    
4383      /**
4384       * Used to parse Records asynchronously.
4385       */
4386      private final class RecordParser implements Processor<UnparsedLDIFRecord,
4387                                                            LDIFRecord>
4388      {
4389        /**
4390         * {@inheritDoc}
4391         */
4392        public LDIFRecord process(final UnparsedLDIFRecord input)
4393               throws LDIFException
4394        {
4395          LDIFRecord record = decodeRecord(input, relativeBasePath, schema);
4396    
4397          if ((record instanceof Entry) && (entryTranslator != null))
4398          {
4399            record = entryTranslator.translate((Entry) record,
4400                 input.getFirstLineNumber());
4401    
4402            if (record == null)
4403            {
4404              record = SKIP_ENTRY;
4405            }
4406          }
4407          if ((record instanceof LDIFChangeRecord) &&
4408              (changeRecordTranslator != null))
4409          {
4410            record = changeRecordTranslator.translate((LDIFChangeRecord) record,
4411                 input.getFirstLineNumber());
4412    
4413            if (record == null)
4414            {
4415              record = SKIP_ENTRY;
4416            }
4417          }
4418          return record;
4419        }
4420      }
4421    }