001    /*
002     * Copyright 2007-2015 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2015 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldif;
022    
023    
024    
025    import java.io.BufferedReader;
026    import java.io.BufferedWriter;
027    import java.io.File;
028    import java.io.FileInputStream;
029    import java.io.FileWriter;
030    import java.io.InputStream;
031    import java.io.InputStreamReader;
032    import java.io.IOException;
033    import java.text.ParseException;
034    import java.util.ArrayList;
035    import java.util.Collection;
036    import java.util.Iterator;
037    import java.util.HashSet;
038    import java.util.LinkedHashMap;
039    import java.util.List;
040    import java.util.Set;
041    import java.util.concurrent.BlockingQueue;
042    import java.util.concurrent.ArrayBlockingQueue;
043    import java.util.concurrent.TimeUnit;
044    import java.util.concurrent.atomic.AtomicBoolean;
045    import java.nio.charset.Charset;
046    
047    import com.unboundid.asn1.ASN1OctetString;
048    import com.unboundid.ldap.matchingrules.CaseIgnoreStringMatchingRule;
049    import com.unboundid.ldap.matchingrules.MatchingRule;
050    import com.unboundid.ldap.sdk.Attribute;
051    import com.unboundid.ldap.sdk.Control;
052    import com.unboundid.ldap.sdk.Entry;
053    import com.unboundid.ldap.sdk.Modification;
054    import com.unboundid.ldap.sdk.ModificationType;
055    import com.unboundid.ldap.sdk.LDAPException;
056    import com.unboundid.ldap.sdk.schema.AttributeTypeDefinition;
057    import com.unboundid.ldap.sdk.schema.Schema;
058    import com.unboundid.util.AggregateInputStream;
059    import com.unboundid.util.Base64;
060    import com.unboundid.util.LDAPSDKThreadFactory;
061    import com.unboundid.util.parallel.AsynchronousParallelProcessor;
062    import com.unboundid.util.parallel.Result;
063    import com.unboundid.util.parallel.ParallelProcessor;
064    import com.unboundid.util.parallel.Processor;
065    
066    import static com.unboundid.ldif.LDIFMessages.*;
067    import static com.unboundid.util.Debug.*;
068    import static com.unboundid.util.StaticUtils.*;
069    import static com.unboundid.util.Validator.*;
070    
071    /**
072     * This class provides an LDIF reader, which can be used to read and decode
073     * entries and change records from a data source using the LDAP Data Interchange
074     * Format as per <A HREF="http://www.ietf.org/rfc/rfc2849.txt">RFC 2849</A>.
075     * <BR>
076     * This class is not synchronized.  If multiple threads read from the
077     * LDIFReader, they must be synchronized externally.
078     * <BR><BR>
079     * <H2>Example</H2>
080     * The following example iterates through all entries contained in an LDIF file
081     * and attempts to add them to a directory server:
082     * <PRE>
083     * LDIFReader ldifReader = new LDIFReader(pathToLDIFFile);
084     *
085     * int entriesRead = 0;
086     * int entriesAdded = 0;
087     * int errorsEncountered = 0;
088     * while (true)
089     * {
090     *   Entry entry;
091     *   try
092     *   {
093     *     entry = ldifReader.readEntry();
094     *     if (entry == null)
095     *     {
096     *       // All entries have been read.
097     *       break;
098     *     }
099     *
100     *     entriesRead++;
101     *   }
102     *   catch (LDIFException le)
103     *   {
104     *     errorsEncountered++;
105     *     if (le.mayContinueReading())
106     *     {
107     *       // A recoverable error occurred while attempting to read a change
108     *       // record, at or near line number le.getLineNumber()
109     *       // The entry will be skipped, but we'll try to keep reading from the
110     *       // LDIF file.
111     *       continue;
112     *     }
113     *     else
114     *     {
115     *       // An unrecoverable error occurred while attempting to read an entry
116     *       // at or near line number le.getLineNumber()
117     *       // No further LDIF processing will be performed.
118     *       break;
119     *     }
120     *   }
121     *   catch (IOException ioe)
122     *   {
123     *     // An I/O error occurred while attempting to read from the LDIF file.
124     *     // No further LDIF processing will be performed.
125     *     errorsEncountered++;
126     *     break;
127     *   }
128     *
129     *   LDAPResult addResult;
130     *   try
131     *   {
132     *     addResult = connection.add(entry);
133     *     // If we got here, then the change should have been processed
134     *     // successfully.
135     *     entriesAdded++;
136     *   }
137     *   catch (LDAPException le)
138     *   {
139     *     // If we got here, then the change attempt failed.
140     *     addResult = le.toLDAPResult();
141     *     errorsEncountered++;
142     *   }
143     * }
144     *
145     * ldifReader.close();
146     * </PRE>
147     */
148    public final class LDIFReader
149    {
150      /**
151       * The default buffer size (128KB) that will be used when reading from the
152       * data source.
153       */
154      public static final int DEFAULT_BUFFER_SIZE = 128 * 1024;
155    
156    
157    
158      /*
159       * When processing asynchronously, this determines how many of the allocated
160       * worker threads are used to parse each batch of read entries.
161       */
162      private static final int ASYNC_MIN_PER_PARSING_THREAD = 3;
163    
164    
165    
166      /**
167       * When processing asynchronously, this specifies the size of the pending and
168       * completed queues.
169       */
170      private static final int ASYNC_QUEUE_SIZE = 500;
171    
172    
173    
174      /**
175       * Special entry used internally to signal that the LDIFReaderEntryTranslator
176       * has signalled that a read Entry should be skipped by returning null,
177       * which normally implies EOF.
178       */
179      private static final Entry SKIP_ENTRY = new Entry("cn=skipped");
180    
181    
182    
183      /**
184       * The default base path that will be prepended to relative paths.  It will
185       * end with a trailing slash.
186       */
187      private static final String DEFAULT_RELATIVE_BASE_PATH;
188      static
189      {
190        final File currentDir;
191        String currentDirString = System.getProperty("user.dir");
192        if (currentDirString == null)
193        {
194          currentDir = new File(".");
195        }
196        else
197        {
198          currentDir = new File(currentDirString);
199        }
200    
201        final String currentDirAbsolutePath = currentDir.getAbsolutePath();
202        if (currentDirAbsolutePath.endsWith(File.separator))
203        {
204          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath;
205        }
206        else
207        {
208          DEFAULT_RELATIVE_BASE_PATH = currentDirAbsolutePath + File.separator;
209        }
210      }
211    
212    
213    
214      // The buffered reader that will be used to read LDIF data.
215      private final BufferedReader reader;
216    
217      // The behavior that should be exhibited when encountering duplicate attribute
218      // values.
219      private volatile DuplicateValueBehavior duplicateValueBehavior;
220    
221      // A line number counter.
222      private long lineNumberCounter = 0;
223    
224      // The change record translator to use, if any.
225      private final LDIFReaderChangeRecordTranslator changeRecordTranslator;
226    
227      // The entry translator to use, if any.
228      private final LDIFReaderEntryTranslator entryTranslator;
229    
230      // The schema that will be used when processing, if applicable.
231      private Schema schema;
232    
233      // Specifies the base path that will be prepended to relative paths for file
234      // URLs.
235      private volatile String relativeBasePath;
236    
237      // The behavior that should be exhibited with regard to illegal trailing
238      // spaces in attribute values.
239      private volatile TrailingSpaceBehavior trailingSpaceBehavior;
240    
241      // True iff we are processing asynchronously.
242      private final boolean isAsync;
243    
244      //
245      // The following only apply to asynchronous processing.
246      //
247    
248      // Parses entries asynchronously.
249      private final AsynchronousParallelProcessor<UnparsedLDIFRecord, LDIFRecord>
250           asyncParser;
251    
252      // Set to true when the end of the input is reached.
253      private final AtomicBoolean asyncParsingComplete;
254    
255      // The records that have been read and parsed.
256      private final BlockingQueue<Result<UnparsedLDIFRecord, LDIFRecord>>
257           asyncParsedRecords;
258    
259    
260    
261      /**
262       * Creates a new LDIF reader that will read data from the specified file.
263       *
264       * @param  path  The path to the file from which the data is to be read.  It
265       *               must not be {@code null}.
266       *
267       * @throws  IOException  If a problem occurs while opening the file for
268       *                       reading.
269       */
270      public LDIFReader(final String path)
271             throws IOException
272      {
273        this(new FileInputStream(path));
274      }
275    
276    
277    
278      /**
279       * Creates a new LDIF reader that will read data from the specified file
280       * and parses the LDIF records asynchronously using the specified number of
281       * threads.
282       *
283       * @param  path  The path to the file from which the data is to be read.  It
284       *               must not be {@code null}.
285       * @param  numParseThreads  If this value is greater than zero, then the
286       *                          specified number of threads will be used to
287       *                          asynchronously read and parse the LDIF file.
288       *
289       * @throws  IOException  If a problem occurs while opening the file for
290       *                       reading.
291       *
292       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
293       *      constructor for more details about asynchronous processing.
294       */
295      public LDIFReader(final String path, final int numParseThreads)
296             throws IOException
297      {
298        this(new FileInputStream(path), numParseThreads);
299      }
300    
301    
302    
303      /**
304       * Creates a new LDIF reader that will read data from the specified file.
305       *
306       * @param  file  The file from which the data is to be read.  It must not be
307       *               {@code null}.
308       *
309       * @throws  IOException  If a problem occurs while opening the file for
310       *                       reading.
311       */
312      public LDIFReader(final File file)
313             throws IOException
314      {
315        this(new FileInputStream(file));
316      }
317    
318    
319    
320      /**
321       * Creates a new LDIF reader that will read data from the specified file
322       * and optionally parses the LDIF records asynchronously using the specified
323       * number of threads.
324       *
325       * @param  file             The file from which the data is to be read.  It
326       *                          must not be {@code null}.
327       * @param  numParseThreads  If this value is greater than zero, then the
328       *                          specified number of threads will be used to
329       *                          asynchronously read and parse the LDIF file.
330       *
331       * @throws  IOException  If a problem occurs while opening the file for
332       *                       reading.
333       */
334      public LDIFReader(final File file, final int numParseThreads)
335             throws IOException
336      {
337        this(new FileInputStream(file), numParseThreads);
338      }
339    
340    
341    
342      /**
343       * Creates a new LDIF reader that will read data from the specified files in
344       * the order in which they are provided and optionally parses the LDIF records
345       * asynchronously using the specified number of threads.
346       *
347       * @param  files            The files from which the data is to be read.  It
348       *                          must not be {@code null} or empty.
349       * @param  numParseThreads  If this value is greater than zero, then the
350       *                          specified number of threads will be used to
351       *                          asynchronously read and parse the LDIF file.
352       * @param entryTranslator   The LDIFReaderEntryTranslator to apply to entries
353       *                          before they are returned.  This is normally
354       *                          {@code null}, which causes entries to be returned
355       *                          unaltered. This is particularly useful when
356       *                          parsing the input file in parallel because the
357       *                          entry translation is also done in parallel.
358       *
359       * @throws  IOException  If a problem occurs while opening the file for
360       *                       reading.
361       */
362      public LDIFReader(final File[] files, final int numParseThreads,
363                        final LDIFReaderEntryTranslator entryTranslator)
364             throws IOException
365      {
366        this(files, numParseThreads, entryTranslator, null);
367      }
368    
369    
370    
371      /**
372       * Creates a new LDIF reader that will read data from the specified files in
373       * the order in which they are provided and optionally parses the LDIF records
374       * asynchronously using the specified number of threads.
375       *
376       * @param  files                   The files from which the data is to be
377       *                                 read.  It must not be {@code null} or
378       *                                 empty.
379       * @param  numParseThreads         If this value is greater than zero, then
380       *                                 the specified number of threads will be
381       *                                 used to asynchronously read and parse the
382       *                                 LDIF file.
383       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
384       *                                 entries before they are returned.  This is
385       *                                 normally {@code null}, which causes entries
386       *                                 to be returned unaltered.  This is
387       *                                 particularly useful when parsing the input
388       *                                 file in parallel because the entry
389       *                                 translation is also done in parallel.
390       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
391       *                                 apply to change records before they are
392       *                                 returned.  This is normally {@code null},
393       *                                 which causes change records to be returned
394       *                                 unaltered.  This is particularly useful
395       *                                 when parsing the input file in parallel
396       *                                 because the change record translation is
397       *                                 also done in parallel.
398       *
399       * @throws  IOException  If a problem occurs while opening the file for
400       *                       reading.
401       */
402      public LDIFReader(final File[] files, final int numParseThreads,
403                  final LDIFReaderEntryTranslator entryTranslator,
404                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
405             throws IOException
406      {
407        this(createAggregateInputStream(files), numParseThreads, entryTranslator,
408             changeRecordTranslator);
409      }
410    
411    
412    
413      /**
414       * Creates a new aggregate input stream that will read data from the specified
415       * files.  If there are multiple files, then a "padding" file will be inserted
416       * between them to ensure that there is at least one blank line between the
417       * end of one file and the beginning of another.
418       *
419       * @param  files  The files from which the data is to be read.  It must not be
420       *                {@code null} or empty.
421       *
422       * @return  The input stream to use to read data from the provided files.
423       *
424       * @throws  IOException  If a problem is encountered while attempting to
425       *                       create the input stream.
426       */
427      private static InputStream createAggregateInputStream(final File... files)
428              throws IOException
429      {
430        if (files.length == 0)
431        {
432          throw new IOException(ERR_READ_NO_LDIF_FILES.get());
433        }
434        else if (files.length == 1)
435        {
436          return new FileInputStream(files[0]);
437        }
438        else
439        {
440          final File spacerFile =
441               File.createTempFile("ldif-reader-spacer", ".ldif");
442          spacerFile.deleteOnExit();
443    
444          final BufferedWriter spacerWriter =
445               new BufferedWriter(new FileWriter(spacerFile));
446          try
447          {
448            spacerWriter.newLine();
449            spacerWriter.newLine();
450          }
451          finally
452          {
453            spacerWriter.close();
454          }
455    
456          final File[] returnArray = new File[(files.length * 2) - 1];
457          returnArray[0] = files[0];
458    
459          int pos = 1;
460          for (int i=1; i < files.length; i++)
461          {
462            returnArray[pos++] = spacerFile;
463            returnArray[pos++] = files[i];
464          }
465    
466          return new AggregateInputStream(returnArray);
467        }
468      }
469    
470    
471    
472      /**
473       * Creates a new LDIF reader that will read data from the provided input
474       * stream.
475       *
476       * @param  inputStream  The input stream from which the data is to be read.
477       *                      It must not be {@code null}.
478       */
479      public LDIFReader(final InputStream inputStream)
480      {
481        this(inputStream, 0);
482      }
483    
484    
485    
486      /**
487       * Creates a new LDIF reader that will read data from the specified stream
488       * and parses the LDIF records asynchronously using the specified number of
489       * threads.
490       *
491       * @param  inputStream  The input stream from which the data is to be read.
492       *                      It must not be {@code null}.
493       * @param  numParseThreads  If this value is greater than zero, then the
494       *                          specified number of threads will be used to
495       *                          asynchronously read and parse the LDIF file.
496       *
497       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
498       *      constructor for more details about asynchronous processing.
499       */
500      public LDIFReader(final InputStream inputStream, final int numParseThreads)
501      {
502        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
503        this(new BufferedReader(new InputStreamReader(inputStream,
504                                                      Charset.forName("UTF-8")),
505                                DEFAULT_BUFFER_SIZE),
506             numParseThreads);
507      }
508    
509    
510    
511      /**
512       * Creates a new LDIF reader that will read data from the specified stream
513       * and parses the LDIF records asynchronously using the specified number of
514       * threads.
515       *
516       * @param  inputStream  The input stream from which the data is to be read.
517       *                      It must not be {@code null}.
518       * @param  numParseThreads  If this value is greater than zero, then the
519       *                          specified number of threads will be used to
520       *                          asynchronously read and parse the LDIF file.
521       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
522       *                         entries before they are returned.  This is normally
523       *                         {@code null}, which causes entries to be returned
524       *                         unaltered. This is particularly useful when parsing
525       *                         the input file in parallel because the entry
526       *                         translation is also done in parallel.
527       *
528       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
529       *      constructor for more details about asynchronous processing.
530       */
531      public LDIFReader(final InputStream inputStream, final int numParseThreads,
532                        final LDIFReaderEntryTranslator entryTranslator)
533      {
534        this(inputStream, numParseThreads, entryTranslator, null);
535      }
536    
537    
538    
539      /**
540       * Creates a new LDIF reader that will read data from the specified stream
541       * and parses the LDIF records asynchronously using the specified number of
542       * threads.
543       *
544       * @param  inputStream             The input stream from which the data is to
545       *                                 be read.  It must not be {@code null}.
546       * @param  numParseThreads         If this value is greater than zero, then
547       *                                 the specified number of threads will be
548       *                                 used to asynchronously read and parse the
549       *                                 LDIF file.
550       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
551       *                                 entries before they are returned.  This is
552       *                                 normally {@code null}, which causes entries
553       *                                 to be returned unaltered.  This is
554       *                                 particularly useful when parsing the input
555       *                                 file in parallel because the entry
556       *                                 translation is also done in parallel.
557       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
558       *                                 apply to change records before they are
559       *                                 returned.  This is normally {@code null},
560       *                                 which causes change records to be returned
561       *                                 unaltered.  This is particularly useful
562       *                                 when parsing the input file in parallel
563       *                                 because the change record translation is
564       *                                 also done in parallel.
565       *
566       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
567       *      constructor for more details about asynchronous processing.
568       */
569      public LDIFReader(final InputStream inputStream, final int numParseThreads,
570                  final LDIFReaderEntryTranslator entryTranslator,
571                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
572      {
573        // UTF-8 is required by RFC 2849.  Java guarantees it's always available.
574        this(new BufferedReader(
575                  new InputStreamReader(inputStream, Charset.forName("UTF-8")),
576                  DEFAULT_BUFFER_SIZE),
577             numParseThreads, entryTranslator, changeRecordTranslator);
578      }
579    
580    
581    
582      /**
583       * Creates a new LDIF reader that will use the provided buffered reader to
584       * read the LDIF data.  The encoding of the underlying Reader must be set to
585       * "UTF-8" as required by RFC 2849.
586       *
587       * @param  reader  The buffered reader that will be used to read the LDIF
588       *                 data.  It must not be {@code null}.
589       */
590      public LDIFReader(final BufferedReader reader)
591      {
592        this(reader, 0);
593      }
594    
595    
596    
597      /**
598       * Creates a new LDIF reader that will read data from the specified buffered
599       * reader and parses the LDIF records asynchronously using the specified
600       * number of threads.  The encoding of the underlying Reader must be set to
601       * "UTF-8" as required by RFC 2849.
602       *
603       * @param reader The buffered reader that will be used to read the LDIF data.
604       *               It must not be {@code null}.
605       * @param  numParseThreads  If this value is greater than zero, then the
606       *                          specified number of threads will be used to
607       *                          asynchronously read and parse the LDIF file.
608       *
609       * @see #LDIFReader(BufferedReader, int, LDIFReaderEntryTranslator)
610       *      constructor for more details about asynchronous processing.
611       */
612      public LDIFReader(final BufferedReader reader, final int numParseThreads)
613      {
614        this(reader, numParseThreads, null);
615      }
616    
617    
618    
619      /**
620       * Creates a new LDIF reader that will read data from the specified buffered
621       * reader and parses the LDIF records asynchronously using the specified
622       * number of threads.  The encoding of the underlying Reader must be set to
623       * "UTF-8" as required by RFC 2849.
624       *
625       * @param reader The buffered reader that will be used to read the LDIF data.
626       *               It must not be {@code null}.
627       * @param  numParseThreads  If this value is greater than zero, then the
628       *                          specified number of threads will be used to
629       *                          asynchronously read and parse the LDIF file.
630       *                          This should only be set to greater than zero when
631       *                          performance analysis has demonstrated that reading
632       *                          and parsing the LDIF is a bottleneck.  The default
633       *                          synchronous processing is normally fast enough.
634       *                          There is little benefit in passing in a value
635       *                          greater than four (unless there is an
636       *                          LDIFReaderEntryTranslator that does time-consuming
637       *                          processing).  A value of zero implies the
638       *                          default behavior of reading and parsing LDIF
639       *                          records synchronously when one of the read
640       *                          methods is called.
641       * @param entryTranslator  The LDIFReaderEntryTranslator to apply to read
642       *                         entries before they are returned.  This is normally
643       *                         {@code null}, which causes entries to be returned
644       *                         unaltered. This is particularly useful when parsing
645       *                         the input file in parallel because the entry
646       *                         translation is also done in parallel.
647       */
648      public LDIFReader(final BufferedReader reader,
649                        final int numParseThreads,
650                        final LDIFReaderEntryTranslator entryTranslator)
651      {
652        this(reader, numParseThreads, entryTranslator, null);
653      }
654    
655    
656    
657      /**
658       * Creates a new LDIF reader that will read data from the specified buffered
659       * reader and parses the LDIF records asynchronously using the specified
660       * number of threads.  The encoding of the underlying Reader must be set to
661       * "UTF-8" as required by RFC 2849.
662       *
663       * @param reader                   The buffered reader that will be used to
664       *                                 read the LDIF data.  It must not be
665       *                                 {@code null}.
666       * @param  numParseThreads         If this value is greater than zero, then
667       *                                 the specified number of threads will be
668       *                                 used to asynchronously read and parse the
669       *                                 LDIF file.
670       * @param  entryTranslator         The LDIFReaderEntryTranslator to apply to
671       *                                 entries before they are returned.  This is
672       *                                 normally {@code null}, which causes entries
673       *                                 to be returned unaltered.  This is
674       *                                 particularly useful when parsing the input
675       *                                 file in parallel because the entry
676       *                                 translation is also done in parallel.
677       * @param  changeRecordTranslator  The LDIFReaderChangeRecordTranslator to
678       *                                 apply to change records before they are
679       *                                 returned.  This is normally {@code null},
680       *                                 which causes change records to be returned
681       *                                 unaltered.  This is particularly useful
682       *                                 when parsing the input file in parallel
683       *                                 because the change record translation is
684       *                                 also done in parallel.
685       */
686      public LDIFReader(final BufferedReader reader, final int numParseThreads,
687                  final LDIFReaderEntryTranslator entryTranslator,
688                  final LDIFReaderChangeRecordTranslator changeRecordTranslator)
689      {
690        ensureNotNull(reader);
691        ensureTrue(numParseThreads >= 0,
692                   "LDIFReader.numParseThreads must not be negative.");
693    
694        this.reader = reader;
695        this.entryTranslator = entryTranslator;
696        this.changeRecordTranslator = changeRecordTranslator;
697    
698        duplicateValueBehavior = DuplicateValueBehavior.STRIP;
699        trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
700    
701        relativeBasePath = DEFAULT_RELATIVE_BASE_PATH;
702    
703        if (numParseThreads == 0)
704        {
705          isAsync = false;
706          asyncParser = null;
707          asyncParsingComplete = null;
708          asyncParsedRecords = null;
709        }
710        else
711        {
712          isAsync = true;
713          asyncParsingComplete = new AtomicBoolean(false);
714    
715          // Decodes entries in parallel.
716          final LDAPSDKThreadFactory threadFactory =
717               new LDAPSDKThreadFactory("LDIFReader Worker", true, null);
718          final ParallelProcessor<UnparsedLDIFRecord, LDIFRecord> parallelParser =
719               new ParallelProcessor<UnparsedLDIFRecord, LDIFRecord>(
720                    new RecordParser(), threadFactory, numParseThreads,
721                    ASYNC_MIN_PER_PARSING_THREAD);
722    
723          final BlockingQueue<UnparsedLDIFRecord> pendingQueue = new
724               ArrayBlockingQueue<UnparsedLDIFRecord>(ASYNC_QUEUE_SIZE);
725    
726          // The output queue must be a little more than twice as big as the input
727          // queue to more easily handle being shutdown in the middle of processing
728          // when the queues are full and threads are blocked.
729          asyncParsedRecords = new ArrayBlockingQueue
730               <Result<UnparsedLDIFRecord, LDIFRecord>>(2 * ASYNC_QUEUE_SIZE + 100);
731    
732          asyncParser = new AsynchronousParallelProcessor
733               <UnparsedLDIFRecord, LDIFRecord>(pendingQueue, parallelParser,
734                                                asyncParsedRecords);
735    
736          final LineReaderThread lineReaderThread = new LineReaderThread();
737          lineReaderThread.start();
738        }
739      }
740    
741    
742    
743      /**
744       * Reads entries from the LDIF file with the specified path and returns them
745       * as a {@code List}.  This is a convenience method that should only be used
746       * for data sets that are small enough so that running out of memory isn't a
747       * concern.
748       *
749       * @param  path  The path to the LDIF file containing the entries to be read.
750       *
751       * @return  A list of the entries read from the given LDIF file.
752       *
753       * @throws  IOException  If a problem occurs while attempting to read data
754       *                       from the specified file.
755       *
756       * @throws  LDIFException  If a problem is encountered while attempting to
757       *                         decode data read as LDIF.
758       */
759      public static List<Entry> readEntries(final String path)
760             throws IOException, LDIFException
761      {
762        return readEntries(new LDIFReader(path));
763      }
764    
765    
766    
767      /**
768       * Reads entries from the specified LDIF file and returns them as a
769       * {@code List}.  This is a convenience method that should only be used for
770       * data sets that are small enough so that running out of memory isn't a
771       * concern.
772       *
773       * @param  file  A reference to the LDIF file containing the entries to be
774       *               read.
775       *
776       * @return  A list of the entries read from the given LDIF file.
777       *
778       * @throws  IOException  If a problem occurs while attempting to read data
779       *                       from the specified file.
780       *
781       * @throws  LDIFException  If a problem is encountered while attempting to
782       *                         decode data read as LDIF.
783       */
784      public static List<Entry> readEntries(final File file)
785             throws IOException, LDIFException
786      {
787        return readEntries(new LDIFReader(file));
788      }
789    
790    
791    
792      /**
793       * Reads and decodes LDIF entries from the provided input stream and
794       * returns them as a {@code List}.  This is a convenience method that should
795       * only be used for data sets that are small enough so that running out of
796       * memory isn't a concern.
797       *
798       * @param  inputStream  The input stream from which the entries should be
799       *                      read.  The input stream will be closed before
800       *                      returning.
801       *
802       * @return  A list of the entries read from the given input stream.
803       *
804       * @throws  IOException  If a problem occurs while attempting to read data
805       *                       from the input stream.
806       *
807       * @throws  LDIFException  If a problem is encountered while attempting to
808       *                         decode data read as LDIF.
809       */
810      public static List<Entry> readEntries(final InputStream inputStream)
811             throws IOException, LDIFException
812      {
813        return readEntries(new LDIFReader(inputStream));
814      }
815    
816    
817    
818      /**
819       * Reads entries from the provided LDIF reader and returns them as a list.
820       *
821       * @param  reader  The reader from which the entries should be read.  It will
822       *                 be closed before returning.
823       *
824       * @return  A list of the entries read from the provided reader.
825       *
826       * @throws  IOException  If a problem was encountered while attempting to read
827       *                       data from the LDIF data source.
828       *
829       * @throws  LDIFException  If a problem is encountered while attempting to
830       *                         decode data read as LDIF.
831       */
832      private static List<Entry> readEntries(final LDIFReader reader)
833              throws IOException, LDIFException
834      {
835        try
836        {
837          final ArrayList<Entry> entries = new ArrayList<Entry>(10);
838          while (true)
839          {
840            final Entry e = reader.readEntry();
841            if (e == null)
842            {
843              break;
844            }
845    
846            entries.add(e);
847          }
848    
849          return entries;
850        }
851        finally
852        {
853          reader.close();
854        }
855      }
856    
857    
858    
859      /**
860       * Closes this LDIF reader and the underlying LDIF source.
861       *
862       * @throws  IOException  If a problem occurs while closing the underlying LDIF
863       *                       source.
864       */
865      public void close()
866             throws IOException
867      {
868        reader.close();
869    
870        if (isAsync())
871        {
872          // Closing the reader will trigger the LineReaderThread to complete, but
873          // not if it's blocked submitting the next UnparsedLDIFRecord.  To avoid
874          // this, we clear out the completed output queue, which is larger than
875          // the input queue, so the LineReaderThread will stop reading and
876          // shutdown the asyncParser.
877          asyncParsedRecords.clear();
878        }
879      }
880    
881    
882    
883      /**
884       * Indicates whether to ignore any duplicate values encountered while reading
885       * LDIF records.
886       *
887       * @return  {@code true} if duplicate values should be ignored, or
888       *          {@code false} if any LDIF records containing duplicate values
889       *          should be rejected.
890       *
891       * @deprecated  Use the {@code getDuplicateValueBehavior} method instead.
892       */
893      @Deprecated()
894      public boolean ignoreDuplicateValues()
895      {
896        return (duplicateValueBehavior == DuplicateValueBehavior.STRIP);
897      }
898    
899    
900    
901      /**
902       * Specifies whether to ignore any duplicate values encountered while reading
903       * LDIF records.
904       *
905       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
906       *                                attribute values encountered while reading
907       *                                LDIF records.
908       *
909       * @deprecated  Use the {@code setDuplicateValueBehavior} method instead.
910       */
911      @Deprecated()
912      public void setIgnoreDuplicateValues(final boolean ignoreDuplicateValues)
913      {
914        if (ignoreDuplicateValues)
915        {
916          duplicateValueBehavior = DuplicateValueBehavior.STRIP;
917        }
918        else
919        {
920          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
921        }
922      }
923    
924    
925    
926      /**
927       * Retrieves the behavior that should be exhibited if the LDIF reader
928       * encounters an entry with duplicate values.
929       *
930       * @return  The behavior that should be exhibited if the LDIF reader
931       *          encounters an entry with duplicate values.
932       */
933      public DuplicateValueBehavior getDuplicateValueBehavior()
934      {
935        return duplicateValueBehavior;
936      }
937    
938    
939    
940      /**
941       * Specifies the behavior that should be exhibited if the LDIF reader
942       * encounters an entry with duplicate values.
943       *
944       * @param  duplicateValueBehavior  The behavior that should be exhibited if
945       *                                 the LDIF reader encounters an entry with
946       *                                 duplicate values.
947       */
948      public void setDuplicateValueBehavior(
949                       final DuplicateValueBehavior duplicateValueBehavior)
950      {
951        this.duplicateValueBehavior = duplicateValueBehavior;
952      }
953    
954    
955    
956      /**
957       * Indicates whether to strip off any illegal trailing spaces that may appear
958       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
959       * specification strongly recommends that any value which legitimately
960       * contains trailing spaces be base64-encoded, and any spaces which appear
961       * after the end of non-base64-encoded values may therefore be considered
962       * invalid.  If any such trailing spaces are encountered in an LDIF record and
963       * they are not to be stripped, then an {@code LDIFException} will be thrown
964       * for that record.
965       * <BR><BR>
966       * Note that this applies only to spaces after the end of a value, and not to
967       * spaces which may appear at the end of a line for a value that is wrapped
968       * and continued on the next line.
969       *
970       * @return  {@code true} if illegal trailing spaces should be stripped off, or
971       *          {@code false} if LDIF records containing illegal trailing spaces
972       *          should be rejected.
973       *
974       * @deprecated  Use the {@code getTrailingSpaceBehavior} method instead.
975       */
976      @Deprecated()
977      public boolean stripTrailingSpaces()
978      {
979        return (trailingSpaceBehavior == TrailingSpaceBehavior.STRIP);
980      }
981    
982    
983    
984      /**
985       * Specifies whether to strip off any illegal trailing spaces that may appear
986       * in LDIF records (e.g., after an entry DN or attribute value).  The LDIF
987       * specification strongly recommends that any value which legitimately
988       * contains trailing spaces be base64-encoded, and any spaces which appear
989       * after the end of non-base64-encoded values may therefore be considered
990       * invalid.  If any such trailing spaces are encountered in an LDIF record and
991       * they are not to be stripped, then an {@code LDIFException} will be thrown
992       * for that record.
993       * <BR><BR>
994       * Note that this applies only to spaces after the end of a value, and not to
995       * spaces which may appear at the end of a line for a value that is wrapped
996       * and continued on the next line.
997       *
998       * @param  stripTrailingSpaces  Indicates whether to strip off any illegal
999       *                              trailing spaces, or {@code false} if LDIF
1000       *                              records containing them should be rejected.
1001       *
1002       * @deprecated  Use the {@code setTrailingSpaceBehavior} method instead.
1003       */
1004      @Deprecated()
1005      public void setStripTrailingSpaces(final boolean stripTrailingSpaces)
1006      {
1007        trailingSpaceBehavior = stripTrailingSpaces
1008             ? TrailingSpaceBehavior.STRIP
1009             : TrailingSpaceBehavior.REJECT;
1010      }
1011    
1012    
1013    
1014      /**
1015       * Retrieves the behavior that should be exhibited when encountering attribute
1016       * values which are not base64-encoded but contain trailing spaces.  The LDIF
1017       * specification strongly recommends that any value which legitimately
1018       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1019       * may be configured to automatically strip these spaces, to preserve them, or
1020       * to reject any entry or change record containing them.
1021       *
1022       * @return  The behavior that should be exhibited when encountering attribute
1023       *          values which are not base64-encoded but contain trailing spaces.
1024       */
1025      public TrailingSpaceBehavior getTrailingSpaceBehavior()
1026      {
1027        return trailingSpaceBehavior;
1028      }
1029    
1030    
1031    
1032      /**
1033       * Specifies the behavior that should be exhibited when encountering attribute
1034       * values which are not base64-encoded but contain trailing spaces.  The LDIF
1035       * specification strongly recommends that any value which legitimately
1036       * contains trailing spaces be base64-encoded, but the LDAP SDK LDIF parser
1037       * may be configured to automatically strip these spaces, to preserve them, or
1038       * to reject any entry or change record containing them.
1039       *
1040       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1041       *                                encountering attribute values which are not
1042       *                                base64-encoded but contain trailing spaces.
1043       */
1044      public void setTrailingSpaceBehavior(
1045                       final TrailingSpaceBehavior trailingSpaceBehavior)
1046      {
1047        this.trailingSpaceBehavior = trailingSpaceBehavior;
1048      }
1049    
1050    
1051    
1052      /**
1053       * Retrieves the base path that will be prepended to relative paths in order
1054       * to obtain an absolute path.  This will only be used for "file:" URLs that
1055       * have paths which do not begin with a slash.
1056       *
1057       * @return  The base path that will be prepended to relative paths in order to
1058       *          obtain an absolute path.
1059       */
1060      public String getRelativeBasePath()
1061      {
1062        return relativeBasePath;
1063      }
1064    
1065    
1066    
1067      /**
1068       * Specifies the base path that will be prepended to relative paths in order
1069       * to obtain an absolute path.  This will only be used for "file:" URLs that
1070       * have paths which do not begin with a space.
1071       *
1072       * @param  relativeBasePath  The base path that will be prepended to relative
1073       *                           paths in order to obtain an absolute path.
1074       */
1075      public void setRelativeBasePath(final String relativeBasePath)
1076      {
1077        setRelativeBasePath(new File(relativeBasePath));
1078      }
1079    
1080    
1081    
1082      /**
1083       * Specifies the base path that will be prepended to relative paths in order
1084       * to obtain an absolute path.  This will only be used for "file:" URLs that
1085       * have paths which do not begin with a space.
1086       *
1087       * @param  relativeBasePath  The base path that will be prepended to relative
1088       *                           paths in order to obtain an absolute path.
1089       */
1090      public void setRelativeBasePath(final File relativeBasePath)
1091      {
1092        final String path = relativeBasePath.getAbsolutePath();
1093        if (path.endsWith(File.separator))
1094        {
1095          this.relativeBasePath = path;
1096        }
1097        else
1098        {
1099          this.relativeBasePath = path + File.separator;
1100        }
1101      }
1102    
1103    
1104    
1105      /**
1106       * Retrieves the schema that will be used when reading LDIF records, if
1107       * defined.
1108       *
1109       * @return  The schema that will be used when reading LDIF records, or
1110       *          {@code null} if no schema should be used and all attributes should
1111       *          be treated as case-insensitive strings.
1112       */
1113      public Schema getSchema()
1114      {
1115        return schema;
1116      }
1117    
1118    
1119    
1120      /**
1121       * Specifies the schema that should be used when reading LDIF records.
1122       *
1123       * @param  schema  The schema that should be used when reading LDIF records,
1124       *                 or {@code null} if no schema should be used and all
1125       *                 attributes should be treated as case-insensitive strings.
1126       */
1127      public void setSchema(final Schema schema)
1128      {
1129        this.schema = schema;
1130      }
1131    
1132    
1133    
1134      /**
1135       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1136       * change record.
1137       *
1138       * @return  The record read from the LDIF source, or {@code null} if there are
1139       *          no more entries to be read.
1140       *
1141       * @throws  IOException  If a problem occurs while trying to read from the
1142       *                       LDIF source.
1143       *
1144       * @throws  LDIFException  If the data read could not be parsed as an entry or
1145       *                         an LDIF change record.
1146       */
1147      public LDIFRecord readLDIFRecord()
1148             throws IOException, LDIFException
1149      {
1150        if (isAsync())
1151        {
1152          return readLDIFRecordAsync();
1153        }
1154        else
1155        {
1156          return readLDIFRecordInternal();
1157        }
1158      }
1159    
1160    
1161    
1162      /**
1163       * Reads an entry from the LDIF source.
1164       *
1165       * @return  The entry read from the LDIF source, or {@code null} if there are
1166       *          no more entries to be read.
1167       *
1168       * @throws  IOException  If a problem occurs while attempting to read from the
1169       *                       LDIF source.
1170       *
1171       * @throws  LDIFException  If the data read could not be parsed as an entry.
1172       */
1173      public Entry readEntry()
1174             throws IOException, LDIFException
1175      {
1176        if (isAsync())
1177        {
1178          return readEntryAsync();
1179        }
1180        else
1181        {
1182          return readEntryInternal();
1183        }
1184      }
1185    
1186    
1187    
1188      /**
1189       * Reads an LDIF change record from the LDIF source.  The LDIF record must
1190       * have a changetype.
1191       *
1192       * @return  The change record read from the LDIF source, or {@code null} if
1193       *          there are no more records to be read.
1194       *
1195       * @throws  IOException  If a problem occurs while attempting to read from the
1196       *                       LDIF source.
1197       *
1198       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1199       *                         change record.
1200       */
1201      public LDIFChangeRecord readChangeRecord()
1202             throws IOException, LDIFException
1203      {
1204        return readChangeRecord(false);
1205      }
1206    
1207    
1208    
1209      /**
1210       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1211       * record does not have a changetype, then it may be assumed to be an add
1212       * change record.
1213       *
1214       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1215       *                     changetype should be retrieved as an add change record.
1216       *                     If this is {@code false} and the record read does not
1217       *                     include a changetype, then an {@code LDIFException}
1218       *                     will be thrown.
1219       *
1220       * @return  The change record read from the LDIF source, or {@code null} if
1221       *          there are no more records to be read.
1222       *
1223       * @throws  IOException  If a problem occurs while attempting to read from the
1224       *                       LDIF source.
1225       *
1226       * @throws  LDIFException  If the data read could not be parsed as an LDIF
1227       *                         change record.
1228       */
1229      public LDIFChangeRecord readChangeRecord(final boolean defaultAdd)
1230             throws IOException, LDIFException
1231      {
1232        if (isAsync())
1233        {
1234          return readChangeRecordAsync(defaultAdd);
1235        }
1236        else
1237        {
1238          return readChangeRecordInternal(defaultAdd);
1239        }
1240      }
1241    
1242    
1243    
1244      /**
1245       * Reads the next {@code LDIFRecord}, which was read and parsed by a different
1246       * thread.
1247       *
1248       * @return  The next parsed record or {@code null} if there are no more
1249       *          records to read.
1250       *
1251       * @throws IOException  If IOException was thrown when reading or parsing
1252       *                      the record.
1253       *
1254       * @throws LDIFException If LDIFException was thrown parsing the record.
1255       */
1256      private LDIFRecord readLDIFRecordAsync()
1257              throws IOException, LDIFException
1258      {
1259        final Result<UnparsedLDIFRecord, LDIFRecord> result =
1260             readLDIFRecordResultAsync();
1261        if (result == null)
1262        {
1263          return null;
1264        }
1265        else
1266        {
1267          return result.getOutput();
1268        }
1269      }
1270    
1271    
1272    
1273      /**
1274       * Reads an entry asynchronously from the LDIF source.
1275       *
1276       * @return The entry read from the LDIF source, or {@code null} if there are
1277       *         no more entries to be read.
1278       *
1279       * @throws IOException   If a problem occurs while attempting to read from the
1280       *                       LDIF source.
1281       * @throws LDIFException If the data read could not be parsed as an entry.
1282       */
1283      private Entry readEntryAsync()
1284              throws IOException, LDIFException
1285      {
1286        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1287        LDIFRecord record = null;
1288        while (record == null)
1289        {
1290          result = readLDIFRecordResultAsync();
1291          if (result == null)
1292          {
1293            return null;
1294          }
1295    
1296          record = result.getOutput();
1297    
1298          // This is a special value that means we should skip this Entry.  We have
1299          // to use something different than null because null means EOF.
1300          if (record == SKIP_ENTRY)
1301          {
1302            record = null;
1303          }
1304        }
1305    
1306        if (record instanceof Entry)
1307        {
1308          return (Entry) record;
1309        }
1310        else if (record instanceof LDIFChangeRecord)
1311        {
1312          try
1313          {
1314            // Some LDIFChangeRecord can be converted to an Entry.  This is really
1315            // an edge case though.
1316            return ((LDIFChangeRecord)record).toEntry();
1317          }
1318          catch (LDIFException e)
1319          {
1320            debugException(e);
1321            final long firstLineNumber = result.getInput().getFirstLineNumber();
1322            throw new LDIFException(e.getExceptionMessage(),
1323                                    firstLineNumber, true, e);
1324          }
1325        }
1326    
1327        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1328                                 "LDIFChangeRecord");
1329      }
1330    
1331    
1332    
1333      /**
1334       * Reads an LDIF change record from the LDIF source asynchronously.
1335       * Optionally, if the LDIF record does not have a changetype, then it may be
1336       * assumed to be an add change record.
1337       *
1338       * @param defaultAdd Indicates whether an LDIF record not containing a
1339       *                   changetype should be retrieved as an add change record.
1340       *                   If this is {@code false} and the record read does not
1341       *                   include a changetype, then an {@code LDIFException} will
1342       *                   be thrown.
1343       *
1344       * @return The change record read from the LDIF source, or {@code null} if
1345       *         there are no more records to be read.
1346       *
1347       * @throws IOException   If a problem occurs while attempting to read from the
1348       *                       LDIF source.
1349       * @throws LDIFException If the data read could not be parsed as an LDIF
1350       *                       change record.
1351       */
1352      private LDIFChangeRecord readChangeRecordAsync(final boolean defaultAdd)
1353              throws IOException, LDIFException
1354      {
1355        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1356        LDIFRecord record = null;
1357        while (record == null)
1358        {
1359          result = readLDIFRecordResultAsync();
1360          if (result == null)
1361          {
1362            return null;
1363          }
1364    
1365          record = result.getOutput();
1366    
1367          // This is a special value that means we should skip this Entry.  We have
1368          // to use something different than null because null means EOF.
1369          if (record == SKIP_ENTRY)
1370          {
1371            record = null;
1372          }
1373        }
1374    
1375        if (record instanceof LDIFChangeRecord)
1376        {
1377          return (LDIFChangeRecord) record;
1378        }
1379        else if (record instanceof Entry)
1380        {
1381          if (defaultAdd)
1382          {
1383            return new LDIFAddChangeRecord((Entry) record);
1384          }
1385          else
1386          {
1387            final long firstLineNumber = result.getInput().getFirstLineNumber();
1388            throw new LDIFException(
1389                 ERR_READ_NOT_CHANGE_RECORD.get(firstLineNumber), firstLineNumber,
1390                 true);
1391          }
1392        }
1393    
1394        throw new AssertionError("LDIFRecords must either be an Entry or an " +
1395                                 "LDIFChangeRecord");
1396      }
1397    
1398    
1399    
1400      /**
1401       * Reads the next LDIF record, which was read and parsed asynchronously by
1402       * separate threads.
1403       *
1404       * @return  The next LDIF record or {@code null} if there are no more records.
1405       *
1406       * @throws  IOException  If a problem occurs while attempting to read from the
1407       *                       LDIF source.
1408       *
1409       * @throws  LDIFException  If the data read could not be parsed as an entry.
1410       */
1411      private Result<UnparsedLDIFRecord, LDIFRecord> readLDIFRecordResultAsync()
1412              throws IOException, LDIFException
1413      {
1414        Result<UnparsedLDIFRecord, LDIFRecord> result = null;
1415    
1416        // If the asynchronous reading and parsing is complete, then we don't have
1417        // to block waiting for the next record to show up on the queue.  If there
1418        // isn't a record there, then return null (EOF) right away.
1419        if (asyncParsingComplete.get())
1420        {
1421          result = asyncParsedRecords.poll();
1422        }
1423        else
1424        {
1425          try
1426          {
1427            // We probably could just do a asyncParsedRecords.take() here, but
1428            // there are some edge case error scenarios where
1429            // asyncParsingComplete might be set without a special EOF sentinel
1430            // Result enqueued.  So to guard against this, we have a very cautious
1431            // polling interval of 1 second.  During normal processing, we never
1432            // have to wait for this to expire, when there is something to do
1433            // (like shutdown).
1434            while ((result == null) && (!asyncParsingComplete.get()))
1435            {
1436              result = asyncParsedRecords.poll(1, TimeUnit.SECONDS);
1437            }
1438    
1439            // There's a very small chance that we missed the value, so double-check
1440            if (result == null)
1441            {
1442              result = asyncParsedRecords.poll();
1443            }
1444          }
1445          catch (InterruptedException e)
1446          {
1447            debugException(e);
1448            throw createIOExceptionWithCause(null, e);
1449          }
1450        }
1451        if (result == null)
1452        {
1453          return null;
1454        }
1455    
1456        rethrow(result.getFailureCause());
1457    
1458        // Check if we reached the end of the input
1459        final UnparsedLDIFRecord unparsedRecord = result.getInput();
1460        if (unparsedRecord.isEOF())
1461        {
1462          // This might have been set already by the LineReaderThread, but
1463          // just in case it hasn't gotten to it yet, do so here.
1464          asyncParsingComplete.set(true);
1465    
1466          // Enqueue this EOF result again for any other thread that might be
1467          // blocked in asyncParsedRecords.take() even though having multiple
1468          // threads call this method concurrently breaks the contract of this
1469          // class.
1470          try
1471          {
1472            asyncParsedRecords.put(result);
1473          }
1474          catch (InterruptedException e)
1475          {
1476            // We shouldn't ever get interrupted because the put won't ever block.
1477            // Once we are done reading, this is the only item left in the queue,
1478            // so we should always be able to re-enqueue it.
1479            debugException(e);
1480          }
1481          return null;
1482        }
1483    
1484        return result;
1485      }
1486    
1487    
1488    
1489      /**
1490       * Indicates whether this LDIF reader was constructed to perform asynchronous
1491       * processing.
1492       *
1493       * @return  {@code true} if this LDIFReader was constructed to perform
1494       *          asynchronous processing, or {@code false} if not.
1495       */
1496      private boolean isAsync()
1497      {
1498        return isAsync;
1499      }
1500    
1501    
1502    
1503      /**
1504       * If not {@code null}, rethrows the specified Throwable as either an
1505       * IOException or LDIFException.
1506       *
1507       * @param t  The exception to rethrow.  If it's {@code null}, then nothing
1508       *           is thrown.
1509       *
1510       * @throws IOException   If t is an IOException or a checked Exception that
1511       *                       is not an LDIFException.
1512       * @throws LDIFException  If t is an LDIFException.
1513       */
1514      static void rethrow(final Throwable t)
1515             throws IOException, LDIFException
1516      {
1517        if (t == null)
1518        {
1519          return;
1520        }
1521    
1522        if (t instanceof IOException)
1523        {
1524          throw (IOException) t;
1525        }
1526        else if (t instanceof LDIFException)
1527        {
1528          throw (LDIFException) t;
1529        }
1530        else if (t instanceof RuntimeException)
1531        {
1532          throw (RuntimeException) t;
1533        }
1534        else if (t instanceof Error)
1535        {
1536          throw (Error) t;
1537        }
1538        else
1539        {
1540          throw createIOExceptionWithCause(null, t);
1541        }
1542      }
1543    
1544    
1545    
1546      /**
1547       * Reads a record from the LDIF source.  It may be either an entry or an LDIF
1548       * change record.
1549       *
1550       * @return The record read from the LDIF source, or {@code null} if there are
1551       *         no more entries to be read.
1552       *
1553       * @throws IOException   If a problem occurs while trying to read from the
1554       *                       LDIF source.
1555       * @throws LDIFException If the data read could not be parsed as an entry or
1556       *                       an LDIF change record.
1557       */
1558      private LDIFRecord readLDIFRecordInternal()
1559           throws IOException, LDIFException
1560      {
1561        final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1562        return decodeRecord(unparsedRecord, relativeBasePath, schema);
1563      }
1564    
1565    
1566    
1567      /**
1568       * Reads an entry from the LDIF source.
1569       *
1570       * @return The entry read from the LDIF source, or {@code null} if there are
1571       *         no more entries to be read.
1572       *
1573       * @throws IOException   If a problem occurs while attempting to read from the
1574       *                       LDIF source.
1575       * @throws LDIFException If the data read could not be parsed as an entry.
1576       */
1577      private Entry readEntryInternal()
1578           throws IOException, LDIFException
1579      {
1580        Entry e = null;
1581        while (e == null)
1582        {
1583          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1584          if (unparsedRecord.isEOF())
1585          {
1586            return null;
1587          }
1588    
1589          e = decodeEntry(unparsedRecord, relativeBasePath);
1590          debugLDIFRead(e);
1591    
1592          if (entryTranslator != null)
1593          {
1594            e = entryTranslator.translate(e, unparsedRecord.getFirstLineNumber());
1595          }
1596        }
1597        return e;
1598      }
1599    
1600    
1601    
1602      /**
1603       * Reads an LDIF change record from the LDIF source.  Optionally, if the LDIF
1604       * record does not have a changetype, then it may be assumed to be an add
1605       * change record.
1606       *
1607       * @param defaultAdd Indicates whether an LDIF record not containing a
1608       *                   changetype should be retrieved as an add change record.
1609       *                   If this is {@code false} and the record read does not
1610       *                   include a changetype, then an {@code LDIFException} will
1611       *                   be thrown.
1612       *
1613       * @return The change record read from the LDIF source, or {@code null} if
1614       *         there are no more records to be read.
1615       *
1616       * @throws IOException   If a problem occurs while attempting to read from the
1617       *                       LDIF source.
1618       * @throws LDIFException If the data read could not be parsed as an LDIF
1619       *                       change record.
1620       */
1621      private LDIFChangeRecord readChangeRecordInternal(final boolean defaultAdd)
1622           throws IOException, LDIFException
1623      {
1624        LDIFChangeRecord r = null;
1625        while (r == null)
1626        {
1627          final UnparsedLDIFRecord unparsedRecord = readUnparsedRecord();
1628          if (unparsedRecord.isEOF())
1629          {
1630            return null;
1631          }
1632    
1633          r = decodeChangeRecord(unparsedRecord, relativeBasePath, defaultAdd,
1634               schema);
1635          debugLDIFRead(r);
1636    
1637          if (changeRecordTranslator != null)
1638          {
1639            r = changeRecordTranslator.translate(r,
1640                 unparsedRecord.getFirstLineNumber());
1641          }
1642        }
1643        return r;
1644      }
1645    
1646    
1647    
1648      /**
1649       * Reads a record (either an entry or a change record) from the LDIF source
1650       * and places it in the line list.
1651       *
1652       * @return  The line number for the first line of the entry that was read.
1653       *
1654       * @throws  IOException  If a problem occurs while attempting to read from the
1655       *                       LDIF source.
1656       *
1657       * @throws  LDIFException  If the data read could not be parsed as a valid
1658       *                         LDIF record.
1659       */
1660      private UnparsedLDIFRecord readUnparsedRecord()
1661             throws IOException, LDIFException
1662      {
1663        final ArrayList<StringBuilder> lineList = new ArrayList<StringBuilder>(20);
1664        boolean lastWasComment = false;
1665        long firstLineNumber = lineNumberCounter + 1;
1666        while (true)
1667        {
1668          final String line = reader.readLine();
1669          lineNumberCounter++;
1670    
1671          if (line == null)
1672          {
1673            // We've hit the end of the LDIF source.  If we haven't read any entry
1674            // data, then return null.  Otherwise, the last entry wasn't followed by
1675            // a blank line, which is OK, and we should decode that entry.
1676            if (lineList.isEmpty())
1677            {
1678              return new UnparsedLDIFRecord(new ArrayList<StringBuilder>(0),
1679                   duplicateValueBehavior, trailingSpaceBehavior, schema, -1);
1680            }
1681            else
1682            {
1683              break;
1684            }
1685          }
1686    
1687          if (line.length() == 0)
1688          {
1689            // It's a blank line.  If we have read entry data, then this signals the
1690            // end of the entry.  Otherwise, it's an extra space between entries,
1691            // which is OK.
1692            lastWasComment = false;
1693            if (lineList.isEmpty())
1694            {
1695              firstLineNumber++;
1696              continue;
1697            }
1698            else
1699            {
1700              break;
1701            }
1702          }
1703    
1704          if (line.charAt(0) == ' ')
1705          {
1706            // The line starts with a space, which means that it must be a
1707            // continuation of the previous line.  This is true even if the last
1708            // line was a comment.
1709            if (lastWasComment)
1710            {
1711              // What we've read is part of a comment, so we don't care about its
1712              // content.
1713            }
1714            else if (lineList.isEmpty())
1715            {
1716              throw new LDIFException(
1717                             ERR_READ_UNEXPECTED_FIRST_SPACE.get(lineNumberCounter),
1718                             lineNumberCounter, false);
1719            }
1720            else
1721            {
1722              lineList.get(lineList.size() - 1).append(line.substring(1));
1723              lastWasComment = false;
1724            }
1725          }
1726          else if (line.charAt(0) == '#')
1727          {
1728            lastWasComment = true;
1729          }
1730          else
1731          {
1732            // We want to make sure that we skip over the "version:" line if it
1733            // exists, but that should only occur at the beginning of an entry where
1734            // it can't be confused with a possible "version" attribute.
1735            if (lineList.isEmpty() && line.startsWith("version:"))
1736            {
1737              lastWasComment = true;
1738            }
1739            else
1740            {
1741              lineList.add(new StringBuilder(line));
1742              lastWasComment = false;
1743            }
1744          }
1745        }
1746    
1747        return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
1748             trailingSpaceBehavior, schema, firstLineNumber);
1749      }
1750    
1751    
1752    
1753      /**
1754       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1755       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1756       * LDIF specification, and it is acceptable to have one or more blank lines
1757       * following the entry. A default trailing space behavior of
1758       * {@code TrailingSpaceBehavior#REJECT} will be used.
1759       *
1760       * @param  ldifLines  The set of lines that comprise the LDIF representation
1761       *                    of the entry.  It must not be {@code null} or empty.
1762       *
1763       * @return  The entry read from LDIF.
1764       *
1765       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1766       *                         entry.
1767       */
1768      public static Entry decodeEntry(final String... ldifLines)
1769             throws LDIFException
1770      {
1771        final Entry e = decodeEntry(prepareRecord(DuplicateValueBehavior.STRIP,
1772             TrailingSpaceBehavior.REJECT, null, ldifLines),
1773             DEFAULT_RELATIVE_BASE_PATH);
1774        debugLDIFRead(e);
1775        return e;
1776      }
1777    
1778    
1779    
1780      /**
1781       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1782       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1783       * LDIF specification, and it is acceptable to have one or more blank lines
1784       * following the entry. A default trailing space behavior of
1785       * {@code TrailingSpaceBehavior#REJECT} will be used.
1786       *
1787       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1788       *                                attribute values encountered while parsing.
1789       * @param  schema                 The schema to use when parsing the record,
1790       *                                if applicable.
1791       * @param  ldifLines              The set of lines that comprise the LDIF
1792       *                                representation of the entry.  It must not be
1793       *                                {@code null} or empty.
1794       *
1795       * @return  The entry read from LDIF.
1796       *
1797       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1798       *                         entry.
1799       */
1800      public static Entry decodeEntry(final boolean ignoreDuplicateValues,
1801                                      final Schema schema,
1802                                      final String... ldifLines)
1803             throws LDIFException
1804      {
1805        return decodeEntry(ignoreDuplicateValues, TrailingSpaceBehavior.REJECT,
1806             schema, ldifLines);
1807      }
1808    
1809    
1810    
1811      /**
1812       * Decodes the provided set of LDIF lines as an entry.  The provided set of
1813       * lines must contain exactly one entry.  Long lines may be wrapped as per the
1814       * LDIF specification, and it is acceptable to have one or more blank lines
1815       * following the entry.
1816       *
1817       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1818       *                                attribute values encountered while parsing.
1819       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1820       *                                encountering attribute values which are not
1821       *                                base64-encoded but contain trailing spaces.
1822       *                                It must not be {@code null}.
1823       * @param  schema                 The schema to use when parsing the record,
1824       *                                if applicable.
1825       * @param  ldifLines              The set of lines that comprise the LDIF
1826       *                                representation of the entry.  It must not be
1827       *                                {@code null} or empty.
1828       *
1829       * @return  The entry read from LDIF.
1830       *
1831       * @throws  LDIFException  If the provided LDIF data cannot be decoded as an
1832       *                         entry.
1833       */
1834      public static Entry decodeEntry(
1835             final boolean ignoreDuplicateValues,
1836             final TrailingSpaceBehavior trailingSpaceBehavior,
1837             final Schema schema,
1838             final String... ldifLines) throws LDIFException
1839      {
1840        final Entry e = decodeEntry(prepareRecord(
1841                  (ignoreDuplicateValues
1842                       ? DuplicateValueBehavior.STRIP
1843                       : DuplicateValueBehavior.REJECT),
1844             trailingSpaceBehavior, schema, ldifLines),
1845             DEFAULT_RELATIVE_BASE_PATH);
1846        debugLDIFRead(e);
1847        return e;
1848      }
1849    
1850    
1851    
1852      /**
1853       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1854       * provided set of lines must contain exactly one change record and it must
1855       * include a changetype.  Long lines may be wrapped as per the LDIF
1856       * specification, and it is acceptable to have one or more blank lines
1857       * following the entry.
1858       *
1859       * @param  ldifLines  The set of lines that comprise the LDIF representation
1860       *                    of the change record.  It must not be {@code null} or
1861       *                    empty.
1862       *
1863       * @return  The change record read from LDIF.
1864       *
1865       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1866       *                         change record.
1867       */
1868      public static LDIFChangeRecord decodeChangeRecord(final String... ldifLines)
1869             throws LDIFException
1870      {
1871        return decodeChangeRecord(false, ldifLines);
1872      }
1873    
1874    
1875    
1876      /**
1877       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1878       * provided set of lines must contain exactly one change record.  Long lines
1879       * may be wrapped as per the LDIF specification, and it is acceptable to have
1880       * one or more blank lines following the entry.
1881       *
1882       * @param  defaultAdd  Indicates whether an LDIF record not containing a
1883       *                     changetype should be retrieved as an add change record.
1884       *                     If this is {@code false} and the record read does not
1885       *                     include a changetype, then an {@code LDIFException}
1886       *                     will be thrown.
1887       * @param  ldifLines  The set of lines that comprise the LDIF representation
1888       *                    of the change record.  It must not be {@code null} or
1889       *                    empty.
1890       *
1891       * @return  The change record read from LDIF.
1892       *
1893       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1894       *                         change record.
1895       */
1896      public static LDIFChangeRecord decodeChangeRecord(final boolean defaultAdd,
1897                                                        final String... ldifLines)
1898             throws LDIFException
1899      {
1900        final LDIFChangeRecord r =
1901             decodeChangeRecord(
1902                  prepareRecord(DuplicateValueBehavior.STRIP,
1903                       TrailingSpaceBehavior.REJECT, null, ldifLines),
1904                  DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
1905        debugLDIFRead(r);
1906        return r;
1907      }
1908    
1909    
1910    
1911      /**
1912       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1913       * provided set of lines must contain exactly one change record.  Long lines
1914       * may be wrapped as per the LDIF specification, and it is acceptable to have
1915       * one or more blank lines following the entry.
1916       *
1917       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1918       *                                attribute values encountered while parsing.
1919       * @param  schema                 The schema to use when processing the change
1920       *                                record, or {@code null} if no schema should
1921       *                                be used and all values should be treated as
1922       *                                case-insensitive strings.
1923       * @param  defaultAdd             Indicates whether an LDIF record not
1924       *                                containing a changetype should be retrieved
1925       *                                as an add change record.  If this is
1926       *                                {@code false} and the record read does not
1927       *                                include a changetype, then an
1928       *                                {@code LDIFException} will be thrown.
1929       * @param  ldifLines              The set of lines that comprise the LDIF
1930       *                                representation of the change record.  It
1931       *                                must not be {@code null} or empty.
1932       *
1933       * @return  The change record read from LDIF.
1934       *
1935       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1936       *                         change record.
1937       */
1938      public static LDIFChangeRecord decodeChangeRecord(
1939                                          final boolean ignoreDuplicateValues,
1940                                          final Schema schema,
1941                                          final boolean defaultAdd,
1942                                          final String... ldifLines)
1943             throws LDIFException
1944      {
1945        return decodeChangeRecord(ignoreDuplicateValues,
1946             TrailingSpaceBehavior.REJECT, schema, defaultAdd, ldifLines);
1947      }
1948    
1949    
1950    
1951      /**
1952       * Decodes the provided set of LDIF lines as an LDIF change record.  The
1953       * provided set of lines must contain exactly one change record.  Long lines
1954       * may be wrapped as per the LDIF specification, and it is acceptable to have
1955       * one or more blank lines following the entry.
1956       *
1957       * @param  ignoreDuplicateValues  Indicates whether to ignore duplicate
1958       *                                attribute values encountered while parsing.
1959       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
1960       *                                encountering attribute values which are not
1961       *                                base64-encoded but contain trailing spaces.
1962       *                                It must not be {@code null}.
1963       * @param  schema                 The schema to use when processing the change
1964       *                                record, or {@code null} if no schema should
1965       *                                be used and all values should be treated as
1966       *                                case-insensitive strings.
1967       * @param  defaultAdd             Indicates whether an LDIF record not
1968       *                                containing a changetype should be retrieved
1969       *                                as an add change record.  If this is
1970       *                                {@code false} and the record read does not
1971       *                                include a changetype, then an
1972       *                                {@code LDIFException} will be thrown.
1973       * @param  ldifLines              The set of lines that comprise the LDIF
1974       *                                representation of the change record.  It
1975       *                                must not be {@code null} or empty.
1976       *
1977       * @return  The change record read from LDIF.
1978       *
1979       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
1980       *                         change record.
1981       */
1982      public static LDIFChangeRecord decodeChangeRecord(
1983                         final boolean ignoreDuplicateValues,
1984                         final TrailingSpaceBehavior trailingSpaceBehavior,
1985                         final Schema schema,
1986                         final boolean defaultAdd,
1987                         final String... ldifLines)
1988             throws LDIFException
1989      {
1990        final LDIFChangeRecord r = decodeChangeRecord(
1991             prepareRecord(
1992                  (ignoreDuplicateValues
1993                       ? DuplicateValueBehavior.STRIP
1994                       : DuplicateValueBehavior.REJECT),
1995                  trailingSpaceBehavior, schema, ldifLines),
1996             DEFAULT_RELATIVE_BASE_PATH, defaultAdd, null);
1997        debugLDIFRead(r);
1998        return r;
1999      }
2000    
2001    
2002    
2003      /**
2004       * Parses the provided set of lines into a list of {@code StringBuilder}
2005       * objects suitable for decoding into an entry or LDIF change record.
2006       * Comments will be ignored and wrapped lines will be unwrapped.
2007       *
2008       * @param  duplicateValueBehavior  The behavior that should be exhibited if
2009       *                                 the LDIF reader encounters an entry with
2010       *                                 duplicate values.
2011       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2012       *                                 encountering attribute values which are not
2013       *                                 base64-encoded but contain trailing spaces.
2014       * @param  schema                  The schema to use when parsing the record,
2015       *                                 if applicable.
2016       * @param  ldifLines               The set of lines that comprise the record
2017       *                                 to decode.  It must not be {@code null} or
2018       *                                 empty.
2019       *
2020       * @return  The prepared list of {@code StringBuilder} objects ready to be
2021       *          decoded.
2022       *
2023       * @throws  LDIFException  If the provided lines do not contain valid LDIF
2024       *                         content.
2025       */
2026      private static UnparsedLDIFRecord prepareRecord(
2027                          final DuplicateValueBehavior duplicateValueBehavior,
2028                          final TrailingSpaceBehavior trailingSpaceBehavior,
2029                          final Schema schema, final String... ldifLines)
2030              throws LDIFException
2031      {
2032        ensureNotNull(ldifLines);
2033        ensureFalse(ldifLines.length == 0,
2034                    "LDIFReader.prepareRecord.ldifLines must not be empty.");
2035    
2036        boolean lastWasComment = false;
2037        final ArrayList<StringBuilder> lineList =
2038             new ArrayList<StringBuilder>(ldifLines.length);
2039        for (int i=0; i < ldifLines.length; i++)
2040        {
2041          final String line = ldifLines[i];
2042          if (line.length() == 0)
2043          {
2044            // This is only acceptable if there are no more non-empty lines in the
2045            // array.
2046            for (int j=i+1; j < ldifLines.length; j++)
2047            {
2048              if (ldifLines[j].length() > 0)
2049              {
2050                throw new LDIFException(ERR_READ_UNEXPECTED_BLANK.get(i), i, true,
2051                                        ldifLines, null);
2052              }
2053    
2054              // If we've gotten here, then we know that we're at the end of the
2055              // entry.  If we have read data, then we can decode it as an entry.
2056              // Otherwise, there was no real data in the provided LDIF lines.
2057              if (lineList.isEmpty())
2058              {
2059                throw new LDIFException(ERR_READ_ONLY_BLANKS.get(), 0, true,
2060                                        ldifLines, null);
2061              }
2062              else
2063              {
2064                return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2065                     trailingSpaceBehavior, schema, 0);
2066              }
2067            }
2068          }
2069    
2070          if (line.charAt(0) == ' ')
2071          {
2072            if (i > 0)
2073            {
2074              if (! lastWasComment)
2075              {
2076                lineList.get(lineList.size() - 1).append(line.substring(1));
2077              }
2078            }
2079            else
2080            {
2081              throw new LDIFException(
2082                             ERR_READ_UNEXPECTED_FIRST_SPACE_NO_NUMBER.get(), 0,
2083                             true, ldifLines, null);
2084            }
2085          }
2086          else if (line.charAt(0) == '#')
2087          {
2088            lastWasComment = true;
2089          }
2090          else
2091          {
2092            lineList.add(new StringBuilder(line));
2093            lastWasComment = false;
2094          }
2095        }
2096    
2097        if (lineList.isEmpty())
2098        {
2099          throw new LDIFException(ERR_READ_NO_DATA.get(), 0, true, ldifLines, null);
2100        }
2101        else
2102        {
2103          return new UnparsedLDIFRecord(lineList, duplicateValueBehavior,
2104               trailingSpaceBehavior, schema, 0);
2105        }
2106      }
2107    
2108    
2109    
2110      /**
2111       * Decodes the unparsed record that was read from the LDIF source.  It may be
2112       * either an entry or an LDIF change record.
2113       *
2114       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2115       *                           input.  It must not be {@code null} or empty.
2116       * @param  relativeBasePath  The base path that will be prepended to relative
2117       *                           paths in order to obtain an absolute path.
2118       * @param  schema            The schema to use when parsing.
2119       *
2120       * @return  The parsed record, or {@code null} if there are no more entries to
2121       *          be read.
2122       *
2123       * @throws  LDIFException  If the data read could not be parsed as an entry or
2124       *                         an LDIF change record.
2125       */
2126      private static LDIFRecord decodeRecord(
2127                                     final UnparsedLDIFRecord unparsedRecord,
2128                                     final String relativeBasePath,
2129                                     final Schema schema)
2130           throws LDIFException
2131      {
2132        // If there was an error reading from the input, then we rethrow it here.
2133        final Exception readError = unparsedRecord.getFailureCause();
2134        if (readError != null)
2135        {
2136          if (readError instanceof LDIFException)
2137          {
2138            // If the error was an LDIFException, which will normally be the case,
2139            // then rethrow it with all of the same state.  We could just
2140            //   throw (LDIFException) readError;
2141            // but that's considered bad form.
2142            final LDIFException ldifEx = (LDIFException) readError;
2143            throw new LDIFException(ldifEx.getMessage(),
2144                                    ldifEx.getLineNumber(),
2145                                    ldifEx.mayContinueReading(),
2146                                    ldifEx.getDataLines(),
2147                                    ldifEx.getCause());
2148          }
2149          else
2150          {
2151            throw new LDIFException(getExceptionMessage(readError),
2152                                    -1, true, readError);
2153          }
2154        }
2155    
2156        if (unparsedRecord.isEOF())
2157        {
2158          return null;
2159        }
2160    
2161        final ArrayList<StringBuilder> lineList = unparsedRecord.getLineList();
2162        if (unparsedRecord.getLineList() == null)
2163        {
2164          return null;  // We can get here if there was an error reading the lines.
2165        }
2166    
2167        final LDIFRecord r;
2168        if (lineList.size() == 1)
2169        {
2170          r = decodeEntry(unparsedRecord, relativeBasePath);
2171        }
2172        else
2173        {
2174          final String lowerSecondLine = toLowerCase(lineList.get(1).toString());
2175          if (lowerSecondLine.startsWith("control:") ||
2176              lowerSecondLine.startsWith("changetype:"))
2177          {
2178            r = decodeChangeRecord(unparsedRecord, relativeBasePath, true, schema);
2179          }
2180          else
2181          {
2182            r = decodeEntry(unparsedRecord, relativeBasePath);
2183          }
2184        }
2185    
2186        debugLDIFRead(r);
2187        return r;
2188      }
2189    
2190    
2191    
2192      /**
2193       * Decodes the provided set of LDIF lines as an entry.  The provided list must
2194       * not contain any blank lines or comments, and lines are not allowed to be
2195       * wrapped.
2196       *
2197       * @param  unparsedRecord   The unparsed LDIF record that was read from the
2198       *                          input.  It must not be {@code null} or empty.
2199       * @param  relativeBasePath  The base path that will be prepended to relative
2200       *                           paths in order to obtain an absolute path.
2201       *
2202       * @return  The entry read from LDIF.
2203       *
2204       * @throws  LDIFException  If the provided LDIF data cannot be read as an
2205       *                         entry.
2206       */
2207      private static Entry decodeEntry(final UnparsedLDIFRecord unparsedRecord,
2208                                       final String relativeBasePath)
2209              throws LDIFException
2210      {
2211        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2212        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2213    
2214        final Iterator<StringBuilder> iterator = ldifLines.iterator();
2215    
2216        // The first line must start with either "version:" or "dn:".  If the first
2217        // line starts with "version:" then the second must start with "dn:".
2218        StringBuilder line = iterator.next();
2219        handleTrailingSpaces(line, null, firstLineNumber,
2220             unparsedRecord.getTrailingSpaceBehavior());
2221        int colonPos = line.indexOf(":");
2222        if ((colonPos > 0) &&
2223            line.substring(0, colonPos).equalsIgnoreCase("version"))
2224        {
2225          // The first line is "version:".  Under most conditions, this will be
2226          // handled by the LDIF reader, but this can happen if you call
2227          // decodeEntry with a set of data that includes a version.  At any rate,
2228          // read the next line, which must specify the DN.
2229          line = iterator.next();
2230          handleTrailingSpaces(line, null, firstLineNumber,
2231               unparsedRecord.getTrailingSpaceBehavior());
2232        }
2233    
2234        colonPos = line.indexOf(":");
2235        if ((colonPos < 0) ||
2236             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2237        {
2238          throw new LDIFException(
2239               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2240               firstLineNumber, true, ldifLines, null);
2241        }
2242    
2243        final String dn;
2244        final int length = line.length();
2245        if (length == (colonPos+1))
2246        {
2247          // The colon was the last character on the line.  This is acceptable and
2248          // indicates that the entry has the null DN.
2249          dn = "";
2250        }
2251        else if (line.charAt(colonPos+1) == ':')
2252        {
2253          // Skip over any spaces leading up to the value, and then the rest of the
2254          // string is the base64-encoded DN.
2255          int pos = colonPos+2;
2256          while ((pos < length) && (line.charAt(pos) == ' '))
2257          {
2258            pos++;
2259          }
2260    
2261          try
2262          {
2263            final byte[] dnBytes = Base64.decode(line.substring(pos));
2264            dn = new String(dnBytes, "UTF-8");
2265          }
2266          catch (final ParseException pe)
2267          {
2268            debugException(pe);
2269            throw new LDIFException(
2270                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2271                                                                pe.getMessage()),
2272                           firstLineNumber, true, ldifLines, pe);
2273          }
2274          catch (final Exception e)
2275          {
2276            debugException(e);
2277            throw new LDIFException(
2278                           ERR_READ_CANNOT_BASE64_DECODE_DN.get(firstLineNumber, e),
2279                           firstLineNumber, true, ldifLines, e);
2280          }
2281        }
2282        else
2283        {
2284          // Skip over any spaces leading up to the value, and then the rest of the
2285          // string is the DN.
2286          int pos = colonPos+1;
2287          while ((pos < length) && (line.charAt(pos) == ' '))
2288          {
2289            pos++;
2290          }
2291    
2292          dn = line.substring(pos);
2293        }
2294    
2295    
2296        // The remaining lines must be the attributes for the entry.  However, we
2297        // will allow the case in which an entry does not have any attributes, to be
2298        // able to support reading search result entries in which no attributes were
2299        // returned.
2300        if (! iterator.hasNext())
2301        {
2302          return new Entry(dn, unparsedRecord.getSchema());
2303        }
2304    
2305        return new Entry(dn, unparsedRecord.getSchema(),
2306             parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2307                  unparsedRecord.getTrailingSpaceBehavior(),
2308                  unparsedRecord.getSchema(), ldifLines, iterator, relativeBasePath,
2309                  firstLineNumber));
2310      }
2311    
2312    
2313    
2314      /**
2315       * Decodes the provided set of LDIF lines as a change record.  The provided
2316       * list must not contain any blank lines or comments, and lines are not
2317       * allowed to be wrapped.
2318       *
2319       * @param  unparsedRecord    The unparsed LDIF record that was read from the
2320       *                           input.  It must not be {@code null} or empty.
2321       * @param  relativeBasePath  The base path that will be prepended to relative
2322       *                           paths in order to obtain an absolute path.
2323       * @param  defaultAdd        Indicates whether an LDIF record not containing a
2324       *                           changetype should be retrieved as an add change
2325       *                           record.  If this is {@code false} and the record
2326       *                           read does not include a changetype, then an
2327       *                           {@code LDIFException} will be thrown.
2328       * @param  schema            The schema to use in parsing.
2329       *
2330       * @return  The change record read from LDIF.
2331       *
2332       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2333       *                         change record.
2334       */
2335      private static LDIFChangeRecord decodeChangeRecord(
2336                                           final UnparsedLDIFRecord unparsedRecord,
2337                                           final String relativeBasePath,
2338                                           final boolean defaultAdd,
2339                                           final Schema schema)
2340              throws LDIFException
2341      {
2342        final ArrayList<StringBuilder> ldifLines = unparsedRecord.getLineList();
2343        final long firstLineNumber = unparsedRecord.getFirstLineNumber();
2344    
2345        Iterator<StringBuilder> iterator = ldifLines.iterator();
2346    
2347        // The first line must start with either "version:" or "dn:".  If the first
2348        // line starts with "version:" then the second must start with "dn:".
2349        StringBuilder line = iterator.next();
2350        handleTrailingSpaces(line, null, firstLineNumber,
2351             unparsedRecord.getTrailingSpaceBehavior());
2352        int colonPos = line.indexOf(":");
2353        int linesRead = 1;
2354        if ((colonPos > 0) &&
2355            line.substring(0, colonPos).equalsIgnoreCase("version"))
2356        {
2357          // The first line is "version:".  Under most conditions, this will be
2358          // handled by the LDIF reader, but this can happen if you call
2359          // decodeEntry with a set of data that includes a version.  At any rate,
2360          // read the next line, which must specify the DN.
2361          line = iterator.next();
2362          linesRead++;
2363          handleTrailingSpaces(line, null, firstLineNumber,
2364               unparsedRecord.getTrailingSpaceBehavior());
2365        }
2366    
2367        colonPos = line.indexOf(":");
2368        if ((colonPos < 0) ||
2369             (! line.substring(0, colonPos).equalsIgnoreCase("dn")))
2370        {
2371          throw new LDIFException(
2372               ERR_READ_DN_LINE_DOESNT_START_WITH_DN.get(firstLineNumber),
2373               firstLineNumber, true, ldifLines, null);
2374        }
2375    
2376        final String dn;
2377        int length = line.length();
2378        if (length == (colonPos+1))
2379        {
2380          // The colon was the last character on the line.  This is acceptable and
2381          // indicates that the entry has the null DN.
2382          dn = "";
2383        }
2384        else if (line.charAt(colonPos+1) == ':')
2385        {
2386          // Skip over any spaces leading up to the value, and then the rest of the
2387          // string is the base64-encoded DN.
2388          int pos = colonPos+2;
2389          while ((pos < length) && (line.charAt(pos) == ' '))
2390          {
2391            pos++;
2392          }
2393    
2394          try
2395          {
2396            final byte[] dnBytes = Base64.decode(line.substring(pos));
2397            dn = new String(dnBytes, "UTF-8");
2398          }
2399          catch (final ParseException pe)
2400          {
2401            debugException(pe);
2402            throw new LDIFException(
2403                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2404                                                                   pe.getMessage()),
2405                           firstLineNumber, true, ldifLines, pe);
2406          }
2407          catch (final Exception e)
2408          {
2409            debugException(e);
2410            throw new LDIFException(
2411                           ERR_READ_CR_CANNOT_BASE64_DECODE_DN.get(firstLineNumber,
2412                                                                   e),
2413                           firstLineNumber, true, ldifLines, e);
2414          }
2415        }
2416        else
2417        {
2418          // Skip over any spaces leading up to the value, and then the rest of the
2419          // string is the DN.
2420          int pos = colonPos+1;
2421          while ((pos < length) && (line.charAt(pos) == ' '))
2422          {
2423            pos++;
2424          }
2425    
2426          dn = line.substring(pos);
2427        }
2428    
2429    
2430        // An LDIF change record may contain zero or more controls, with the end of
2431        // the controls signified by the changetype.  The changetype element must be
2432        // present, unless defaultAdd is true in which case the first thing that is
2433        // neither control or changetype will trigger the start of add attribute
2434        // parsing.
2435        if (! iterator.hasNext())
2436        {
2437          throw new LDIFException(ERR_READ_CR_TOO_SHORT.get(firstLineNumber),
2438                                  firstLineNumber, true, ldifLines, null);
2439        }
2440    
2441        String changeType = null;
2442        ArrayList<Control> controls = null;
2443        while (true)
2444        {
2445          line = iterator.next();
2446          handleTrailingSpaces(line, dn, firstLineNumber,
2447               unparsedRecord.getTrailingSpaceBehavior());
2448          colonPos = line.indexOf(":");
2449          if (colonPos < 0)
2450          {
2451            throw new LDIFException(
2452                 ERR_READ_CR_SECOND_LINE_MISSING_COLON.get(firstLineNumber),
2453                 firstLineNumber, true, ldifLines, null);
2454          }
2455    
2456          final String token = toLowerCase(line.substring(0, colonPos));
2457          if (token.equals("control"))
2458          {
2459            if (controls == null)
2460            {
2461              controls = new ArrayList<Control>(5);
2462            }
2463    
2464            controls.add(decodeControl(line, colonPos, firstLineNumber, ldifLines,
2465                 relativeBasePath));
2466          }
2467          else if (token.equals("changetype"))
2468          {
2469            changeType =
2470                 decodeChangeType(line, colonPos, firstLineNumber, ldifLines);
2471            break;
2472          }
2473          else if (defaultAdd)
2474          {
2475            // The line we read wasn't a control or changetype declaration, so we'll
2476            // assume it's an attribute in an add record.  However, we're not ready
2477            // for that yet, and since we can't rewind an iterator we'll create a
2478            // new one that hasn't yet gotten to this line.
2479            changeType = "add";
2480            iterator = ldifLines.iterator();
2481            for (int i=0; i < linesRead; i++)
2482            {
2483              iterator.next();
2484            }
2485            break;
2486          }
2487          else
2488          {
2489            throw new LDIFException(
2490                 ERR_READ_CR_CT_LINE_DOESNT_START_WITH_CONTROL_OR_CT.get(
2491                      firstLineNumber),
2492                 firstLineNumber, true, ldifLines, null);
2493          }
2494    
2495          linesRead++;
2496        }
2497    
2498    
2499        // Make sure that the change type is acceptable and then decode the rest of
2500        // the change record accordingly.
2501        final String lowerChangeType = toLowerCase(changeType);
2502        if (lowerChangeType.equals("add"))
2503        {
2504          // There must be at least one more line.  If not, then that's an error.
2505          // Otherwise, parse the rest of the data as attribute-value pairs.
2506          if (iterator.hasNext())
2507          {
2508            final Collection<Attribute> attrs =
2509                 parseAttributes(dn, unparsedRecord.getDuplicateValueBehavior(),
2510                      unparsedRecord.getTrailingSpaceBehavior(),
2511                      unparsedRecord.getSchema(), ldifLines, iterator,
2512                      relativeBasePath, firstLineNumber);
2513            final Attribute[] attributes = new Attribute[attrs.size()];
2514            final Iterator<Attribute> attrIterator = attrs.iterator();
2515            for (int i=0; i < attributes.length; i++)
2516            {
2517              attributes[i] = attrIterator.next();
2518            }
2519    
2520            return new LDIFAddChangeRecord(dn, attributes, controls);
2521          }
2522          else
2523          {
2524            throw new LDIFException(ERR_READ_CR_NO_ATTRIBUTES.get(firstLineNumber),
2525                                    firstLineNumber, true, ldifLines, null);
2526          }
2527        }
2528        else if (lowerChangeType.equals("delete"))
2529        {
2530          // There shouldn't be any more data.  If there is, then that's an error.
2531          // Otherwise, we can just return the delete change record with what we
2532          // already know.
2533          if (iterator.hasNext())
2534          {
2535            throw new LDIFException(
2536                           ERR_READ_CR_EXTRA_DELETE_DATA.get(firstLineNumber),
2537                           firstLineNumber, true, ldifLines, null);
2538          }
2539          else
2540          {
2541            return new LDIFDeleteChangeRecord(dn, controls);
2542          }
2543        }
2544        else if (lowerChangeType.equals("modify"))
2545        {
2546          // There must be at least one more line.  If not, then that's an error.
2547          // Otherwise, parse the rest of the data as a set of modifications.
2548          if (iterator.hasNext())
2549          {
2550            final Modification[] mods = parseModifications(dn,
2551                 unparsedRecord.getTrailingSpaceBehavior(), ldifLines, iterator,
2552                 firstLineNumber, schema);
2553            return new LDIFModifyChangeRecord(dn, mods, controls);
2554          }
2555          else
2556          {
2557            throw new LDIFException(ERR_READ_CR_NO_MODS.get(firstLineNumber),
2558                                    firstLineNumber, true, ldifLines, null);
2559          }
2560        }
2561        else if (lowerChangeType.equals("moddn") ||
2562                 lowerChangeType.equals("modrdn"))
2563        {
2564          // There must be at least one more line.  If not, then that's an error.
2565          // Otherwise, parse the rest of the data as a set of modifications.
2566          if (iterator.hasNext())
2567          {
2568            return parseModifyDNChangeRecord(ldifLines, iterator, dn, controls,
2569                 unparsedRecord.getTrailingSpaceBehavior(), firstLineNumber);
2570          }
2571          else
2572          {
2573            throw new LDIFException(ERR_READ_CR_NO_NEWRDN.get(firstLineNumber),
2574                                    firstLineNumber, true, ldifLines, null);
2575          }
2576        }
2577        else
2578        {
2579          throw new LDIFException(ERR_READ_CR_INVALID_CT.get(changeType,
2580                                                             firstLineNumber),
2581                                  firstLineNumber, true, ldifLines, null);
2582        }
2583      }
2584    
2585    
2586    
2587      /**
2588       * Decodes information about a control from the provided line.
2589       *
2590       * @param  line              The line to process.
2591       * @param  colonPos          The position of the colon that separates the
2592       *                           control token string from tbe encoded control.
2593       * @param  firstLineNumber   The line number for the start of the record.
2594       * @param  ldifLines         The lines that comprise the LDIF representation
2595       *                           of the full record being parsed.
2596       * @param  relativeBasePath  The base path that will be prepended to relative
2597       *                           paths in order to obtain an absolute path.
2598       *
2599       * @return  The decoded control.
2600       *
2601       * @throws  LDIFException  If a problem is encountered while trying to decode
2602       *                         the changetype.
2603       */
2604      private static Control decodeControl(final StringBuilder line,
2605                                           final int colonPos,
2606                                           final long firstLineNumber,
2607                                           final ArrayList<StringBuilder> ldifLines,
2608                                           final String relativeBasePath)
2609              throws LDIFException
2610      {
2611        final String controlString;
2612        int length = line.length();
2613        if (length == (colonPos+1))
2614        {
2615          // The colon was the last character on the line.  This is not
2616          // acceptable.
2617          throw new LDIFException(
2618               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2619               firstLineNumber, true, ldifLines, null);
2620        }
2621        else if (line.charAt(colonPos+1) == ':')
2622        {
2623          // Skip over any spaces leading up to the value, and then the rest of
2624          // the string is the base64-encoded control representation.  This is
2625          // unusual and unnecessary, but is nevertheless acceptable.
2626          int pos = colonPos+2;
2627          while ((pos < length) && (line.charAt(pos) == ' '))
2628          {
2629            pos++;
2630          }
2631    
2632          try
2633          {
2634            final byte[] controlBytes = Base64.decode(line.substring(pos));
2635            controlString =  new String(controlBytes, "UTF-8");
2636          }
2637          catch (final ParseException pe)
2638          {
2639            debugException(pe);
2640            throw new LDIFException(
2641                           ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(
2642                                firstLineNumber, pe.getMessage()),
2643                           firstLineNumber, true, ldifLines, pe);
2644          }
2645          catch (final Exception e)
2646          {
2647            debugException(e);
2648            throw new LDIFException(
2649                 ERR_READ_CANNOT_BASE64_DECODE_CONTROL.get(firstLineNumber, e),
2650                 firstLineNumber, true, ldifLines, e);
2651          }
2652        }
2653        else
2654        {
2655          // Skip over any spaces leading up to the value, and then the rest of
2656          // the string is the encoded control.
2657          int pos = colonPos+1;
2658          while ((pos < length) && (line.charAt(pos) == ' '))
2659          {
2660            pos++;
2661          }
2662    
2663          controlString = line.substring(pos);
2664        }
2665    
2666        // If the resulting control definition is empty, then that's invalid.
2667        if (controlString.length() == 0)
2668        {
2669          throw new LDIFException(
2670               ERR_READ_CONTROL_LINE_NO_CONTROL_VALUE.get(firstLineNumber),
2671               firstLineNumber, true, ldifLines, null);
2672        }
2673    
2674    
2675        // The first element of the control must be the OID, and it must be followed
2676        // by a space (to separate it from the criticality), a colon (to separate it
2677        // from the value and indicate a default criticality of false), or the end
2678        // of the line (to indicate a default criticality of false and no value).
2679        String oid = null;
2680        boolean hasCriticality = false;
2681        boolean hasValue = false;
2682        int pos = 0;
2683        length = controlString.length();
2684        while (pos < length)
2685        {
2686          final char c = controlString.charAt(pos);
2687          if (c == ':')
2688          {
2689            // This indicates that there is no criticality and that the value
2690            // immediately follows the OID.
2691            oid = controlString.substring(0, pos++);
2692            hasValue = true;
2693            break;
2694          }
2695          else if (c == ' ')
2696          {
2697            // This indicates that there is a criticality.  We don't know anything
2698            // about the presence of a value yet.
2699            oid = controlString.substring(0, pos++);
2700            hasCriticality = true;
2701            break;
2702          }
2703          else
2704          {
2705            pos++;
2706          }
2707        }
2708    
2709        if (oid == null)
2710        {
2711          // This indicates that the string representation of the control is only
2712          // the OID.
2713          return new Control(controlString, false);
2714        }
2715    
2716    
2717        // See if we need to read the criticality.  If so, then do so now.
2718        // Otherwise, assume a default criticality of false.
2719        final boolean isCritical;
2720        if (hasCriticality)
2721        {
2722          // Skip over any spaces before the criticality.
2723          while (controlString.charAt(pos) == ' ')
2724          {
2725            pos++;
2726          }
2727    
2728          // Read until we find a colon or the end of the string.
2729          final int criticalityStartPos = pos;
2730          while (pos < length)
2731          {
2732            final char c = controlString.charAt(pos);
2733            if (c == ':')
2734            {
2735              hasValue = true;
2736              break;
2737            }
2738            else
2739            {
2740              pos++;
2741            }
2742          }
2743    
2744          final String criticalityString =
2745               toLowerCase(controlString.substring(criticalityStartPos, pos));
2746          if (criticalityString.equals("true"))
2747          {
2748            isCritical = true;
2749          }
2750          else if (criticalityString.equals("false"))
2751          {
2752            isCritical = false;
2753          }
2754          else
2755          {
2756            throw new LDIFException(
2757                 ERR_READ_CONTROL_LINE_INVALID_CRITICALITY.get(criticalityString,
2758                      firstLineNumber),
2759                 firstLineNumber, true, ldifLines, null);
2760          }
2761    
2762          if (hasValue)
2763          {
2764            pos++;
2765          }
2766        }
2767        else
2768        {
2769          isCritical = false;
2770        }
2771    
2772        // See if we need to read the value.  If so, then do so now.  It may be
2773        // a string, or it may be base64-encoded.  It could conceivably even be read
2774        // from a URL.
2775        final ASN1OctetString value;
2776        if (hasValue)
2777        {
2778          // The character immediately after the colon that precedes the value may
2779          // be one of the following:
2780          // - A second colon (optionally followed by a single space) to indicate
2781          //   that the value is base64-encoded.
2782          // - A less-than symbol to indicate that the value should be read from a
2783          //   location specified by a URL.
2784          // - A single space that precedes the non-base64-encoded value.
2785          // - The first character of the non-base64-encoded value.
2786          switch (controlString.charAt(pos))
2787          {
2788            case ':':
2789              try
2790              {
2791                if (controlString.length() == (pos+1))
2792                {
2793                  value = new ASN1OctetString();
2794                }
2795                else if (controlString.charAt(pos+1) == ' ')
2796                {
2797                  value = new ASN1OctetString(
2798                       Base64.decode(controlString.substring(pos+2)));
2799                }
2800                else
2801                {
2802                  value = new ASN1OctetString(
2803                       Base64.decode(controlString.substring(pos+1)));
2804                }
2805              }
2806              catch (final Exception e)
2807              {
2808                debugException(e);
2809                throw new LDIFException(
2810                     ERR_READ_CONTROL_LINE_CANNOT_BASE64_DECODE_VALUE.get(
2811                          firstLineNumber, getExceptionMessage(e)),
2812                     firstLineNumber, true, ldifLines, e);
2813              }
2814              break;
2815            case '<':
2816              try
2817              {
2818                final String urlString;
2819                if (controlString.charAt(pos+1) == ' ')
2820                {
2821                  urlString = controlString.substring(pos+2);
2822                }
2823                else
2824                {
2825                  urlString = controlString.substring(pos+1);
2826                }
2827                value = new ASN1OctetString(retrieveURLBytes(urlString,
2828                     relativeBasePath, firstLineNumber));
2829              }
2830              catch (final Exception e)
2831              {
2832                debugException(e);
2833                throw new LDIFException(
2834                     ERR_READ_CONTROL_LINE_CANNOT_RETRIEVE_VALUE_FROM_URL.get(
2835                          firstLineNumber, getExceptionMessage(e)),
2836                     firstLineNumber, true, ldifLines, e);
2837              }
2838              break;
2839            case ' ':
2840              value = new ASN1OctetString(controlString.substring(pos+1));
2841              break;
2842            default:
2843              value = new ASN1OctetString(controlString.substring(pos));
2844              break;
2845          }
2846        }
2847        else
2848        {
2849          value = null;
2850        }
2851    
2852        return new Control(oid, isCritical, value);
2853      }
2854    
2855    
2856    
2857      /**
2858       * Decodes the changetype element from the provided line.
2859       *
2860       * @param  line             The line to process.
2861       * @param  colonPos         The position of the colon that separates the
2862       *                          changetype string from its value.
2863       * @param  firstLineNumber  The line number for the start of the record.
2864       * @param  ldifLines        The lines that comprise the LDIF representation of
2865       *                          the full record being parsed.
2866       *
2867       * @return  The decoded changetype string.
2868       *
2869       * @throws  LDIFException  If a problem is encountered while trying to decode
2870       *                         the changetype.
2871       */
2872      private static String decodeChangeType(final StringBuilder line,
2873                                 final int colonPos, final long firstLineNumber,
2874                                 final ArrayList<StringBuilder> ldifLines)
2875              throws LDIFException
2876      {
2877        final int length = line.length();
2878        if (length == (colonPos+1))
2879        {
2880          // The colon was the last character on the line.  This is not
2881          // acceptable.
2882          throw new LDIFException(
2883               ERR_READ_CT_LINE_NO_CT_VALUE.get(firstLineNumber), firstLineNumber,
2884               true, ldifLines, null);
2885        }
2886        else if (line.charAt(colonPos+1) == ':')
2887        {
2888          // Skip over any spaces leading up to the value, and then the rest of
2889          // the string is the base64-encoded changetype.  This is unusual and
2890          // unnecessary, but is nevertheless acceptable.
2891          int pos = colonPos+2;
2892          while ((pos < length) && (line.charAt(pos) == ' '))
2893          {
2894            pos++;
2895          }
2896    
2897          try
2898          {
2899            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
2900            return new String(changeTypeBytes, "UTF-8");
2901          }
2902          catch (final ParseException pe)
2903          {
2904            debugException(pe);
2905            throw new LDIFException(
2906                           ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber,
2907                                                                pe.getMessage()),
2908                           firstLineNumber, true, ldifLines, pe);
2909          }
2910          catch (final Exception e)
2911          {
2912            debugException(e);
2913            throw new LDIFException(
2914                 ERR_READ_CANNOT_BASE64_DECODE_CT.get(firstLineNumber, e),
2915                 firstLineNumber, true, ldifLines, e);
2916          }
2917        }
2918        else
2919        {
2920          // Skip over any spaces leading up to the value, and then the rest of
2921          // the string is the changetype.
2922          int pos = colonPos+1;
2923          while ((pos < length) && (line.charAt(pos) == ' '))
2924          {
2925            pos++;
2926          }
2927    
2928          return line.substring(pos);
2929        }
2930      }
2931    
2932    
2933    
2934      /**
2935       * Parses the data available through the provided iterator as a collection of
2936       * attributes suitable for use in an entry or an add change record.
2937       *
2938       * @param  dn                      The DN of the record being read.
2939       * @param  duplicateValueBehavior  The behavior that should be exhibited if
2940       *                                 the LDIF reader encounters an entry with
2941       *                                 duplicate values.
2942       * @param  trailingSpaceBehavior   The behavior that should be exhibited when
2943       *                                 encountering attribute values which are not
2944       *                                 base64-encoded but contain trailing spaces.
2945       * @param  schema                  The schema to use when parsing the
2946       *                                 attributes, or {@code null} if none is
2947       *                                 needed.
2948       * @param  ldifLines               The lines that comprise the LDIF
2949       *                                 representation of the full record being
2950       *                                 parsed.
2951       * @param  iterator                The iterator to use to access the attribute
2952       *                                 lines.
2953       * @param  relativeBasePath        The base path that will be prepended to
2954       *                                 relative paths in order to obtain an
2955       *                                 absolute path.
2956       * @param  firstLineNumber         The line number for the start of the
2957       *                                 record.
2958       *
2959       * @return  The collection of attributes that were read.
2960       *
2961       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
2962       *                         set of attributes.
2963       */
2964      private static ArrayList<Attribute> parseAttributes(final String dn,
2965           final DuplicateValueBehavior duplicateValueBehavior,
2966           final TrailingSpaceBehavior trailingSpaceBehavior, final Schema schema,
2967           final ArrayList<StringBuilder> ldifLines,
2968           final Iterator<StringBuilder> iterator, final String relativeBasePath,
2969           final long firstLineNumber)
2970              throws LDIFException
2971      {
2972        final LinkedHashMap<String,Object> attributes =
2973             new LinkedHashMap<String,Object>(ldifLines.size());
2974        while (iterator.hasNext())
2975        {
2976          final StringBuilder line = iterator.next();
2977          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
2978          final int colonPos = line.indexOf(":");
2979          if (colonPos <= 0)
2980          {
2981            throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
2982                                    firstLineNumber, true, ldifLines, null);
2983          }
2984    
2985          final String attributeName = line.substring(0, colonPos);
2986          final String lowerName     = toLowerCase(attributeName);
2987    
2988          final MatchingRule matchingRule;
2989          if (schema == null)
2990          {
2991            matchingRule = CaseIgnoreStringMatchingRule.getInstance();
2992          }
2993          else
2994          {
2995            matchingRule =
2996                 MatchingRule.selectEqualityMatchingRule(attributeName, schema);
2997          }
2998    
2999          Attribute attr;
3000          final LDIFAttribute ldifAttr;
3001          final Object attrObject = attributes.get(lowerName);
3002          if (attrObject == null)
3003          {
3004            attr     = null;
3005            ldifAttr = null;
3006          }
3007          else
3008          {
3009            if (attrObject instanceof Attribute)
3010            {
3011              attr     = (Attribute) attrObject;
3012              ldifAttr = new LDIFAttribute(attr.getName(), matchingRule,
3013                                           attr.getRawValues()[0]);
3014              attributes.put(lowerName, ldifAttr);
3015            }
3016            else
3017            {
3018              attr     = null;
3019              ldifAttr = (LDIFAttribute) attrObject;
3020            }
3021          }
3022    
3023          final int length = line.length();
3024          if (length == (colonPos+1))
3025          {
3026            // This means that the attribute has a zero-length value, which is
3027            // acceptable.
3028            if (attrObject == null)
3029            {
3030              attr = new Attribute(attributeName, matchingRule, "");
3031              attributes.put(lowerName, attr);
3032            }
3033            else
3034            {
3035              try
3036              {
3037                if (! ldifAttr.addValue(new ASN1OctetString(),
3038                           duplicateValueBehavior))
3039                {
3040                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3041                  {
3042                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3043                         firstLineNumber, attributeName), firstLineNumber, true,
3044                         ldifLines, null);
3045                  }
3046                }
3047              }
3048              catch (LDAPException le)
3049              {
3050                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3051                     firstLineNumber, attributeName, getExceptionMessage(le)),
3052                     firstLineNumber, true, ldifLines, le);
3053              }
3054            }
3055          }
3056          else if (line.charAt(colonPos+1) == ':')
3057          {
3058            // Skip over any spaces leading up to the value, and then the rest of
3059            // the string is the base64-encoded attribute value.
3060            int pos = colonPos+2;
3061            while ((pos < length) && (line.charAt(pos) == ' '))
3062            {
3063              pos++;
3064            }
3065    
3066            try
3067            {
3068              final byte[] valueBytes = Base64.decode(line.substring(pos));
3069              if (attrObject == null)
3070              {
3071                attr = new Attribute(attributeName, matchingRule, valueBytes);
3072                attributes.put(lowerName, attr);
3073              }
3074              else
3075              {
3076                try
3077                {
3078                  if (! ldifAttr.addValue(new ASN1OctetString(valueBytes),
3079                             duplicateValueBehavior))
3080                  {
3081                    if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3082                    {
3083                      throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3084                           firstLineNumber, attributeName), firstLineNumber, true,
3085                           ldifLines, null);
3086                    }
3087                  }
3088                }
3089                catch (LDAPException le)
3090                {
3091                  throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3092                       firstLineNumber, attributeName, getExceptionMessage(le)),
3093                       firstLineNumber, true, ldifLines, le);
3094                }
3095              }
3096            }
3097            catch (final ParseException pe)
3098            {
3099              debugException(pe);
3100              throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3101                                           attributeName,  firstLineNumber,
3102                                           pe.getMessage()),
3103                                      firstLineNumber, true, ldifLines, pe);
3104            }
3105          }
3106          else if (line.charAt(colonPos+1) == '<')
3107          {
3108            // Skip over any spaces leading up to the value, and then the rest of
3109            // the string is a URL that indicates where to get the real content.
3110            // At the present time, we'll only support the file URLs.
3111            int pos = colonPos+2;
3112            while ((pos < length) && (line.charAt(pos) == ' '))
3113            {
3114              pos++;
3115            }
3116    
3117            final byte[] urlBytes;
3118            final String urlString = line.substring(pos);
3119            try
3120            {
3121              urlBytes =
3122                   retrieveURLBytes(urlString, relativeBasePath, firstLineNumber);
3123            }
3124            catch (final Exception e)
3125            {
3126              debugException(e);
3127              throw new LDIFException(
3128                   ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3129                        firstLineNumber, e),
3130                   firstLineNumber, true, ldifLines, e);
3131            }
3132    
3133            if (attrObject == null)
3134            {
3135              attr = new Attribute(attributeName, matchingRule, urlBytes);
3136              attributes.put(lowerName, attr);
3137            }
3138            else
3139            {
3140              try
3141              {
3142                if (! ldifAttr.addValue(new ASN1OctetString(urlBytes),
3143                     duplicateValueBehavior))
3144                {
3145                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3146                  {
3147                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3148                         firstLineNumber, attributeName), firstLineNumber, true,
3149                         ldifLines, null);
3150                  }
3151                }
3152              }
3153              catch (final LDIFException le)
3154              {
3155                debugException(le);
3156                throw le;
3157              }
3158              catch (final Exception e)
3159              {
3160                debugException(e);
3161                throw new LDIFException(
3162                     ERR_READ_URL_EXCEPTION.get(attributeName, urlString,
3163                          firstLineNumber, e),
3164                     firstLineNumber, true, ldifLines, e);
3165              }
3166            }
3167          }
3168          else
3169          {
3170            // Skip over any spaces leading up to the value, and then the rest of
3171            // the string is the value.
3172            int pos = colonPos+1;
3173            while ((pos < length) && (line.charAt(pos) == ' '))
3174            {
3175              pos++;
3176            }
3177    
3178            final String valueString = line.substring(pos);
3179            if (attrObject == null)
3180            {
3181              attr = new Attribute(attributeName, matchingRule, valueString);
3182              attributes.put(lowerName, attr);
3183            }
3184            else
3185            {
3186              try
3187              {
3188                if (! ldifAttr.addValue(new ASN1OctetString(valueString),
3189                           duplicateValueBehavior))
3190                {
3191                  if (duplicateValueBehavior != DuplicateValueBehavior.STRIP)
3192                  {
3193                    throw new LDIFException(ERR_READ_DUPLICATE_VALUE.get(dn,
3194                         firstLineNumber, attributeName), firstLineNumber, true,
3195                         ldifLines, null);
3196                  }
3197                }
3198              }
3199              catch (LDAPException le)
3200              {
3201                throw new LDIFException(ERR_READ_VALUE_SYNTAX_VIOLATION.get(dn,
3202                     firstLineNumber, attributeName, getExceptionMessage(le)),
3203                     firstLineNumber, true, ldifLines, le);
3204              }
3205            }
3206          }
3207        }
3208    
3209        final ArrayList<Attribute> attrList =
3210             new ArrayList<Attribute>(attributes.size());
3211        for (final Object o : attributes.values())
3212        {
3213          if (o instanceof Attribute)
3214          {
3215            attrList.add((Attribute) o);
3216          }
3217          else
3218          {
3219            attrList.add(((LDIFAttribute) o).toAttribute());
3220          }
3221        }
3222    
3223        return attrList;
3224      }
3225    
3226    
3227    
3228      /**
3229       * Retrieves the bytes that make up the file referenced by the given URL.
3230       *
3231       * @param  urlString         The string representation of the URL to retrieve.
3232       * @param  relativeBasePath  The base path that will be prepended to relative
3233       *                           paths in order to obtain an absolute path.
3234       * @param  firstLineNumber   The line number for the start of the record.
3235       *
3236       * @return  The bytes contained in the specified file, or an empty array if
3237       *          the specified file is empty.
3238       *
3239       * @throws  LDIFException  If the provided URL is malformed or references a
3240       *                         nonexistent file.
3241       *
3242       * @throws  IOException  If a problem is encountered while attempting to read
3243       *                       from the target file.
3244       */
3245      private static byte[] retrieveURLBytes(final String urlString,
3246                                             final String relativeBasePath,
3247                                             final long firstLineNumber)
3248              throws LDIFException, IOException
3249      {
3250        int pos;
3251        String path;
3252        final String lowerURLString = toLowerCase(urlString);
3253        if (lowerURLString.startsWith("file:/"))
3254        {
3255          pos = 6;
3256          while ((pos < urlString.length()) && (urlString.charAt(pos) == '/'))
3257          {
3258            pos++;
3259          }
3260    
3261          path = urlString.substring(pos-1);
3262        }
3263        else if (lowerURLString.startsWith("file:"))
3264        {
3265          // A file: URL that doesn't include a slash will be interpreted as a
3266          // relative path.
3267          path = relativeBasePath + urlString.substring(5);
3268        }
3269        else
3270        {
3271          throw new LDIFException(ERR_READ_URL_INVALID_SCHEME.get(urlString),
3272               firstLineNumber, true);
3273        }
3274    
3275        final File f = new File(path);
3276        if (! f.exists())
3277        {
3278          throw new LDIFException(
3279               ERR_READ_URL_NO_SUCH_FILE.get(urlString, f.getAbsolutePath()),
3280               firstLineNumber, true);
3281        }
3282    
3283        // In order to conserve memory, we'll only allow values to be read from
3284        // files no larger than 10 megabytes.
3285        final long fileSize = f.length();
3286        if (fileSize > (10 * 1024 * 1024))
3287        {
3288          throw new LDIFException(
3289               ERR_READ_URL_FILE_TOO_LARGE.get(urlString, f.getAbsolutePath(),
3290                    (10*1024*1024)),
3291               firstLineNumber, true);
3292        }
3293    
3294        int fileBytesRemaining = (int) fileSize;
3295        final byte[] fileData = new byte[(int) fileSize];
3296        final FileInputStream fis = new FileInputStream(f);
3297        try
3298        {
3299          int fileBytesRead = 0;
3300          while (fileBytesRead < fileSize)
3301          {
3302            final int bytesRead =
3303                 fis.read(fileData, fileBytesRead, fileBytesRemaining);
3304            if (bytesRead < 0)
3305            {
3306              // We hit the end of the file before we expected to.  This shouldn't
3307              // happen unless the file size changed since we first looked at it,
3308              // which we won't allow.
3309              throw new LDIFException(
3310                   ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString,
3311                        f.getAbsolutePath()),
3312                   firstLineNumber, true);
3313            }
3314    
3315            fileBytesRead      += bytesRead;
3316            fileBytesRemaining -= bytesRead;
3317          }
3318    
3319          if (fis.read() != -1)
3320          {
3321            // There is still more data to read.  This shouldn't happen unless the
3322            // file size changed since we first looked at it, which we won't allow.
3323            throw new LDIFException(
3324                 ERR_READ_URL_FILE_SIZE_CHANGED.get(urlString, f.getAbsolutePath()),
3325                 firstLineNumber, true);
3326          }
3327        }
3328        finally
3329        {
3330          fis.close();
3331        }
3332    
3333        return fileData;
3334      }
3335    
3336    
3337    
3338      /**
3339       * Parses the data available through the provided iterator into an array of
3340       * modifications suitable for use in a modify change record.
3341       *
3342       * @param  dn                     The DN of the entry being parsed.
3343       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3344       *                                encountering attribute values which are not
3345       *                                base64-encoded but contain trailing spaces.
3346       * @param  ldifLines              The lines that comprise the LDIF
3347       *                                representation of the full record being
3348       *                                parsed.
3349       * @param  iterator               The iterator to use to access the
3350       *                                modification data.
3351       * @param  firstLineNumber        The line number for the start of the record.
3352       * @param  schema                 The schema to use in processing.
3353       *
3354       * @return  An array containing the modifications that were read.
3355       *
3356       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3357       *                         set of modifications.
3358       */
3359      private static Modification[] parseModifications(final String dn,
3360                          final TrailingSpaceBehavior trailingSpaceBehavior,
3361                          final ArrayList<StringBuilder> ldifLines,
3362                          final Iterator<StringBuilder> iterator,
3363                          final long firstLineNumber, final Schema schema)
3364              throws LDIFException
3365      {
3366        final ArrayList<Modification> modList =
3367             new ArrayList<Modification>(ldifLines.size());
3368    
3369        while (iterator.hasNext())
3370        {
3371          // The first line must start with "add:", "delete:", "replace:", or
3372          // "increment:" followed by an attribute name.
3373          StringBuilder line = iterator.next();
3374          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3375          int colonPos = line.indexOf(":");
3376          if (colonPos < 0)
3377          {
3378            throw new LDIFException(ERR_READ_MOD_CR_NO_MODTYPE.get(firstLineNumber),
3379                                    firstLineNumber, true, ldifLines, null);
3380          }
3381    
3382          final ModificationType modType;
3383          final String modTypeStr = toLowerCase(line.substring(0, colonPos));
3384          if (modTypeStr.equals("add"))
3385          {
3386            modType = ModificationType.ADD;
3387          }
3388          else if (modTypeStr.equals("delete"))
3389          {
3390            modType = ModificationType.DELETE;
3391          }
3392          else if (modTypeStr.equals("replace"))
3393          {
3394            modType = ModificationType.REPLACE;
3395          }
3396          else if (modTypeStr.equals("increment"))
3397          {
3398            modType = ModificationType.INCREMENT;
3399          }
3400          else
3401          {
3402            throw new LDIFException(ERR_READ_MOD_CR_INVALID_MODTYPE.get(modTypeStr,
3403                                         firstLineNumber),
3404                                    firstLineNumber, true, ldifLines, null);
3405          }
3406    
3407          String attributeName;
3408          int length = line.length();
3409          if (length == (colonPos+1))
3410          {
3411            // The colon was the last character on the line.  This is not
3412            // acceptable.
3413            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3414                                         firstLineNumber),
3415                                    firstLineNumber, true, ldifLines, null);
3416          }
3417          else if (line.charAt(colonPos+1) == ':')
3418          {
3419            // Skip over any spaces leading up to the value, and then the rest of
3420            // the string is the base64-encoded attribute name.
3421            int pos = colonPos+2;
3422            while ((pos < length) && (line.charAt(pos) == ' '))
3423            {
3424              pos++;
3425            }
3426    
3427            try
3428            {
3429              final byte[] dnBytes = Base64.decode(line.substring(pos));
3430              attributeName = new String(dnBytes, "UTF-8");
3431            }
3432            catch (final ParseException pe)
3433            {
3434              debugException(pe);
3435              throw new LDIFException(
3436                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3437                        firstLineNumber, pe.getMessage()),
3438                   firstLineNumber, true, ldifLines, pe);
3439            }
3440            catch (final Exception e)
3441            {
3442              debugException(e);
3443              throw new LDIFException(
3444                   ERR_READ_MOD_CR_MODTYPE_CANNOT_BASE64_DECODE_ATTR.get(
3445                        firstLineNumber, e),
3446                   firstLineNumber, true, ldifLines, e);
3447            }
3448          }
3449          else
3450          {
3451            // Skip over any spaces leading up to the value, and then the rest of
3452            // the string is the attribute name.
3453            int pos = colonPos+1;
3454            while ((pos < length) && (line.charAt(pos) == ' '))
3455            {
3456              pos++;
3457            }
3458    
3459            attributeName = line.substring(pos);
3460          }
3461    
3462          if (attributeName.length() == 0)
3463          {
3464            throw new LDIFException(ERR_READ_MOD_CR_MODTYPE_NO_ATTR.get(
3465                                         firstLineNumber),
3466                                    firstLineNumber, true, ldifLines, null);
3467          }
3468    
3469    
3470          // The next zero or more lines may be the set of attribute values.  Keep
3471          // reading until we reach the end of the iterator or until we find a line
3472          // with just a "-".
3473          final ArrayList<ASN1OctetString> valueList =
3474               new ArrayList<ASN1OctetString>(ldifLines.size());
3475          while (iterator.hasNext())
3476          {
3477            line = iterator.next();
3478            handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3479            if (line.toString().equals("-"))
3480            {
3481              break;
3482            }
3483    
3484            colonPos = line.indexOf(":");
3485            if (colonPos < 0)
3486            {
3487              throw new LDIFException(ERR_READ_NO_ATTR_COLON.get(firstLineNumber),
3488                                      firstLineNumber, true, ldifLines, null);
3489            }
3490            else if (! line.substring(0, colonPos).equalsIgnoreCase(attributeName))
3491            {
3492              // There are a couple of cases in which this might be acceptable:
3493              // - If the two names are logically equivalent, but have an alternate
3494              //   name (or OID) for the target attribute type, or if there are
3495              //   attribute options and the options are just in a different order.
3496              // - If this is the first value for the target attribute and the
3497              //   alternate name includes a "binary" option that the original
3498              //   attribute name did not have.  In this case, all subsequent values
3499              //   will also be required to have the binary option.
3500              final String alternateName = line.substring(0, colonPos);
3501    
3502    
3503              // Check to see if the base names are equivalent.
3504              boolean baseNameEquivalent = false;
3505              final String expectedBaseName = Attribute.getBaseName(attributeName);
3506              final String alternateBaseName = Attribute.getBaseName(alternateName);
3507              if (alternateBaseName.equalsIgnoreCase(expectedBaseName))
3508              {
3509                baseNameEquivalent = true;
3510              }
3511              else
3512              {
3513                if (schema != null)
3514                {
3515                  final AttributeTypeDefinition expectedAT =
3516                       schema.getAttributeType(expectedBaseName);
3517                  final AttributeTypeDefinition alternateAT =
3518                       schema.getAttributeType(alternateBaseName);
3519                  if ((expectedAT != null) && (alternateAT != null) &&
3520                      expectedAT.equals(alternateAT))
3521                  {
3522                    baseNameEquivalent = true;
3523                  }
3524                }
3525              }
3526    
3527    
3528              // Check to see if the attribute options are equivalent.
3529              final Set<String> expectedOptions =
3530                   Attribute.getOptions(attributeName);
3531              final Set<String> lowerExpectedOptions =
3532                   new HashSet<String>(expectedOptions.size());
3533              for (final String s : expectedOptions)
3534              {
3535                lowerExpectedOptions.add(toLowerCase(s));
3536              }
3537    
3538              final Set<String> alternateOptions =
3539                   Attribute.getOptions(alternateName);
3540              final Set<String> lowerAlternateOptions =
3541                   new HashSet<String>(alternateOptions.size());
3542              for (final String s : alternateOptions)
3543              {
3544                lowerAlternateOptions.add(toLowerCase(s));
3545              }
3546    
3547              final boolean optionsEquivalent =
3548                   lowerAlternateOptions.equals(lowerExpectedOptions);
3549    
3550    
3551              if (baseNameEquivalent && optionsEquivalent)
3552              {
3553                // This is fine.  The two attribute descriptions are logically
3554                // equivalent.  We'll continue using the attribute description that
3555                // was provided first.
3556              }
3557              else if (valueList.isEmpty() && baseNameEquivalent &&
3558                       lowerAlternateOptions.remove("binary") &&
3559                       lowerAlternateOptions.equals(lowerExpectedOptions))
3560              {
3561                // This means that the provided value is the first value for the
3562                // attribute, and that the only significant difference is that the
3563                // provided attribute description included an unexpected "binary"
3564                // option.  We'll accept this, but will require any additional
3565                // values for this modification to also include the binary option,
3566                // and we'll use the binary option in the attribute that is
3567                // eventually created.
3568                attributeName = alternateName;
3569              }
3570              else
3571              {
3572                // This means that either the base names are different or the sets
3573                // of options are incompatible.  This is not acceptable.
3574                throw new LDIFException(ERR_READ_MOD_CR_ATTR_MISMATCH.get(
3575                                             firstLineNumber,
3576                                             line.substring(0, colonPos),
3577                                             attributeName),
3578                                        firstLineNumber, true, ldifLines, null);
3579              }
3580            }
3581    
3582            length = line.length();
3583            final ASN1OctetString value;
3584            if (length == (colonPos+1))
3585            {
3586              // The colon was the last character on the line.  This is fine.
3587              value = new ASN1OctetString();
3588            }
3589            else if (line.charAt(colonPos+1) == ':')
3590            {
3591              // Skip over any spaces leading up to the value, and then the rest of
3592              // the string is the base64-encoded value.  This is unusual and
3593              // unnecessary, but is nevertheless acceptable.
3594              int pos = colonPos+2;
3595              while ((pos < length) && (line.charAt(pos) == ' '))
3596              {
3597                pos++;
3598              }
3599    
3600              try
3601              {
3602                value = new ASN1OctetString(Base64.decode(line.substring(pos)));
3603              }
3604              catch (final ParseException pe)
3605              {
3606                debugException(pe);
3607                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3608                     attributeName, firstLineNumber, pe.getMessage()),
3609                     firstLineNumber, true, ldifLines, pe);
3610              }
3611              catch (final Exception e)
3612              {
3613                debugException(e);
3614                throw new LDIFException(ERR_READ_CANNOT_BASE64_DECODE_ATTR.get(
3615                                             firstLineNumber, e),
3616                                        firstLineNumber, true, ldifLines, e);
3617              }
3618            }
3619            else
3620            {
3621              // Skip over any spaces leading up to the value, and then the rest of
3622              // the string is the value.
3623              int pos = colonPos+1;
3624              while ((pos < length) && (line.charAt(pos) == ' '))
3625              {
3626                pos++;
3627              }
3628    
3629              value = new ASN1OctetString(line.substring(pos));
3630            }
3631    
3632            valueList.add(value);
3633          }
3634    
3635          final ASN1OctetString[] values = new ASN1OctetString[valueList.size()];
3636          valueList.toArray(values);
3637    
3638          // If it's an add modification type, then there must be at least one
3639          // value.
3640          if ((modType.intValue() == ModificationType.ADD.intValue()) &&
3641              (values.length == 0))
3642          {
3643            throw new LDIFException(ERR_READ_MOD_CR_NO_ADD_VALUES.get(attributeName,
3644                                         firstLineNumber),
3645                                    firstLineNumber, true, ldifLines, null);
3646          }
3647    
3648          // If it's an increment modification type, then there must be exactly one
3649          // value.
3650          if ((modType.intValue() == ModificationType.INCREMENT.intValue()) &&
3651              (values.length != 1))
3652          {
3653            throw new LDIFException(ERR_READ_MOD_CR_INVALID_INCR_VALUE_COUNT.get(
3654                                         firstLineNumber, attributeName),
3655                                    firstLineNumber, true, ldifLines, null);
3656          }
3657    
3658          modList.add(new Modification(modType, attributeName, values));
3659        }
3660    
3661        final Modification[] mods = new Modification[modList.size()];
3662        modList.toArray(mods);
3663        return mods;
3664      }
3665    
3666    
3667    
3668      /**
3669       * Parses the data available through the provided iterator as the body of a
3670       * modify DN change record (i.e., the newrdn, deleteoldrdn, and optional
3671       * newsuperior lines).
3672       *
3673       * @param  ldifLines              The lines that comprise the LDIF
3674       *                                representation of the full record being
3675       *                                parsed.
3676       * @param  iterator               The iterator to use to access the modify DN
3677       *                                data.
3678       * @param  dn                     The current DN of the entry.
3679       * @param  controls               The set of controls to include in the change
3680       *                                record.
3681       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3682       *                                encountering attribute values which are not
3683       *                                base64-encoded but contain trailing spaces.
3684       * @param  firstLineNumber        The line number for the start of the record.
3685       *
3686       * @return  The decoded modify DN change record.
3687       *
3688       * @throws  LDIFException  If the provided LDIF data cannot be decoded as a
3689       *                         modify DN change record.
3690       */
3691      private static LDIFModifyDNChangeRecord parseModifyDNChangeRecord(
3692           final ArrayList<StringBuilder> ldifLines,
3693           final Iterator<StringBuilder> iterator, final String dn,
3694           final List<Control> controls,
3695           final TrailingSpaceBehavior trailingSpaceBehavior,
3696           final long firstLineNumber)
3697           throws LDIFException
3698      {
3699        // The next line must be the new RDN, and it must start with "newrdn:".
3700        StringBuilder line = iterator.next();
3701        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3702        int colonPos = line.indexOf(":");
3703        if ((colonPos < 0) ||
3704            (! line.substring(0, colonPos).equalsIgnoreCase("newrdn")))
3705        {
3706          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_COLON.get(
3707                                       firstLineNumber),
3708                                  firstLineNumber, true, ldifLines, null);
3709        }
3710    
3711        final String newRDN;
3712        int length = line.length();
3713        if (length == (colonPos+1))
3714        {
3715          // The colon was the last character on the line.  This is not acceptable.
3716          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3717                                       firstLineNumber),
3718                                  firstLineNumber, true, ldifLines, null);
3719        }
3720        else if (line.charAt(colonPos+1) == ':')
3721        {
3722          // Skip over any spaces leading up to the value, and then the rest of the
3723          // string is the base64-encoded new RDN.
3724          int pos = colonPos+2;
3725          while ((pos < length) && (line.charAt(pos) == ' '))
3726          {
3727            pos++;
3728          }
3729    
3730          try
3731          {
3732            final byte[] dnBytes = Base64.decode(line.substring(pos));
3733            newRDN = new String(dnBytes, "UTF-8");
3734          }
3735          catch (final ParseException pe)
3736          {
3737            debugException(pe);
3738            throw new LDIFException(
3739                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3740                                                                   pe.getMessage()),
3741                 firstLineNumber, true, ldifLines, pe);
3742          }
3743          catch (final Exception e)
3744          {
3745            debugException(e);
3746            throw new LDIFException(
3747                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWRDN.get(firstLineNumber,
3748                                                                   e),
3749                 firstLineNumber, true, ldifLines, e);
3750          }
3751        }
3752        else
3753        {
3754          // Skip over any spaces leading up to the value, and then the rest of the
3755          // string is the new RDN.
3756          int pos = colonPos+1;
3757          while ((pos < length) && (line.charAt(pos) == ' '))
3758          {
3759            pos++;
3760          }
3761    
3762          newRDN = line.substring(pos);
3763        }
3764    
3765        if (newRDN.length() == 0)
3766        {
3767          throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWRDN_VALUE.get(
3768                                       firstLineNumber),
3769                                  firstLineNumber, true, ldifLines, null);
3770        }
3771    
3772    
3773        // The next line must be the deleteOldRDN flag, and it must start with
3774        // 'deleteoldrdn:'.
3775        if (! iterator.hasNext())
3776        {
3777          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3778                                       firstLineNumber),
3779                                  firstLineNumber, true, ldifLines, null);
3780        }
3781    
3782        line = iterator.next();
3783        handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3784        colonPos = line.indexOf(":");
3785        if ((colonPos < 0) ||
3786            (! line.substring(0, colonPos).equalsIgnoreCase("deleteoldrdn")))
3787        {
3788          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_COLON.get(
3789                                       firstLineNumber),
3790                                  firstLineNumber, true, ldifLines, null);
3791        }
3792    
3793        final String deleteOldRDNStr;
3794        length = line.length();
3795        if (length == (colonPos+1))
3796        {
3797          // The colon was the last character on the line.  This is not acceptable.
3798          throw new LDIFException(ERR_READ_MODDN_CR_NO_DELOLDRDN_VALUE.get(
3799                                       firstLineNumber),
3800                                  firstLineNumber, true, ldifLines, null);
3801        }
3802        else if (line.charAt(colonPos+1) == ':')
3803        {
3804          // Skip over any spaces leading up to the value, and then the rest of the
3805          // string is the base64-encoded value.  This is unusual and
3806          // unnecessary, but is nevertheless acceptable.
3807          int pos = colonPos+2;
3808          while ((pos < length) && (line.charAt(pos) == ' '))
3809          {
3810            pos++;
3811          }
3812    
3813          try
3814          {
3815            final byte[] changeTypeBytes = Base64.decode(line.substring(pos));
3816            deleteOldRDNStr = new String(changeTypeBytes, "UTF-8");
3817          }
3818          catch (final ParseException pe)
3819          {
3820            debugException(pe);
3821            throw new LDIFException(
3822                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3823                      firstLineNumber, pe.getMessage()),
3824                 firstLineNumber, true, ldifLines, pe);
3825          }
3826          catch (final Exception e)
3827          {
3828            debugException(e);
3829            throw new LDIFException(
3830                 ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_DELOLDRDN.get(
3831                      firstLineNumber, e),
3832                 firstLineNumber, true, ldifLines, e);
3833          }
3834        }
3835        else
3836        {
3837          // Skip over any spaces leading up to the value, and then the rest of the
3838          // string is the value.
3839          int pos = colonPos+1;
3840          while ((pos < length) && (line.charAt(pos) == ' '))
3841          {
3842            pos++;
3843          }
3844    
3845          deleteOldRDNStr = line.substring(pos);
3846        }
3847    
3848        final boolean deleteOldRDN;
3849        if (deleteOldRDNStr.equals("0"))
3850        {
3851          deleteOldRDN = false;
3852        }
3853        else if (deleteOldRDNStr.equals("1"))
3854        {
3855          deleteOldRDN = true;
3856        }
3857        else if (deleteOldRDNStr.equalsIgnoreCase("false") ||
3858                 deleteOldRDNStr.equalsIgnoreCase("no"))
3859        {
3860          // This is technically illegal, but we'll allow it.
3861          deleteOldRDN = false;
3862        }
3863        else if (deleteOldRDNStr.equalsIgnoreCase("true") ||
3864                 deleteOldRDNStr.equalsIgnoreCase("yes"))
3865        {
3866          // This is also technically illegal, but we'll allow it.
3867          deleteOldRDN = false;
3868        }
3869        else
3870        {
3871          throw new LDIFException(ERR_READ_MODDN_CR_INVALID_DELOLDRDN.get(
3872                                       deleteOldRDNStr, firstLineNumber),
3873                                  firstLineNumber, true, ldifLines, null);
3874        }
3875    
3876    
3877        // If there is another line, then it must be the new superior DN and it must
3878        // start with "newsuperior:".  If this is absent, then it's fine.
3879        final String newSuperiorDN;
3880        if (iterator.hasNext())
3881        {
3882          line = iterator.next();
3883          handleTrailingSpaces(line, dn, firstLineNumber, trailingSpaceBehavior);
3884          colonPos = line.indexOf(":");
3885          if ((colonPos < 0) ||
3886              (! line.substring(0, colonPos).equalsIgnoreCase("newsuperior")))
3887          {
3888            throw new LDIFException(ERR_READ_MODDN_CR_NO_NEWSUPERIOR_COLON.get(
3889                                         firstLineNumber),
3890                                    firstLineNumber, true, ldifLines, null);
3891          }
3892    
3893          length = line.length();
3894          if (length == (colonPos+1))
3895          {
3896            // The colon was the last character on the line.  This is fine.
3897            newSuperiorDN = "";
3898          }
3899          else if (line.charAt(colonPos+1) == ':')
3900          {
3901            // Skip over any spaces leading up to the value, and then the rest of
3902            // the string is the base64-encoded new superior DN.
3903            int pos = colonPos+2;
3904            while ((pos < length) && (line.charAt(pos) == ' '))
3905            {
3906              pos++;
3907            }
3908    
3909            try
3910            {
3911              final byte[] dnBytes = Base64.decode(line.substring(pos));
3912              newSuperiorDN = new String(dnBytes, "UTF-8");
3913            }
3914            catch (final ParseException pe)
3915            {
3916              debugException(pe);
3917              throw new LDIFException(
3918                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3919                        firstLineNumber, pe.getMessage()),
3920                   firstLineNumber, true, ldifLines, pe);
3921            }
3922            catch (final Exception e)
3923            {
3924              debugException(e);
3925              throw new LDIFException(
3926                   ERR_READ_MODDN_CR_CANNOT_BASE64_DECODE_NEWSUPERIOR.get(
3927                        firstLineNumber, e),
3928                   firstLineNumber, true, ldifLines, e);
3929            }
3930          }
3931          else
3932          {
3933            // Skip over any spaces leading up to the value, and then the rest of
3934            // the string is the new superior DN.
3935            int pos = colonPos+1;
3936            while ((pos < length) && (line.charAt(pos) == ' '))
3937            {
3938              pos++;
3939            }
3940    
3941            newSuperiorDN = line.substring(pos);
3942          }
3943        }
3944        else
3945        {
3946          newSuperiorDN = null;
3947        }
3948    
3949    
3950        // There must not be any more lines.
3951        if (iterator.hasNext())
3952        {
3953          throw new LDIFException(ERR_READ_CR_EXTRA_MODDN_DATA.get(firstLineNumber),
3954                                  firstLineNumber, true, ldifLines, null);
3955        }
3956    
3957        return new LDIFModifyDNChangeRecord(dn, newRDN, deleteOldRDN,
3958             newSuperiorDN, controls);
3959      }
3960    
3961    
3962    
3963      /**
3964       * Examines the line contained in the provided buffer to determine whether it
3965       * may contain one or more illegal trailing spaces.  If it does, then those
3966       * spaces will either be stripped out or an exception will be thrown to
3967       * indicate that they are illegal.
3968       *
3969       * @param  buffer                 The buffer to be examined.
3970       * @param  dn                     The DN of the LDIF record being parsed.  It
3971       *                                may be {@code null} if the DN is not yet
3972       *                                known (e.g., because the provided line is
3973       *                                expected to contain that DN).
3974       * @param  firstLineNumber        The approximate line number in the LDIF
3975       *                                source on which the LDIF record begins.
3976       * @param  trailingSpaceBehavior  The behavior that should be exhibited when
3977       *                                encountering attribute values which are not
3978       *                                base64-encoded but contain trailing spaces.
3979       *
3980       * @throws  LDIFException  If the line contained in the provided buffer ends
3981       *                         with one or more illegal trailing spaces and
3982       *                         {@code stripTrailingSpaces} was provided with a
3983       *                         value of {@code false}.
3984       */
3985      private static void handleTrailingSpaces(final StringBuilder buffer,
3986                               final String dn, final long firstLineNumber,
3987                               final TrailingSpaceBehavior trailingSpaceBehavior)
3988              throws LDIFException
3989      {
3990        int pos = buffer.length() - 1;
3991        boolean trailingFound = false;
3992        while ((pos >= 0) && (buffer.charAt(pos) == ' '))
3993        {
3994          trailingFound = true;
3995          pos--;
3996        }
3997    
3998        if (trailingFound && (buffer.charAt(pos) != ':'))
3999        {
4000          switch (trailingSpaceBehavior)
4001          {
4002            case STRIP:
4003              buffer.setLength(pos+1);
4004              break;
4005    
4006            case REJECT:
4007              if (dn == null)
4008              {
4009                throw new LDIFException(
4010                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITHOUT_DN.get(firstLineNumber,
4011                          buffer.toString()),
4012                     firstLineNumber, true);
4013              }
4014              else
4015              {
4016                throw new LDIFException(
4017                     ERR_READ_ILLEGAL_TRAILING_SPACE_WITH_DN.get(dn,
4018                          firstLineNumber, buffer.toString()),
4019                     firstLineNumber, true);
4020              }
4021    
4022            case RETAIN:
4023            default:
4024              // No action will be taken.
4025              break;
4026          }
4027        }
4028      }
4029    
4030    
4031    
4032      /**
4033       * This represents an unparsed LDIFRecord.  It stores the line number of the
4034       * first line of the record and each line of the record.
4035       */
4036      private static final class UnparsedLDIFRecord
4037      {
4038        private final ArrayList<StringBuilder> lineList;
4039        private final long firstLineNumber;
4040        private final Exception failureCause;
4041        private final boolean isEOF;
4042        private final DuplicateValueBehavior duplicateValueBehavior;
4043        private final Schema schema;
4044        private final TrailingSpaceBehavior trailingSpaceBehavior;
4045    
4046    
4047    
4048        /**
4049         * Constructor.
4050         *
4051         * @param  lineList                The lines that comprise the LDIF record.
4052         * @param  duplicateValueBehavior  The behavior to exhibit if the entry
4053         *                                 contains duplicate attribute values.
4054         * @param  trailingSpaceBehavior   Specifies the behavior to exhibit when
4055         *                                 encountering trailing spaces in
4056         *                                 non-base64-encoded attribute values.
4057         * @param  schema                  The schema to use when parsing, if
4058         *                                 applicable.
4059         * @param  firstLineNumber         The first line number of the LDIF record.
4060         */
4061        private UnparsedLDIFRecord(final ArrayList<StringBuilder> lineList,
4062                     final DuplicateValueBehavior duplicateValueBehavior,
4063                     final TrailingSpaceBehavior trailingSpaceBehavior,
4064                     final Schema schema, final long firstLineNumber)
4065        {
4066          this.lineList               = lineList;
4067          this.firstLineNumber        = firstLineNumber;
4068          this.duplicateValueBehavior = duplicateValueBehavior;
4069          this.trailingSpaceBehavior  = trailingSpaceBehavior;
4070          this.schema                 = schema;
4071    
4072          failureCause = null;
4073          isEOF =
4074               (firstLineNumber < 0) || ((lineList != null) && lineList.isEmpty());
4075        }
4076    
4077    
4078    
4079        /**
4080         * Constructor.
4081         *
4082         * @param failureCause  The Exception thrown when reading from the input.
4083         */
4084        private UnparsedLDIFRecord(final Exception failureCause)
4085        {
4086          this.failureCause = failureCause;
4087    
4088          lineList               = null;
4089          firstLineNumber        = 0;
4090          duplicateValueBehavior = DuplicateValueBehavior.REJECT;
4091          trailingSpaceBehavior  = TrailingSpaceBehavior.REJECT;
4092          schema                 = null;
4093          isEOF                  = false;
4094        }
4095    
4096    
4097    
4098        /**
4099         * Return the lines that comprise the LDIF record.
4100         *
4101         * @return  The lines that comprise the LDIF record.
4102         */
4103        private ArrayList<StringBuilder> getLineList()
4104        {
4105          return lineList;
4106        }
4107    
4108    
4109    
4110        /**
4111         * Retrieves the behavior to exhibit when encountering duplicate attribute
4112         * values.
4113         *
4114         * @return  The behavior to exhibit when encountering duplicate attribute
4115         *          values.
4116         */
4117        private DuplicateValueBehavior getDuplicateValueBehavior()
4118        {
4119          return duplicateValueBehavior;
4120        }
4121    
4122    
4123    
4124        /**
4125         * Retrieves the behavior that should be exhibited when encountering
4126         * attribute values which are not base64-encoded but contain trailing
4127         * spaces.  The LDIF specification strongly recommends that any value which
4128         * legitimately contains trailing spaces be base64-encoded, but the LDAP SDK
4129         * LDIF parser may be configured to automatically strip these spaces, to
4130         * preserve them, or to reject any entry or change record containing them.
4131         *
4132         * @return  The behavior that should be exhibited when encountering
4133         *          attribute values which are not base64-encoded but contain
4134         *          trailing spaces.
4135         */
4136        private TrailingSpaceBehavior getTrailingSpaceBehavior()
4137        {
4138          return trailingSpaceBehavior;
4139        }
4140    
4141    
4142    
4143        /**
4144         * Retrieves the schema that should be used when parsing the record, if
4145         * applicable.
4146         *
4147         * @return  The schema that should be used when parsing the record, or
4148         *          {@code null} if none should be used.
4149         */
4150        private Schema getSchema()
4151        {
4152          return schema;
4153        }
4154    
4155    
4156    
4157        /**
4158         * Return the first line number of the LDIF record.
4159         *
4160         * @return  The first line number of the LDIF record.
4161         */
4162        private long getFirstLineNumber()
4163        {
4164          return firstLineNumber;
4165        }
4166    
4167    
4168    
4169        /**
4170         * Return {@code true} iff the end of the input was reached.
4171         *
4172         * @return  {@code true} iff the end of the input was reached.
4173         */
4174        private boolean isEOF()
4175        {
4176          return isEOF;
4177        }
4178    
4179    
4180    
4181        /**
4182         * Returns the reason that reading the record lines failed.  This normally
4183         * is only non-null if something bad happened to the input stream (like
4184         * a disk read error).
4185         *
4186         * @return  The reason that reading the record lines failed.
4187         */
4188        private Exception getFailureCause()
4189        {
4190          return failureCause;
4191        }
4192      }
4193    
4194    
4195      /**
4196       * When processing in asynchronous mode, this thread is responsible for
4197       * reading the raw unparsed records from the input and submitting them for
4198       * processing.
4199       */
4200      private final class LineReaderThread
4201           extends Thread
4202      {
4203        /**
4204         * Constructor.
4205         */
4206        private LineReaderThread()
4207        {
4208          super("Asynchronous LDIF line reader");
4209          setDaemon(true);
4210        }
4211    
4212    
4213    
4214        /**
4215         * Reads raw, unparsed records from the input and submits them for
4216         * processing until the input is finished or closed.
4217         */
4218        @Override()
4219        public void run()
4220        {
4221          try
4222          {
4223            boolean stopProcessing = false;
4224            while (!stopProcessing)
4225            {
4226              UnparsedLDIFRecord unparsedRecord = null;
4227              try
4228              {
4229                unparsedRecord = readUnparsedRecord();
4230              }
4231              catch (IOException e)
4232              {
4233                debugException(e);
4234                unparsedRecord = new UnparsedLDIFRecord(e);
4235                stopProcessing = true;
4236              }
4237              catch (Exception e)
4238              {
4239                debugException(e);
4240                unparsedRecord = new UnparsedLDIFRecord(e);
4241              }
4242    
4243              try
4244              {
4245                asyncParser.submit(unparsedRecord);
4246              }
4247              catch (InterruptedException e)
4248              {
4249                debugException(e);
4250                // If this thread is interrupted, then someone wants us to stop
4251                // processing, so that's what we'll do.
4252                stopProcessing = true;
4253              }
4254    
4255              if ((unparsedRecord == null) || (unparsedRecord.isEOF()))
4256              {
4257                stopProcessing = true;
4258              }
4259            }
4260          }
4261          finally
4262          {
4263            try
4264            {
4265              asyncParser.shutdown();
4266            }
4267            catch (InterruptedException e)
4268            {
4269              debugException(e);
4270            }
4271            finally
4272            {
4273              asyncParsingComplete.set(true);
4274            }
4275          }
4276        }
4277      }
4278    
4279    
4280    
4281      /**
4282       * Used to parse Records asynchronously.
4283       */
4284      private final class RecordParser implements Processor<UnparsedLDIFRecord,
4285                                                            LDIFRecord>
4286      {
4287        /**
4288         * {@inheritDoc}
4289         */
4290        public LDIFRecord process(final UnparsedLDIFRecord input)
4291               throws LDIFException
4292        {
4293          LDIFRecord record = decodeRecord(input, relativeBasePath, schema);
4294    
4295          if ((record instanceof Entry) && (entryTranslator != null))
4296          {
4297            record = entryTranslator.translate((Entry) record,
4298                 input.getFirstLineNumber());
4299    
4300            if (record == null)
4301            {
4302              record = SKIP_ENTRY;
4303            }
4304          }
4305          if ((record instanceof LDIFChangeRecord) &&
4306              (changeRecordTranslator != null))
4307          {
4308            record = changeRecordTranslator.translate((LDIFChangeRecord) record,
4309                 input.getFirstLineNumber());
4310    
4311            if (record == null)
4312            {
4313              record = SKIP_ENTRY;
4314            }
4315          }
4316          return record;
4317        }
4318      }
4319    }