001    /*
002     * Copyright 2008-2016 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2016 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.File;
026    import java.io.FileInputStream;
027    import java.io.InputStream;
028    import java.io.IOException;
029    import java.io.OutputStream;
030    import java.util.ArrayList;
031    import java.util.Iterator;
032    import java.util.TreeMap;
033    import java.util.LinkedHashMap;
034    import java.util.List;
035    import java.util.concurrent.atomic.AtomicLong;
036    import java.util.zip.GZIPInputStream;
037    
038    import com.unboundid.ldap.sdk.Entry;
039    import com.unboundid.ldap.sdk.LDAPConnection;
040    import com.unboundid.ldap.sdk.LDAPException;
041    import com.unboundid.ldap.sdk.ResultCode;
042    import com.unboundid.ldap.sdk.Version;
043    import com.unboundid.ldap.sdk.schema.Schema;
044    import com.unboundid.ldap.sdk.schema.EntryValidator;
045    import com.unboundid.ldif.DuplicateValueBehavior;
046    import com.unboundid.ldif.LDIFException;
047    import com.unboundid.ldif.LDIFReader;
048    import com.unboundid.ldif.LDIFReaderEntryTranslator;
049    import com.unboundid.ldif.LDIFWriter;
050    import com.unboundid.util.LDAPCommandLineTool;
051    import com.unboundid.util.ThreadSafety;
052    import com.unboundid.util.ThreadSafetyLevel;
053    import com.unboundid.util.args.ArgumentException;
054    import com.unboundid.util.args.ArgumentParser;
055    import com.unboundid.util.args.BooleanArgument;
056    import com.unboundid.util.args.FileArgument;
057    import com.unboundid.util.args.IntegerArgument;
058    import com.unboundid.util.args.StringArgument;
059    
060    import static com.unboundid.util.StaticUtils.*;
061    
062    
063    
064    /**
065     * This class provides a simple tool that can be used to validate that the
066     * contents of an LDIF file are valid.  This includes ensuring that the contents
067     * can be parsed as valid LDIF, and it can also ensure that the LDIF content
068     * conforms to the server schema.  It will obtain the schema by connecting to
069     * the server and retrieving the default schema (i.e., the schema which governs
070     * the root DSE).  By default, a thorough set of validation will be performed,
071     * but it is possible to disable certain types of validation.
072     * <BR><BR>
073     * Some of the APIs demonstrated by this example include:
074     * <UL>
075     *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
076     *       package)</LI>
077     *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
078     *       package)</LI>
079     *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
080     *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
081     *       package)</LI>
082     * </UL>
083     * <BR><BR>
084     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
085     * class (to obtain the information to use to connect to the server to read the
086     * schema), as well as the following additional arguments:
087     * <UL>
088     *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
089     *       containing files with schema definitions.  If this argument is
090     *       provided, then no attempt will be made to communicate with a directory
091     *       server.</LI>
092     *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
093     *       file to be validated.</LI>
094     *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
095     *       compressed.</LI>
096     *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
097     *       to be written with information about all entries that failed
098     *       validation.</LI>
099     *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
100     *       concurrent threads to use when processing the LDIF.  If this is not
101     *       provided, then a default of one thread will be used.</LI>
102     *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
103     *       process should ignore validation failures due to entries that contain
104     *       object classes not defined in the server schema.</LI>
105     *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
106     *       should ignore validation failures due to entries that contain
107     *       attributes not defined in the server schema.</LI>
108     *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
109     *       ignore validation failures due to entries with malformed DNs.</LI>
110     *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
111     *       should ignore validation failures due to entries that contain an RDN
112     *       attribute value that is not present in the set of entry
113     *       attributes.</LI>
114     *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
115     *       process should ignore validation failures due to entries that either do
116     *       not have a structural object class or that have multiple structural
117     *       object classes.</LI>
118     *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
119     *       process should ignore validation failures due to entries containing
120     *       auxiliary classes that are not allowed by a DIT content rule, or
121     *       abstract classes that are not subclassed by an auxiliary or structural
122     *       class contained in the entry.</LI>
123     *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
124     *       should ignore validation failures due to entries including attributes
125     *       that are not allowed or are explicitly prohibited by a DIT content
126     *       rule.</LI>
127     *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
128     *       should ignore validation failures due to entries missing required
129     *       attributes.</LI>
130     *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
131     *       process should ignore validation failures due to single-valued
132     *       attributes containing multiple values.</LI>
133     *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
134     *       should ignore validation failures due to attribute values which violate
135     *       the associated attribute syntax.</LI>
136     *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
137     *       process should ignore validation failures due to attribute values which
138     *       violate the associated attribute syntax, but only for the specified
139     *       attribute types.</LI>
140     *   <LI>"--ignoreNameForms" -- indicates that the validation process should
141     *       ignore validation failures due to name form violations (in which the
142     *       entry's RDN does not comply with the associated name form).</LI>
143     * </UL>
144     */
145    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
146    public final class ValidateLDIF
147           extends LDAPCommandLineTool
148           implements LDIFReaderEntryTranslator
149    {
150      /**
151       * The end-of-line character for this platform.
152       */
153      private static final String EOL = System.getProperty("line.separator", "\n");
154    
155    
156    
157      // The arguments used by this program.
158      private BooleanArgument ignoreDuplicateValues;
159      private BooleanArgument ignoreUndefinedObjectClasses;
160      private BooleanArgument ignoreUndefinedAttributes;
161      private BooleanArgument ignoreMalformedDNs;
162      private BooleanArgument ignoreMissingRDNValues;
163      private BooleanArgument ignoreMissingSuperiorObjectClasses;
164      private BooleanArgument ignoreStructuralObjectClasses;
165      private BooleanArgument ignoreProhibitedObjectClasses;
166      private BooleanArgument ignoreProhibitedAttributes;
167      private BooleanArgument ignoreMissingAttributes;
168      private BooleanArgument ignoreSingleValuedAttributes;
169      private BooleanArgument ignoreAttributeSyntax;
170      private BooleanArgument ignoreNameForms;
171      private BooleanArgument isCompressed;
172      private FileArgument    schemaDirectory;
173      private FileArgument    ldifFile;
174      private FileArgument    rejectFile;
175      private IntegerArgument numThreads;
176      private StringArgument  ignoreSyntaxViolationsForAttribute;
177    
178      // The counter used to keep track of the number of entries processed.
179      private final AtomicLong entriesProcessed = new AtomicLong(0L);
180    
181      // The counter used to keep track of the number of entries that could not be
182      // parsed as valid entries.
183      private final AtomicLong malformedEntries = new AtomicLong(0L);
184    
185      // The entry validator that will be used to validate the entries.
186      private EntryValidator entryValidator;
187    
188      // The LDIF writer that will be used to write rejected entries.
189      private LDIFWriter rejectWriter;
190    
191    
192    
193      /**
194       * Parse the provided command line arguments and make the appropriate set of
195       * changes.
196       *
197       * @param  args  The command line arguments provided to this program.
198       */
199      public static void main(final String[] args)
200      {
201        final ResultCode resultCode = main(args, System.out, System.err);
202        if (resultCode != ResultCode.SUCCESS)
203        {
204          System.exit(resultCode.intValue());
205        }
206      }
207    
208    
209    
210      /**
211       * Parse the provided command line arguments and make the appropriate set of
212       * changes.
213       *
214       * @param  args       The command line arguments provided to this program.
215       * @param  outStream  The output stream to which standard out should be
216       *                    written.  It may be {@code null} if output should be
217       *                    suppressed.
218       * @param  errStream  The output stream to which standard error should be
219       *                    written.  It may be {@code null} if error messages
220       *                    should be suppressed.
221       *
222       * @return  A result code indicating whether the processing was successful.
223       */
224      public static ResultCode main(final String[] args,
225                                    final OutputStream outStream,
226                                    final OutputStream errStream)
227      {
228        final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
229        return validateLDIF.runTool(args);
230      }
231    
232    
233    
234      /**
235       * Creates a new instance of this tool.
236       *
237       * @param  outStream  The output stream to which standard out should be
238       *                    written.  It may be {@code null} if output should be
239       *                    suppressed.
240       * @param  errStream  The output stream to which standard error should be
241       *                    written.  It may be {@code null} if error messages
242       *                    should be suppressed.
243       */
244      public ValidateLDIF(final OutputStream outStream,
245                          final OutputStream errStream)
246      {
247        super(outStream, errStream);
248      }
249    
250    
251    
252      /**
253       * Retrieves the name for this tool.
254       *
255       * @return  The name for this tool.
256       */
257      @Override()
258      public String getToolName()
259      {
260        return "validate-ldif";
261      }
262    
263    
264    
265      /**
266       * Retrieves the description for this tool.
267       *
268       * @return  The description for this tool.
269       */
270      @Override()
271      public String getToolDescription()
272      {
273        return "Validate the contents of an LDIF file " +
274               "against the server schema.";
275      }
276    
277    
278    
279      /**
280       * Retrieves the version string for this tool.
281       *
282       * @return  The version string for this tool.
283       */
284      @Override()
285      public String getToolVersion()
286      {
287        return Version.NUMERIC_VERSION_STRING;
288      }
289    
290    
291    
292      /**
293       * Indicates whether this tool should provide support for an interactive mode,
294       * in which the tool offers a mode in which the arguments can be provided in
295       * a text-driven menu rather than requiring them to be given on the command
296       * line.  If interactive mode is supported, it may be invoked using the
297       * "--interactive" argument.  Alternately, if interactive mode is supported
298       * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
299       * interactive mode may be invoked by simply launching the tool without any
300       * arguments.
301       *
302       * @return  {@code true} if this tool supports interactive mode, or
303       *          {@code false} if not.
304       */
305      @Override()
306      public boolean supportsInteractiveMode()
307      {
308        return true;
309      }
310    
311    
312    
313      /**
314       * Indicates whether this tool defaults to launching in interactive mode if
315       * the tool is invoked without any command-line arguments.  This will only be
316       * used if {@link #supportsInteractiveMode()} returns {@code true}.
317       *
318       * @return  {@code true} if this tool defaults to using interactive mode if
319       *          launched without any command-line arguments, or {@code false} if
320       *          not.
321       */
322      @Override()
323      public boolean defaultsToInteractiveMode()
324      {
325        return true;
326      }
327    
328    
329    
330      /**
331       * Indicates whether this tool should provide arguments for redirecting output
332       * to a file.  If this method returns {@code true}, then the tool will offer
333       * an "--outputFile" argument that will specify the path to a file to which
334       * all standard output and standard error content will be written, and it will
335       * also offer a "--teeToStandardOut" argument that can only be used if the
336       * "--outputFile" argument is present and will cause all output to be written
337       * to both the specified output file and to standard output.
338       *
339       * @return  {@code true} if this tool should provide arguments for redirecting
340       *          output to a file, or {@code false} if not.
341       */
342      @Override()
343      protected boolean supportsOutputFile()
344      {
345        return true;
346      }
347    
348    
349    
350      /**
351       * Indicates whether this tool should default to interactively prompting for
352       * the bind password if a password is required but no argument was provided
353       * to indicate how to get the password.
354       *
355       * @return  {@code true} if this tool should default to interactively
356       *          prompting for the bind password, or {@code false} if not.
357       */
358      @Override()
359      protected boolean defaultToPromptForBindPassword()
360      {
361        return true;
362      }
363    
364    
365    
366      /**
367       * Indicates whether this tool supports the use of a properties file for
368       * specifying default values for arguments that aren't specified on the
369       * command line.
370       *
371       * @return  {@code true} if this tool supports the use of a properties file
372       *          for specifying default values for arguments that aren't specified
373       *          on the command line, or {@code false} if not.
374       */
375      @Override()
376      public boolean supportsPropertiesFile()
377      {
378        return true;
379      }
380    
381    
382    
383      /**
384       * Indicates whether the LDAP-specific arguments should include alternate
385       * versions of all long identifiers that consist of multiple words so that
386       * they are available in both camelCase and dash-separated versions.
387       *
388       * @return  {@code true} if this tool should provide multiple versions of
389       *          long identifiers for LDAP-specific arguments, or {@code false} if
390       *          not.
391       */
392      @Override()
393      protected boolean includeAlternateLongIdentifiers()
394      {
395        return true;
396      }
397    
398    
399    
400      /**
401       * Adds the arguments used by this program that aren't already provided by the
402       * generic {@code LDAPCommandLineTool} framework.
403       *
404       * @param  parser  The argument parser to which the arguments should be added.
405       *
406       * @throws  ArgumentException  If a problem occurs while adding the arguments.
407       */
408      @Override()
409      public void addNonLDAPArguments(final ArgumentParser parser)
410             throws ArgumentException
411      {
412        String description = "The path to the LDIF file to process.";
413        ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
414                                    true, true, true, false);
415        ldifFile.addLongIdentifier("ldif-file");
416        parser.addArgument(ldifFile);
417    
418        description = "Indicates that the specified LDIF file is compressed " +
419                      "using gzip compression.";
420        isCompressed = new BooleanArgument('c', "isCompressed", description);
421        isCompressed.addLongIdentifier("is-compressed");
422        parser.addArgument(isCompressed);
423    
424        description = "The path to the file to which rejected entries should be " +
425                      "written.";
426        rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
427                                      description, false, true, true, false);
428        rejectFile.addLongIdentifier("reject-file");
429        parser.addArgument(rejectFile);
430    
431        description = "The path to a directory containing one or more LDIF files " +
432                      "with the schema information to use.  If this is provided, " +
433                      "then no LDAP communication will be performed.";
434        schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
435             "{path}", description, true, true, false, true);
436        schemaDirectory.addLongIdentifier("schema-directory");
437        parser.addArgument(schemaDirectory);
438    
439        description = "The number of threads to use when processing the LDIF file.";
440        numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
441             description, 1, Integer.MAX_VALUE, 1);
442        numThreads.addLongIdentifier("num-threads");
443        parser.addArgument(numThreads);
444    
445        description = "Ignore validation failures due to entries containing " +
446                      "duplicate values for the same attribute.";
447        ignoreDuplicateValues =
448             new BooleanArgument(null, "ignoreDuplicateValues", description);
449        ignoreDuplicateValues.setArgumentGroupName(
450             "Validation Strictness Arguments");
451        ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values");
452        parser.addArgument(ignoreDuplicateValues);
453    
454        description = "Ignore validation failures due to object classes not " +
455                      "defined in the schema.";
456        ignoreUndefinedObjectClasses =
457             new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
458        ignoreUndefinedObjectClasses.setArgumentGroupName(
459             "Validation Strictness Arguments");
460        ignoreUndefinedObjectClasses.addLongIdentifier(
461             "ignore-undefined-object-classes");
462        parser.addArgument(ignoreUndefinedObjectClasses);
463    
464        description = "Ignore validation failures due to attributes not defined " +
465                      "in the schema.";
466        ignoreUndefinedAttributes =
467             new BooleanArgument(null, "ignoreUndefinedAttributes", description);
468        ignoreUndefinedAttributes.setArgumentGroupName(
469             "Validation Strictness Arguments");
470        ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes");
471        parser.addArgument(ignoreUndefinedAttributes);
472    
473        description = "Ignore validation failures due to entries with malformed " +
474                      "DNs.";
475        ignoreMalformedDNs =
476             new BooleanArgument(null, "ignoreMalformedDNs", description);
477        ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
478        ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns");
479        parser.addArgument(ignoreMalformedDNs);
480    
481        description = "Ignore validation failures due to entries with RDN " +
482                      "attribute values that are missing from the set of entry " +
483                      "attributes.";
484        ignoreMissingRDNValues =
485             new BooleanArgument(null, "ignoreMissingRDNValues", description);
486        ignoreMissingRDNValues.setArgumentGroupName(
487             "Validation Strictness Arguments");
488        ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values");
489        parser.addArgument(ignoreMissingRDNValues);
490    
491        description = "Ignore validation failures due to entries without exactly " +
492                      "structural object class.";
493        ignoreStructuralObjectClasses =
494             new BooleanArgument(null, "ignoreStructuralObjectClasses",
495                                 description);
496        ignoreStructuralObjectClasses.setArgumentGroupName(
497             "Validation Strictness Arguments");
498        ignoreStructuralObjectClasses.addLongIdentifier(
499             "ignore-structural-object-classes");
500        parser.addArgument(ignoreStructuralObjectClasses);
501    
502        description = "Ignore validation failures due to entries with object " +
503                      "classes that are not allowed.";
504        ignoreProhibitedObjectClasses =
505             new BooleanArgument(null, "ignoreProhibitedObjectClasses",
506                                 description);
507        ignoreProhibitedObjectClasses.setArgumentGroupName(
508             "Validation Strictness Arguments");
509        ignoreProhibitedObjectClasses.addLongIdentifier(
510             "ignore-prohibited-object-classes");
511        parser.addArgument(ignoreProhibitedObjectClasses);
512    
513        description = "Ignore validation failures due to entries that are " +
514                      "one or more superior object classes.";
515        ignoreMissingSuperiorObjectClasses =
516             new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
517                  description);
518        ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
519             "Validation Strictness Arguments");
520        ignoreMissingSuperiorObjectClasses.addLongIdentifier(
521             "ignore-missing-superior-object-classes");
522        parser.addArgument(ignoreMissingSuperiorObjectClasses);
523    
524        description = "Ignore validation failures due to entries with attributes " +
525                      "that are not allowed.";
526        ignoreProhibitedAttributes =
527             new BooleanArgument(null, "ignoreProhibitedAttributes", description);
528        ignoreProhibitedAttributes.setArgumentGroupName(
529             "Validation Strictness Arguments");
530        ignoreProhibitedAttributes.addLongIdentifier(
531             "ignore-prohibited-attributes");
532        parser.addArgument(ignoreProhibitedAttributes);
533    
534        description = "Ignore validation failures due to entries missing " +
535                      "required attributes.";
536        ignoreMissingAttributes =
537             new BooleanArgument(null, "ignoreMissingAttributes", description);
538        ignoreMissingAttributes.setArgumentGroupName(
539             "Validation Strictness Arguments");
540        ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes");
541        parser.addArgument(ignoreMissingAttributes);
542    
543        description = "Ignore validation failures due to entries with multiple " +
544                      "values for single-valued attributes.";
545        ignoreSingleValuedAttributes =
546             new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
547        ignoreSingleValuedAttributes.setArgumentGroupName(
548             "Validation Strictness Arguments");
549        ignoreSingleValuedAttributes.addLongIdentifier(
550             "ignore-single-valued-attributes");
551        parser.addArgument(ignoreSingleValuedAttributes);
552    
553        description = "Ignore validation failures due to entries with attribute " +
554                      "values that violate their associated syntax.  If this is " +
555                      "provided, then no attribute syntax violations will be " +
556                      "flagged.  If this is not provided, then all attribute " +
557                      "syntax violations will be flagged except for violations " +
558                      "in those attributes excluded by the " +
559                      "--ignoreSyntaxViolationsForAttribute argument.";
560        ignoreAttributeSyntax =
561             new BooleanArgument(null, "ignoreAttributeSyntax", description);
562        ignoreAttributeSyntax.setArgumentGroupName(
563             "Validation Strictness Arguments");
564        ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax");
565        parser.addArgument(ignoreAttributeSyntax);
566    
567        description = "The name or OID of an attribute for which to ignore " +
568                      "validation failures due to violations of the associated " +
569                      "attribute syntax.  This argument can only be used if the " +
570                      "--ignoreAttributeSyntax argument is not provided.";
571        ignoreSyntaxViolationsForAttribute = new StringArgument(null,
572             "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
573        ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
574             "Validation Strictness Arguments");
575        ignoreSyntaxViolationsForAttribute.addLongIdentifier(
576             "ignore-syntax-violations-for-attribute");
577        parser.addArgument(ignoreSyntaxViolationsForAttribute);
578    
579        description = "Ignore validation failures due to entries with RDNs " +
580                      "that violate the associated name form definition.";
581        ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
582        ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
583        ignoreNameForms.addLongIdentifier("ignore-name-forms");
584        parser.addArgument(ignoreNameForms);
585    
586    
587        // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
588        // cannot be used together.
589        parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
590             ignoreSyntaxViolationsForAttribute);
591      }
592    
593    
594    
595      /**
596       * Performs the actual processing for this tool.  In this case, it gets a
597       * connection to the directory server and uses it to retrieve the server
598       * schema.  It then reads the LDIF file and validates each entry accordingly.
599       *
600       * @return  The result code for the processing that was performed.
601       */
602      @Override()
603      public ResultCode doToolProcessing()
604      {
605        // Get the connection to the directory server and use it to read the schema.
606        final Schema schema;
607        if (schemaDirectory.isPresent())
608        {
609          final File schemaDir = schemaDirectory.getValue();
610    
611          try
612          {
613            final TreeMap<String,File> fileMap = new TreeMap<String,File>();
614            for (final File f : schemaDir.listFiles())
615            {
616              final String name = f.getName();
617              if (f.isFile() && name.endsWith(".ldif"))
618              {
619                fileMap.put(name, f);
620              }
621            }
622    
623            if (fileMap.isEmpty())
624            {
625              err("No LDIF files found in directory " +
626                  schemaDir.getAbsolutePath());
627              return ResultCode.PARAM_ERROR;
628            }
629    
630            final ArrayList<File> fileList = new ArrayList<File>(fileMap.values());
631            schema = Schema.getSchema(fileList);
632          }
633          catch (Exception e)
634          {
635            err("Unable to read schema from files in directory " +
636                schemaDir.getAbsolutePath() + ":  " + getExceptionMessage(e));
637            return ResultCode.LOCAL_ERROR;
638          }
639        }
640        else
641        {
642          try
643          {
644            final LDAPConnection connection = getConnection();
645            schema = connection.getSchema();
646            connection.close();
647          }
648          catch (LDAPException le)
649          {
650            err("Unable to connect to the directory server and read the schema:  ",
651                le.getMessage());
652            return le.getResultCode();
653          }
654        }
655    
656    
657        // Create the entry validator and initialize its configuration.
658        entryValidator = new EntryValidator(schema);
659        entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
660        entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
661        entryValidator.setCheckEntryMissingRDNValues(
662             !ignoreMissingRDNValues.isPresent());
663        entryValidator.setCheckMissingAttributes(
664             !ignoreMissingAttributes.isPresent());
665        entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
666        entryValidator.setCheckProhibitedAttributes(
667             !ignoreProhibitedAttributes.isPresent());
668        entryValidator.setCheckProhibitedObjectClasses(
669             !ignoreProhibitedObjectClasses.isPresent());
670        entryValidator.setCheckMissingSuperiorObjectClasses(
671             !ignoreMissingSuperiorObjectClasses.isPresent());
672        entryValidator.setCheckSingleValuedAttributes(
673             !ignoreSingleValuedAttributes.isPresent());
674        entryValidator.setCheckStructuralObjectClasses(
675             !ignoreStructuralObjectClasses.isPresent());
676        entryValidator.setCheckUndefinedAttributes(
677             !ignoreUndefinedAttributes.isPresent());
678        entryValidator.setCheckUndefinedObjectClasses(
679             !ignoreUndefinedObjectClasses.isPresent());
680    
681        if (ignoreSyntaxViolationsForAttribute.isPresent())
682        {
683          entryValidator.setIgnoreSyntaxViolationAttributeTypes(
684               ignoreSyntaxViolationsForAttribute.getValues());
685        }
686    
687    
688        // Create an LDIF reader that can be used to read through the LDIF file.
689        final LDIFReader ldifReader;
690        rejectWriter = null;
691        try
692        {
693          InputStream inputStream = new FileInputStream(ldifFile.getValue());
694          if (isCompressed.isPresent())
695          {
696            inputStream = new GZIPInputStream(inputStream);
697          }
698          ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
699        }
700        catch (Exception e)
701        {
702          err("Unable to open the LDIF reader:  ", getExceptionMessage(e));
703          return ResultCode.LOCAL_ERROR;
704        }
705    
706        ldifReader.setSchema(schema);
707        if (ignoreDuplicateValues.isPresent())
708        {
709          ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
710        }
711        else
712        {
713          ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
714        }
715    
716        try
717        {
718          // Create an LDIF writer that can be used to write information about
719          // rejected entries.
720          try
721          {
722            if (rejectFile.isPresent())
723            {
724              rejectWriter = new LDIFWriter(rejectFile.getValue());
725            }
726          }
727          catch (Exception e)
728          {
729            err("Unable to create the reject writer:  ", getExceptionMessage(e));
730            return ResultCode.LOCAL_ERROR;
731          }
732    
733          ResultCode resultCode = ResultCode.SUCCESS;
734          while (true)
735          {
736            try
737            {
738              final Entry e = ldifReader.readEntry();
739              if (e == null)
740              {
741                // Because we're performing parallel processing and returning null
742                // from the translate method, LDIFReader.readEntry() should never
743                // return a non-null value.  However, it can throw an LDIFException
744                // if it encounters an invalid entry, or an IOException if there's
745                // a problem reading from the file, so we should still iterate
746                // through all of the entries to catch and report on those problems.
747                break;
748              }
749            }
750            catch (LDIFException le)
751            {
752              malformedEntries.incrementAndGet();
753    
754              if (resultCode == ResultCode.SUCCESS)
755              {
756                resultCode = ResultCode.DECODING_ERROR;
757              }
758    
759              if (rejectWriter != null)
760              {
761                try
762                {
763                  rejectWriter.writeComment(
764                       "Unable to parse an entry read from LDIF:", false, false);
765                  if (le.mayContinueReading())
766                  {
767                    rejectWriter.writeComment(getExceptionMessage(le), false, true);
768                  }
769                  else
770                  {
771                    rejectWriter.writeComment(getExceptionMessage(le), false,
772                                              false);
773                    rejectWriter.writeComment("Unable to continue LDIF processing.",
774                                              false, true);
775                    err("Aborting LDIF processing:  ", getExceptionMessage(le));
776                    return ResultCode.LOCAL_ERROR;
777                  }
778                }
779                catch (IOException ioe)
780                {
781                  err("Unable to write to the reject file:",
782                      getExceptionMessage(ioe));
783                  err("LDIF parse failure that triggered the rejection:  ",
784                      getExceptionMessage(le));
785                  return ResultCode.LOCAL_ERROR;
786                }
787              }
788            }
789            catch (IOException ioe)
790            {
791    
792              if (rejectWriter != null)
793              {
794                try
795                {
796                  rejectWriter.writeComment("I/O error reading from LDIF:", false,
797                                            false);
798                  rejectWriter.writeComment(getExceptionMessage(ioe), false,
799                                            true);
800                  return ResultCode.LOCAL_ERROR;
801                }
802                catch (Exception ex)
803                {
804                  err("I/O error reading from LDIF:", getExceptionMessage(ioe));
805                  return ResultCode.LOCAL_ERROR;
806                }
807              }
808            }
809          }
810    
811          if (malformedEntries.get() > 0)
812          {
813            out(malformedEntries.get() + " entries were malformed and could not " +
814                "be read from the LDIF file.");
815          }
816    
817          if (entryValidator.getInvalidEntries() > 0)
818          {
819            if (resultCode == ResultCode.SUCCESS)
820            {
821              resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
822            }
823    
824            for (final String s : entryValidator.getInvalidEntrySummary(true))
825            {
826              out(s);
827            }
828          }
829          else
830          {
831            if (malformedEntries.get() == 0)
832            {
833              out("No errors were encountered.");
834            }
835          }
836    
837          return resultCode;
838        }
839        finally
840        {
841          try
842          {
843            ldifReader.close();
844          }
845          catch (Exception e) {}
846    
847          try
848          {
849            if (rejectWriter != null)
850            {
851              rejectWriter.close();
852            }
853          }
854          catch (Exception e) {}
855        }
856      }
857    
858    
859    
860      /**
861       * Examines the provided entry to determine whether it conforms to the
862       * server schema.
863       *
864       * @param  entry           The entry to be examined.
865       * @param  firstLineNumber The line number of the LDIF source on which the
866       *                         provided entry begins.
867       *
868       * @return  The updated entry.  This method will always return {@code null}
869       *          because all of the real processing needed for the entry is
870       *          performed in this method and the entry isn't needed any more
871       *          after this method is done.
872       */
873      public Entry translate(final Entry entry, final long firstLineNumber)
874      {
875        final ArrayList<String> invalidReasons = new ArrayList<String>(5);
876        if (! entryValidator.entryIsValid(entry, invalidReasons))
877        {
878          if (rejectWriter != null)
879          {
880            synchronized (this)
881            {
882              try
883              {
884                rejectWriter.writeEntry(entry, listToString(invalidReasons));
885              }
886              catch (IOException ioe) {}
887            }
888          }
889        }
890    
891        final long numEntries = entriesProcessed.incrementAndGet();
892        if ((numEntries % 1000L) == 0L)
893        {
894          out("Processed ", numEntries, " entries.");
895        }
896    
897        return null;
898      }
899    
900    
901    
902      /**
903       * Converts the provided list of strings into a single string.  It will
904       * contain line breaks after all but the last element.
905       *
906       * @param  l  The list of strings to convert to a single string.
907       *
908       * @return  The string from the provided list, or {@code null} if the provided
909       *          list is empty or {@code null}.
910       */
911      private static String listToString(final List<String> l)
912      {
913        if ((l == null) || (l.isEmpty()))
914        {
915          return null;
916        }
917    
918        final StringBuilder buffer = new StringBuilder();
919        final Iterator<String> iterator = l.iterator();
920        while (iterator.hasNext())
921        {
922          buffer.append(iterator.next());
923          if (iterator.hasNext())
924          {
925            buffer.append(EOL);
926          }
927        }
928    
929        return buffer.toString();
930      }
931    
932    
933    
934      /**
935       * {@inheritDoc}
936       */
937      @Override()
938      public LinkedHashMap<String[],String> getExampleUsages()
939      {
940        final LinkedHashMap<String[],String> examples =
941             new LinkedHashMap<String[],String>(2);
942    
943        String[] args =
944        {
945          "--hostname", "server.example.com",
946          "--port", "389",
947          "--ldifFile", "data.ldif",
948          "--rejectFile", "rejects.ldif",
949          "--numThreads", "4"
950        };
951        String description =
952             "Validate the contents of the 'data.ldif' file using the schema " +
953             "defined in the specified directory server using four concurrent " +
954             "threads.  All types of validation will be performed, and " +
955             "information about any errors will be written to the 'rejects.ldif' " +
956             "file.";
957        examples.put(args, description);
958    
959    
960        args = new String[]
961        {
962          "--schemaDirectory", "/ds/config/schema",
963          "--ldifFile", "data.ldif",
964          "--rejectFile", "rejects.ldif",
965          "--ignoreStructuralObjectClasses",
966          "--ignoreAttributeSyntax"
967        };
968        description =
969             "Validate the contents of the 'data.ldif' file using the schema " +
970             "defined in LDIF files contained in the /ds/config/schema directory " +
971             "using a single thread.  Any errors resulting from entries that do " +
972             "not have exactly one structural object class or from values which " +
973             "violate the syntax for their associated attribute types will be " +
974             "ignored.  Information about any other failures will be written to " +
975             "the 'rejects.ldif' file.";
976        examples.put(args, description);
977    
978        return examples;
979      }
980    
981    
982    
983      /**
984       * @return EntryValidator
985       *
986       * Returns the EntryValidator
987       */
988      public EntryValidator getEntryValidator()
989      {
990        return entryValidator;
991      }
992    }