001/*
002 * Copyright 2008-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2008-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2008-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.examples;
037
038
039
040import java.io.File;
041import java.io.FileInputStream;
042import java.io.InputStream;
043import java.io.IOException;
044import java.io.OutputStream;
045import java.util.ArrayList;
046import java.util.Iterator;
047import java.util.TreeMap;
048import java.util.LinkedHashMap;
049import java.util.List;
050import java.util.concurrent.atomic.AtomicLong;
051import java.util.zip.GZIPInputStream;
052
053import com.unboundid.ldap.sdk.Entry;
054import com.unboundid.ldap.sdk.LDAPConnection;
055import com.unboundid.ldap.sdk.LDAPException;
056import com.unboundid.ldap.sdk.ResultCode;
057import com.unboundid.ldap.sdk.Version;
058import com.unboundid.ldap.sdk.schema.Schema;
059import com.unboundid.ldap.sdk.schema.EntryValidator;
060import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
061import com.unboundid.ldif.DuplicateValueBehavior;
062import com.unboundid.ldif.LDIFException;
063import com.unboundid.ldif.LDIFReader;
064import com.unboundid.ldif.LDIFReaderEntryTranslator;
065import com.unboundid.ldif.LDIFWriter;
066import com.unboundid.util.Debug;
067import com.unboundid.util.LDAPCommandLineTool;
068import com.unboundid.util.NotNull;
069import com.unboundid.util.Nullable;
070import com.unboundid.util.StaticUtils;
071import com.unboundid.util.ThreadSafety;
072import com.unboundid.util.ThreadSafetyLevel;
073import com.unboundid.util.args.ArgumentException;
074import com.unboundid.util.args.ArgumentParser;
075import com.unboundid.util.args.BooleanArgument;
076import com.unboundid.util.args.FileArgument;
077import com.unboundid.util.args.IntegerArgument;
078import com.unboundid.util.args.StringArgument;
079
080
081
082/**
083 * This class provides a simple tool that can be used to validate that the
084 * contents of an LDIF file are valid.  This includes ensuring that the contents
085 * can be parsed as valid LDIF, and it can also ensure that the LDIF content
086 * conforms to the server schema.  It will obtain the schema by connecting to
087 * the server and retrieving the default schema (i.e., the schema which governs
088 * the root DSE).  By default, a thorough set of validation will be performed,
089 * but it is possible to disable certain types of validation.
090 * <BR><BR>
091 * Some of the APIs demonstrated by this example include:
092 * <UL>
093 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
094 *       package)</LI>
095 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
096 *       package)</LI>
097 *   <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI>
098 *   <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema}
099 *       package)</LI>
100 * </UL>
101 * <BR><BR>
102 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
103 * class (to obtain the information to use to connect to the server to read the
104 * schema), as well as the following additional arguments:
105 * <UL>
106 *   <LI>"--schemaDirectory {path}" -- specifies the path to a directory
107 *       containing files with schema definitions.  If this argument is
108 *       provided, then no attempt will be made to communicate with a directory
109 *       server.</LI>
110 *   <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF
111 *       file to be validated.</LI>
112 *   <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is
113 *       compressed.</LI>
114 *   <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file
115 *       to be written with information about all entries that failed
116 *       validation.</LI>
117 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
118 *       concurrent threads to use when processing the LDIF.  If this is not
119 *       provided, then a default of one thread will be used.</LI>
120 *   <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation
121 *       process should ignore validation failures due to entries that contain
122 *       object classes not defined in the server schema.</LI>
123 *   <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process
124 *       should ignore validation failures due to entries that contain
125 *       attributes not defined in the server schema.</LI>
126 *   <LI>"--ignoreMalformedDNs" -- indicates that the validation process should
127 *       ignore validation failures due to entries with malformed DNs.</LI>
128 *   <LI>"--ignoreMissingRDNValues" -- indicates that the validation process
129 *       should ignore validation failures due to entries that contain an RDN
130 *       attribute value that is not present in the set of entry
131 *       attributes.</LI>
132 *   <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation
133 *       process should ignore validation failures due to entries that either do
134 *       not have a structural object class or that have multiple structural
135 *       object classes.</LI>
136 *   <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation
137 *       process should ignore validation failures due to entries containing
138 *       auxiliary classes that are not allowed by a DIT content rule, or
139 *       abstract classes that are not subclassed by an auxiliary or structural
140 *       class contained in the entry.</LI>
141 *   <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process
142 *       should ignore validation failures due to entries including attributes
143 *       that are not allowed or are explicitly prohibited by a DIT content
144 *       rule.</LI>
145 *   <LI>"--ignoreMissingAttributes" -- indicates that the validation process
146 *       should ignore validation failures due to entries missing required
147 *       attributes.</LI>
148 *   <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation
149 *       process should ignore validation failures due to single-valued
150 *       attributes containing multiple values.</LI>
151 *   <LI>"--ignoreAttributeSyntax" -- indicates that the validation process
152 *       should ignore validation failures due to attribute values which violate
153 *       the associated attribute syntax.</LI>
154 *   <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation
155 *       process should ignore validation failures due to attribute values which
156 *       violate the associated attribute syntax, but only for the specified
157 *       attribute types.</LI>
158 *   <LI>"--ignoreNameForms" -- indicates that the validation process should
159 *       ignore validation failures due to name form violations (in which the
160 *       entry's RDN does not comply with the associated name form).</LI>
161 * </UL>
162 */
163@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
164public final class ValidateLDIF
165       extends LDAPCommandLineTool
166       implements LDIFReaderEntryTranslator
167{
168  /**
169   * The end-of-line character for this platform.
170   */
171  @NotNull private static final String EOL =
172       StaticUtils.getSystemProperty("line.separator", "\n");
173
174
175
176  // The arguments used by this program.
177  @Nullable private BooleanArgument ignoreDuplicateValues;
178  @Nullable private BooleanArgument ignoreUndefinedObjectClasses;
179  @Nullable private BooleanArgument ignoreUndefinedAttributes;
180  @Nullable private BooleanArgument ignoreMalformedDNs;
181  @Nullable private BooleanArgument ignoreMissingRDNValues;
182  @Nullable private BooleanArgument ignoreMissingSuperiorObjectClasses;
183  @Nullable private BooleanArgument ignoreStructuralObjectClasses;
184  @Nullable private BooleanArgument ignoreProhibitedObjectClasses;
185  @Nullable private BooleanArgument ignoreProhibitedAttributes;
186  @Nullable private BooleanArgument ignoreMissingAttributes;
187  @Nullable private BooleanArgument ignoreSingleValuedAttributes;
188  @Nullable private BooleanArgument ignoreAttributeSyntax;
189  @Nullable private BooleanArgument ignoreNameForms;
190  @Nullable private BooleanArgument isCompressed;
191  @Nullable private FileArgument    schemaDirectory;
192  @Nullable private FileArgument    ldifFile;
193  @Nullable private FileArgument    rejectFile;
194  @Nullable private FileArgument    encryptionPassphraseFile;
195  @Nullable private IntegerArgument numThreads;
196  @Nullable private StringArgument  ignoreSyntaxViolationsForAttribute;
197
198  // The counter used to keep track of the number of entries processed.
199  @NotNull private final AtomicLong entriesProcessed = new AtomicLong(0L);
200
201  // The counter used to keep track of the number of entries that could not be
202  // parsed as valid entries.
203  @NotNull private final AtomicLong malformedEntries = new AtomicLong(0L);
204
205  // The entry validator that will be used to validate the entries.
206  @Nullable private EntryValidator entryValidator;
207
208  // The LDIF writer that will be used to write rejected entries.
209  @Nullable private LDIFWriter rejectWriter;
210
211
212
213  /**
214   * Parse the provided command line arguments and make the appropriate set of
215   * changes.
216   *
217   * @param  args  The command line arguments provided to this program.
218   */
219  public static void main(@NotNull final String[] args)
220  {
221    final ResultCode resultCode = main(args, System.out, System.err);
222    if (resultCode != ResultCode.SUCCESS)
223    {
224      System.exit(resultCode.intValue());
225    }
226  }
227
228
229
230  /**
231   * Parse the provided command line arguments and make the appropriate set of
232   * changes.
233   *
234   * @param  args       The command line arguments provided to this program.
235   * @param  outStream  The output stream to which standard out should be
236   *                    written.  It may be {@code null} if output should be
237   *                    suppressed.
238   * @param  errStream  The output stream to which standard error should be
239   *                    written.  It may be {@code null} if error messages
240   *                    should be suppressed.
241   *
242   * @return  A result code indicating whether the processing was successful.
243   */
244  @NotNull()
245  public static ResultCode main(@NotNull final String[] args,
246                                @Nullable final OutputStream outStream,
247                                @Nullable final OutputStream errStream)
248  {
249    final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream);
250    return validateLDIF.runTool(args);
251  }
252
253
254
255  /**
256   * Creates a new instance of this tool.
257   *
258   * @param  outStream  The output stream to which standard out should be
259   *                    written.  It may be {@code null} if output should be
260   *                    suppressed.
261   * @param  errStream  The output stream to which standard error should be
262   *                    written.  It may be {@code null} if error messages
263   *                    should be suppressed.
264   */
265  public ValidateLDIF(@Nullable final OutputStream outStream,
266                      @Nullable final OutputStream errStream)
267  {
268    super(outStream, errStream);
269  }
270
271
272
273  /**
274   * Retrieves the name for this tool.
275   *
276   * @return  The name for this tool.
277   */
278  @Override()
279  @NotNull()
280  public String getToolName()
281  {
282    return "validate-ldif";
283  }
284
285
286
287  /**
288   * Retrieves the description for this tool.
289   *
290   * @return  The description for this tool.
291   */
292  @Override()
293  @NotNull()
294  public String getToolDescription()
295  {
296    return "Validate the contents of an LDIF file " +
297           "against the server schema.";
298  }
299
300
301
302  /**
303   * Retrieves the version string for this tool.
304   *
305   * @return  The version string for this tool.
306   */
307  @Override()
308  @NotNull()
309  public String getToolVersion()
310  {
311    return Version.NUMERIC_VERSION_STRING;
312  }
313
314
315
316  /**
317   * Indicates whether this tool should provide support for an interactive mode,
318   * in which the tool offers a mode in which the arguments can be provided in
319   * a text-driven menu rather than requiring them to be given on the command
320   * line.  If interactive mode is supported, it may be invoked using the
321   * "--interactive" argument.  Alternately, if interactive mode is supported
322   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
323   * interactive mode may be invoked by simply launching the tool without any
324   * arguments.
325   *
326   * @return  {@code true} if this tool supports interactive mode, or
327   *          {@code false} if not.
328   */
329  @Override()
330  public boolean supportsInteractiveMode()
331  {
332    return true;
333  }
334
335
336
337  /**
338   * Indicates whether this tool defaults to launching in interactive mode if
339   * the tool is invoked without any command-line arguments.  This will only be
340   * used if {@link #supportsInteractiveMode()} returns {@code true}.
341   *
342   * @return  {@code true} if this tool defaults to using interactive mode if
343   *          launched without any command-line arguments, or {@code false} if
344   *          not.
345   */
346  @Override()
347  public boolean defaultsToInteractiveMode()
348  {
349    return true;
350  }
351
352
353
354  /**
355   * Indicates whether this tool should provide arguments for redirecting output
356   * to a file.  If this method returns {@code true}, then the tool will offer
357   * an "--outputFile" argument that will specify the path to a file to which
358   * all standard output and standard error content will be written, and it will
359   * also offer a "--teeToStandardOut" argument that can only be used if the
360   * "--outputFile" argument is present and will cause all output to be written
361   * to both the specified output file and to standard output.
362   *
363   * @return  {@code true} if this tool should provide arguments for redirecting
364   *          output to a file, or {@code false} if not.
365   */
366  @Override()
367  protected boolean supportsOutputFile()
368  {
369    return true;
370  }
371
372
373
374  /**
375   * Indicates whether this tool should default to interactively prompting for
376   * the bind password if a password is required but no argument was provided
377   * to indicate how to get the password.
378   *
379   * @return  {@code true} if this tool should default to interactively
380   *          prompting for the bind password, or {@code false} if not.
381   */
382  @Override()
383  protected boolean defaultToPromptForBindPassword()
384  {
385    return true;
386  }
387
388
389
390  /**
391   * Indicates whether this tool supports the use of a properties file for
392   * specifying default values for arguments that aren't specified on the
393   * command line.
394   *
395   * @return  {@code true} if this tool supports the use of a properties file
396   *          for specifying default values for arguments that aren't specified
397   *          on the command line, or {@code false} if not.
398   */
399  @Override()
400  public boolean supportsPropertiesFile()
401  {
402    return true;
403  }
404
405
406
407  /**
408   * Indicates whether the LDAP-specific arguments should include alternate
409   * versions of all long identifiers that consist of multiple words so that
410   * they are available in both camelCase and dash-separated versions.
411   *
412   * @return  {@code true} if this tool should provide multiple versions of
413   *          long identifiers for LDAP-specific arguments, or {@code false} if
414   *          not.
415   */
416  @Override()
417  protected boolean includeAlternateLongIdentifiers()
418  {
419    return true;
420  }
421
422
423
424  /**
425   * Indicates whether this tool should provide a command-line argument that
426   * allows for low-level SSL debugging.  If this returns {@code true}, then an
427   * "--enableSSLDebugging}" argument will be added that sets the
428   * "javax.net.debug" system property to "all" before attempting any
429   * communication.
430   *
431   * @return  {@code true} if this tool should offer an "--enableSSLDebugging"
432   *          argument, or {@code false} if not.
433   */
434  @Override()
435  protected boolean supportsSSLDebugging()
436  {
437    return true;
438  }
439
440
441
442  /**
443   * Adds the arguments used by this program that aren't already provided by the
444   * generic {@code LDAPCommandLineTool} framework.
445   *
446   * @param  parser  The argument parser to which the arguments should be added.
447   *
448   * @throws  ArgumentException  If a problem occurs while adding the arguments.
449   */
450  @Override()
451  public void addNonLDAPArguments(@NotNull final ArgumentParser parser)
452         throws ArgumentException
453  {
454    String description = "The path to the LDIF file to process.  The tool " +
455         "will automatically attempt to detect whether the file is " +
456         "encrypted or compressed.";
457    ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description,
458                                true, true, true, false);
459    ldifFile.addLongIdentifier("ldif-file", true);
460    parser.addArgument(ldifFile);
461
462
463    // Add an argument that makes it possible to read a compressed LDIF file.
464    // Note that this argument is no longer needed for dealing with compressed
465    // files, since the tool will automatically detect whether a file is
466    // compressed.  However, the argument is still provided for the purpose of
467    // backward compatibility.
468    description = "Indicates that the specified LDIF file is compressed " +
469                  "using gzip compression.";
470    isCompressed = new BooleanArgument('c', "isCompressed", description);
471    isCompressed.addLongIdentifier("is-compressed", true);
472    isCompressed.setHidden(true);
473    parser.addArgument(isCompressed);
474
475
476    // Add an argument that indicates that the tool should read the encryption
477    // passphrase from a file.
478    description = "Indicates that the specified LDIF file is encrypted and " +
479         "that the encryption passphrase is contained in the specified " +
480         "file.  If the LDIF data is encrypted and this argument is not " +
481         "provided, then the tool will interactively prompt for the " +
482         "encryption passphrase.";
483    encryptionPassphraseFile = new FileArgument(null,
484         "encryptionPassphraseFile", false, 1, null, description, true, true,
485         true, false);
486    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
487         true);
488    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
489    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
490         true);
491    parser.addArgument(encryptionPassphraseFile);
492
493
494    description = "The path to the file to which rejected entries should be " +
495                  "written.";
496    rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}",
497                                  description, false, true, true, false);
498    rejectFile.addLongIdentifier("reject-file", true);
499    parser.addArgument(rejectFile);
500
501    description = "The path to a directory containing one or more LDIF files " +
502                  "with the schema information to use.  If this is provided, " +
503                  "then no LDAP communication will be performed.";
504    schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1,
505         "{path}", description, true, true, false, true);
506    schemaDirectory.addLongIdentifier("schema-directory", true);
507    parser.addArgument(schemaDirectory);
508
509    description = "The number of threads to use when processing the LDIF file.";
510    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
511         description, 1, Integer.MAX_VALUE, 1);
512    numThreads.addLongIdentifier("num-threads", true);
513    parser.addArgument(numThreads);
514
515    description = "Ignore validation failures due to entries containing " +
516                  "duplicate values for the same attribute.";
517    ignoreDuplicateValues =
518         new BooleanArgument(null, "ignoreDuplicateValues", description);
519    ignoreDuplicateValues.setArgumentGroupName(
520         "Validation Strictness Arguments");
521    ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true);
522    parser.addArgument(ignoreDuplicateValues);
523
524    description = "Ignore validation failures due to object classes not " +
525                  "defined in the schema.";
526    ignoreUndefinedObjectClasses =
527         new BooleanArgument(null, "ignoreUndefinedObjectClasses", description);
528    ignoreUndefinedObjectClasses.setArgumentGroupName(
529         "Validation Strictness Arguments");
530    ignoreUndefinedObjectClasses.addLongIdentifier(
531         "ignore-undefined-object-classes", true);
532    parser.addArgument(ignoreUndefinedObjectClasses);
533
534    description = "Ignore validation failures due to attributes not defined " +
535                  "in the schema.";
536    ignoreUndefinedAttributes =
537         new BooleanArgument(null, "ignoreUndefinedAttributes", description);
538    ignoreUndefinedAttributes.setArgumentGroupName(
539         "Validation Strictness Arguments");
540    ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes",
541         true);
542    parser.addArgument(ignoreUndefinedAttributes);
543
544    description = "Ignore validation failures due to entries with malformed " +
545                  "DNs.";
546    ignoreMalformedDNs =
547         new BooleanArgument(null, "ignoreMalformedDNs", description);
548    ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments");
549    ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true);
550    parser.addArgument(ignoreMalformedDNs);
551
552    description = "Ignore validation failures due to entries with RDN " +
553                  "attribute values that are missing from the set of entry " +
554                  "attributes.";
555    ignoreMissingRDNValues =
556         new BooleanArgument(null, "ignoreMissingRDNValues", description);
557    ignoreMissingRDNValues.setArgumentGroupName(
558         "Validation Strictness Arguments");
559    ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true);
560    parser.addArgument(ignoreMissingRDNValues);
561
562    description = "Ignore validation failures due to entries without exactly " +
563                  "structural object class.";
564    ignoreStructuralObjectClasses =
565         new BooleanArgument(null, "ignoreStructuralObjectClasses",
566                             description);
567    ignoreStructuralObjectClasses.setArgumentGroupName(
568         "Validation Strictness Arguments");
569    ignoreStructuralObjectClasses.addLongIdentifier(
570         "ignore-structural-object-classes", true);
571    parser.addArgument(ignoreStructuralObjectClasses);
572
573    description = "Ignore validation failures due to entries with object " +
574                  "classes that are not allowed.";
575    ignoreProhibitedObjectClasses =
576         new BooleanArgument(null, "ignoreProhibitedObjectClasses",
577                             description);
578    ignoreProhibitedObjectClasses.setArgumentGroupName(
579         "Validation Strictness Arguments");
580    ignoreProhibitedObjectClasses.addLongIdentifier(
581         "ignore-prohibited-object-classes", true);
582    parser.addArgument(ignoreProhibitedObjectClasses);
583
584    description = "Ignore validation failures due to entries that are " +
585                  "one or more superior object classes.";
586    ignoreMissingSuperiorObjectClasses =
587         new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses",
588              description);
589    ignoreMissingSuperiorObjectClasses.setArgumentGroupName(
590         "Validation Strictness Arguments");
591    ignoreMissingSuperiorObjectClasses.addLongIdentifier(
592         "ignore-missing-superior-object-classes", true);
593    parser.addArgument(ignoreMissingSuperiorObjectClasses);
594
595    description = "Ignore validation failures due to entries with attributes " +
596                  "that are not allowed.";
597    ignoreProhibitedAttributes =
598         new BooleanArgument(null, "ignoreProhibitedAttributes", description);
599    ignoreProhibitedAttributes.setArgumentGroupName(
600         "Validation Strictness Arguments");
601    ignoreProhibitedAttributes.addLongIdentifier(
602         "ignore-prohibited-attributes", true);
603    parser.addArgument(ignoreProhibitedAttributes);
604
605    description = "Ignore validation failures due to entries missing " +
606                  "required attributes.";
607    ignoreMissingAttributes =
608         new BooleanArgument(null, "ignoreMissingAttributes", description);
609    ignoreMissingAttributes.setArgumentGroupName(
610         "Validation Strictness Arguments");
611    ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes",
612         true);
613    parser.addArgument(ignoreMissingAttributes);
614
615    description = "Ignore validation failures due to entries with multiple " +
616                  "values for single-valued attributes.";
617    ignoreSingleValuedAttributes =
618         new BooleanArgument(null, "ignoreSingleValuedAttributes", description);
619    ignoreSingleValuedAttributes.setArgumentGroupName(
620         "Validation Strictness Arguments");
621    ignoreSingleValuedAttributes.addLongIdentifier(
622         "ignore-single-valued-attributes", true);
623    parser.addArgument(ignoreSingleValuedAttributes);
624
625    description = "Ignore validation failures due to entries with attribute " +
626                  "values that violate their associated syntax.  If this is " +
627                  "provided, then no attribute syntax violations will be " +
628                  "flagged.  If this is not provided, then all attribute " +
629                  "syntax violations will be flagged except for violations " +
630                  "in those attributes excluded by the " +
631                  "--ignoreSyntaxViolationsForAttribute argument.";
632    ignoreAttributeSyntax =
633         new BooleanArgument(null, "ignoreAttributeSyntax", description);
634    ignoreAttributeSyntax.setArgumentGroupName(
635         "Validation Strictness Arguments");
636    ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true);
637    parser.addArgument(ignoreAttributeSyntax);
638
639    description = "The name or OID of an attribute for which to ignore " +
640                  "validation failures due to violations of the associated " +
641                  "attribute syntax.  This argument can only be used if the " +
642                  "--ignoreAttributeSyntax argument is not provided.";
643    ignoreSyntaxViolationsForAttribute = new StringArgument(null,
644         "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description);
645    ignoreSyntaxViolationsForAttribute.setArgumentGroupName(
646         "Validation Strictness Arguments");
647    ignoreSyntaxViolationsForAttribute.addLongIdentifier(
648         "ignore-syntax-violations-for-attribute", true);
649    parser.addArgument(ignoreSyntaxViolationsForAttribute);
650
651    description = "Ignore validation failures due to entries with RDNs " +
652                  "that violate the associated name form definition.";
653    ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description);
654    ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments");
655    ignoreNameForms.addLongIdentifier("ignore-name-forms", true);
656    parser.addArgument(ignoreNameForms);
657
658
659    // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments
660    // cannot be used together.
661    parser.addExclusiveArgumentSet(ignoreAttributeSyntax,
662         ignoreSyntaxViolationsForAttribute);
663  }
664
665
666
667  /**
668   * Performs the actual processing for this tool.  In this case, it gets a
669   * connection to the directory server and uses it to retrieve the server
670   * schema.  It then reads the LDIF file and validates each entry accordingly.
671   *
672   * @return  The result code for the processing that was performed.
673   */
674  @Override()
675  @NotNull()
676  public ResultCode doToolProcessing()
677  {
678    // Get the connection to the directory server and use it to read the schema.
679    final Schema schema;
680    if (schemaDirectory.isPresent())
681    {
682      final File schemaDir = schemaDirectory.getValue();
683
684      try
685      {
686        final TreeMap<String,File> fileMap = new TreeMap<>();
687        for (final File f : schemaDir.listFiles())
688        {
689          final String name = f.getName();
690          if (f.isFile() && name.endsWith(".ldif"))
691          {
692            fileMap.put(name, f);
693          }
694        }
695
696        if (fileMap.isEmpty())
697        {
698          err("No LDIF files found in directory " +
699              schemaDir.getAbsolutePath());
700          return ResultCode.PARAM_ERROR;
701        }
702
703        final ArrayList<File> fileList = new ArrayList<>(fileMap.values());
704        schema = Schema.getSchema(fileList);
705      }
706      catch (final Exception e)
707      {
708        Debug.debugException(e);
709        err("Unable to read schema from files in directory " +
710            schemaDir.getAbsolutePath() + ":  " +
711             StaticUtils.getExceptionMessage(e));
712        return ResultCode.LOCAL_ERROR;
713      }
714    }
715    else
716    {
717      try
718      {
719        final LDAPConnection connection = getConnection();
720        schema = connection.getSchema();
721        connection.close();
722      }
723      catch (final LDAPException le)
724      {
725        Debug.debugException(le);
726        err("Unable to connect to the directory server and read the schema:  ",
727            le.getMessage());
728        return le.getResultCode();
729      }
730    }
731
732
733    // Get the encryption passphrase, if it was provided.
734    String encryptionPassphrase = null;
735    if (encryptionPassphraseFile.isPresent())
736    {
737      try
738      {
739        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
740             encryptionPassphraseFile.getValue());
741      }
742      catch (final LDAPException e)
743      {
744        Debug.debugException(e);
745        err(e.getMessage());
746        return e.getResultCode();
747      }
748    }
749
750
751    // Create the entry validator and initialize its configuration.
752    entryValidator = new EntryValidator(schema);
753    entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent());
754    entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent());
755    entryValidator.setCheckEntryMissingRDNValues(
756         !ignoreMissingRDNValues.isPresent());
757    entryValidator.setCheckMissingAttributes(
758         !ignoreMissingAttributes.isPresent());
759    entryValidator.setCheckNameForms(!ignoreNameForms.isPresent());
760    entryValidator.setCheckProhibitedAttributes(
761         !ignoreProhibitedAttributes.isPresent());
762    entryValidator.setCheckProhibitedObjectClasses(
763         !ignoreProhibitedObjectClasses.isPresent());
764    entryValidator.setCheckMissingSuperiorObjectClasses(
765         !ignoreMissingSuperiorObjectClasses.isPresent());
766    entryValidator.setCheckSingleValuedAttributes(
767         !ignoreSingleValuedAttributes.isPresent());
768    entryValidator.setCheckStructuralObjectClasses(
769         !ignoreStructuralObjectClasses.isPresent());
770    entryValidator.setCheckUndefinedAttributes(
771         !ignoreUndefinedAttributes.isPresent());
772    entryValidator.setCheckUndefinedObjectClasses(
773         !ignoreUndefinedObjectClasses.isPresent());
774
775    if (ignoreSyntaxViolationsForAttribute.isPresent())
776    {
777      entryValidator.setIgnoreSyntaxViolationAttributeTypes(
778           ignoreSyntaxViolationsForAttribute.getValues());
779    }
780
781
782    // Create an LDIF reader that can be used to read through the LDIF file.
783    final LDIFReader ldifReader;
784    rejectWriter = null;
785    try
786    {
787      InputStream inputStream = new FileInputStream(ldifFile.getValue());
788
789      inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream(
790           inputStream, encryptionPassphrase, false,
791           "LDIF file '" + ldifFile.getValue().getPath() +
792                "' is encrypted.  Please enter the encryption passphrase:",
793             "ERROR:  The provided passphrase was incorrect.",
794             getOut(), getErr()).getFirst();
795
796      if (isCompressed.isPresent())
797      {
798        inputStream = new GZIPInputStream(inputStream);
799      }
800      else
801      {
802        inputStream =
803             ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
804      }
805
806      ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this);
807    }
808    catch (final Exception e)
809    {
810      Debug.debugException(e);
811      err("Unable to open the LDIF reader:  ",
812           StaticUtils.getExceptionMessage(e));
813      return ResultCode.LOCAL_ERROR;
814    }
815
816    ldifReader.setSchema(schema);
817    if (ignoreDuplicateValues.isPresent())
818    {
819      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP);
820    }
821    else
822    {
823      ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT);
824    }
825
826    try
827    {
828      // Create an LDIF writer that can be used to write information about
829      // rejected entries.
830      try
831      {
832        if (rejectFile.isPresent())
833        {
834          rejectWriter = new LDIFWriter(rejectFile.getValue());
835        }
836      }
837      catch (final Exception e)
838      {
839        Debug.debugException(e);
840        err("Unable to create the reject writer:  ",
841             StaticUtils.getExceptionMessage(e));
842        return ResultCode.LOCAL_ERROR;
843      }
844
845      ResultCode resultCode = ResultCode.SUCCESS;
846      while (true)
847      {
848        try
849        {
850          final Entry e = ldifReader.readEntry();
851          if (e == null)
852          {
853            // Because we're performing parallel processing and returning null
854            // from the translate method, LDIFReader.readEntry() should never
855            // return a non-null value.  However, it can throw an LDIFException
856            // if it encounters an invalid entry, or an IOException if there's
857            // a problem reading from the file, so we should still iterate
858            // through all of the entries to catch and report on those problems.
859            break;
860          }
861        }
862        catch (final LDIFException le)
863        {
864          Debug.debugException(le);
865          malformedEntries.incrementAndGet();
866
867          if (resultCode == ResultCode.SUCCESS)
868          {
869            resultCode = ResultCode.DECODING_ERROR;
870          }
871
872          if (rejectWriter != null)
873          {
874            try
875            {
876              rejectWriter.writeComment(
877                   "Unable to parse an entry read from LDIF:", false, false);
878              if (le.mayContinueReading())
879              {
880                rejectWriter.writeComment(
881                     StaticUtils.getExceptionMessage(le), false, true);
882              }
883              else
884              {
885                rejectWriter.writeComment(
886                     StaticUtils.getExceptionMessage(le), false,
887                     false);
888                rejectWriter.writeComment("Unable to continue LDIF processing.",
889                     false, true);
890                err("Aborting LDIF processing:  ",
891                     StaticUtils.getExceptionMessage(le));
892                return ResultCode.LOCAL_ERROR;
893              }
894            }
895            catch (final IOException ioe)
896            {
897              Debug.debugException(ioe);
898              err("Unable to write to the reject file:",
899                  StaticUtils.getExceptionMessage(ioe));
900              err("LDIF parse failure that triggered the rejection:  ",
901                  StaticUtils.getExceptionMessage(le));
902              return ResultCode.LOCAL_ERROR;
903            }
904          }
905        }
906        catch (final IOException ioe)
907        {
908          Debug.debugException(ioe);
909
910          if (rejectWriter != null)
911          {
912            try
913            {
914              rejectWriter.writeComment("I/O error reading from LDIF:", false,
915                   false);
916              rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe),
917                   false, true);
918              return ResultCode.LOCAL_ERROR;
919            }
920            catch (final Exception ex)
921            {
922              Debug.debugException(ex);
923              err("I/O error reading from LDIF:",
924                   StaticUtils.getExceptionMessage(ioe));
925              return ResultCode.LOCAL_ERROR;
926            }
927          }
928        }
929      }
930
931      if (malformedEntries.get() > 0)
932      {
933        out(malformedEntries.get() + " entries were malformed and could not " +
934            "be read from the LDIF file.");
935      }
936
937      if (entryValidator.getInvalidEntries() > 0)
938      {
939        if (resultCode == ResultCode.SUCCESS)
940        {
941          resultCode = ResultCode.OBJECT_CLASS_VIOLATION;
942        }
943
944        for (final String s : entryValidator.getInvalidEntrySummary(true))
945        {
946          out(s);
947        }
948      }
949      else
950      {
951        if (malformedEntries.get() == 0)
952        {
953          out("No errors were encountered.");
954        }
955      }
956
957      return resultCode;
958    }
959    finally
960    {
961      try
962      {
963        ldifReader.close();
964      }
965      catch (final Exception e)
966      {
967        Debug.debugException(e);
968      }
969
970      try
971      {
972        if (rejectWriter != null)
973        {
974          rejectWriter.close();
975        }
976      }
977      catch (final Exception e)
978      {
979        Debug.debugException(e);
980      }
981    }
982  }
983
984
985
986  /**
987   * Examines the provided entry to determine whether it conforms to the
988   * server schema.
989   *
990   * @param  entry           The entry to be examined.
991   * @param  firstLineNumber The line number of the LDIF source on which the
992   *                         provided entry begins.
993   *
994   * @return  The updated entry.  This method will always return {@code null}
995   *          because all of the real processing needed for the entry is
996   *          performed in this method and the entry isn't needed any more
997   *          after this method is done.
998   */
999  @Override()
1000  @Nullable()
1001  public Entry translate(@NotNull final Entry entry, final long firstLineNumber)
1002  {
1003    final ArrayList<String> invalidReasons = new ArrayList<>(5);
1004    if (! entryValidator.entryIsValid(entry, invalidReasons))
1005    {
1006      if (rejectWriter != null)
1007      {
1008        synchronized (this)
1009        {
1010          try
1011          {
1012            rejectWriter.writeEntry(entry, listToString(invalidReasons));
1013          }
1014          catch (final IOException ioe)
1015          {
1016            Debug.debugException(ioe);
1017          }
1018        }
1019      }
1020    }
1021
1022    final long numEntries = entriesProcessed.incrementAndGet();
1023    if ((numEntries % 1000L) == 0L)
1024    {
1025      out("Processed ", numEntries, " entries.");
1026    }
1027
1028    return null;
1029  }
1030
1031
1032
1033  /**
1034   * Converts the provided list of strings into a single string.  It will
1035   * contain line breaks after all but the last element.
1036   *
1037   * @param  l  The list of strings to convert to a single string.
1038   *
1039   * @return  The string from the provided list, or {@code null} if the provided
1040   *          list is empty or {@code null}.
1041   */
1042  @Nullable()
1043  private static String listToString(@Nullable final List<String> l)
1044  {
1045    if ((l == null) || (l.isEmpty()))
1046    {
1047      return null;
1048    }
1049
1050    final StringBuilder buffer = new StringBuilder();
1051    final Iterator<String> iterator = l.iterator();
1052    while (iterator.hasNext())
1053    {
1054      buffer.append(iterator.next());
1055      if (iterator.hasNext())
1056      {
1057        buffer.append(EOL);
1058      }
1059    }
1060
1061    return buffer.toString();
1062  }
1063
1064
1065
1066  /**
1067   * {@inheritDoc}
1068   */
1069  @Override()
1070  @NotNull()
1071  public LinkedHashMap<String[],String> getExampleUsages()
1072  {
1073    final LinkedHashMap<String[],String> examples =
1074         new LinkedHashMap<>(StaticUtils.computeMapCapacity(2));
1075
1076    String[] args =
1077    {
1078      "--hostname", "server.example.com",
1079      "--port", "389",
1080      "--ldifFile", "data.ldif",
1081      "--rejectFile", "rejects.ldif",
1082      "--numThreads", "4"
1083    };
1084    String description =
1085         "Validate the contents of the 'data.ldif' file using the schema " +
1086         "defined in the specified directory server using four concurrent " +
1087         "threads.  All types of validation will be performed, and " +
1088         "information about any errors will be written to the 'rejects.ldif' " +
1089         "file.";
1090    examples.put(args, description);
1091
1092
1093    args = new String[]
1094    {
1095      "--schemaDirectory", "/ds/config/schema",
1096      "--ldifFile", "data.ldif",
1097      "--rejectFile", "rejects.ldif",
1098      "--ignoreStructuralObjectClasses",
1099      "--ignoreAttributeSyntax"
1100    };
1101    description =
1102         "Validate the contents of the 'data.ldif' file using the schema " +
1103         "defined in LDIF files contained in the /ds/config/schema directory " +
1104         "using a single thread.  Any errors resulting from entries that do " +
1105         "not have exactly one structural object class or from values which " +
1106         "violate the syntax for their associated attribute types will be " +
1107         "ignored.  Information about any other failures will be written to " +
1108         "the 'rejects.ldif' file.";
1109    examples.put(args, description);
1110
1111    return examples;
1112  }
1113
1114
1115
1116  /**
1117   * @return EntryValidator
1118   *
1119   * Returns the EntryValidator
1120   */
1121  @Nullable()
1122  public EntryValidator getEntryValidator()
1123  {
1124    return entryValidator;
1125  }
1126}