001/* 002 * Copyright 2008-2024 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2008-2024 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2008-2024 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.examples; 037 038 039 040import java.io.File; 041import java.io.FileInputStream; 042import java.io.InputStream; 043import java.io.IOException; 044import java.io.OutputStream; 045import java.util.ArrayList; 046import java.util.Iterator; 047import java.util.TreeMap; 048import java.util.LinkedHashMap; 049import java.util.List; 050import java.util.concurrent.atomic.AtomicLong; 051import java.util.zip.GZIPInputStream; 052 053import com.unboundid.ldap.sdk.Entry; 054import com.unboundid.ldap.sdk.LDAPConnection; 055import com.unboundid.ldap.sdk.LDAPException; 056import com.unboundid.ldap.sdk.ResultCode; 057import com.unboundid.ldap.sdk.Version; 058import com.unboundid.ldap.sdk.schema.Schema; 059import com.unboundid.ldap.sdk.schema.EntryValidator; 060import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 061import com.unboundid.ldif.DuplicateValueBehavior; 062import com.unboundid.ldif.LDIFException; 063import com.unboundid.ldif.LDIFReader; 064import com.unboundid.ldif.LDIFReaderEntryTranslator; 065import com.unboundid.ldif.LDIFWriter; 066import com.unboundid.util.Debug; 067import com.unboundid.util.LDAPCommandLineTool; 068import com.unboundid.util.NotNull; 069import com.unboundid.util.Nullable; 070import com.unboundid.util.StaticUtils; 071import com.unboundid.util.ThreadSafety; 072import com.unboundid.util.ThreadSafetyLevel; 073import com.unboundid.util.args.ArgumentException; 074import com.unboundid.util.args.ArgumentParser; 075import com.unboundid.util.args.BooleanArgument; 076import com.unboundid.util.args.FileArgument; 077import com.unboundid.util.args.IntegerArgument; 078import com.unboundid.util.args.StringArgument; 079 080 081 082/** 083 * This class provides a simple tool that can be used to validate that the 084 * contents of an LDIF file are valid. This includes ensuring that the contents 085 * can be parsed as valid LDIF, and it can also ensure that the LDIF content 086 * conforms to the server schema. It will obtain the schema by connecting to 087 * the server and retrieving the default schema (i.e., the schema which governs 088 * the root DSE). By default, a thorough set of validation will be performed, 089 * but it is possible to disable certain types of validation. 090 * <BR><BR> 091 * Some of the APIs demonstrated by this example include: 092 * <UL> 093 * <LI>Argument Parsing (from the {@code com.unboundid.util.args} 094 * package)</LI> 095 * <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util} 096 * package)</LI> 097 * <LI>LDIF Processing (from the {@code com.unboundid.ldif} package)</LI> 098 * <LI>Schema Parsing (from the {@code com.unboundid.ldap.sdk.schema} 099 * package)</LI> 100 * </UL> 101 * <BR><BR> 102 * Supported arguments include those allowed by the {@link LDAPCommandLineTool} 103 * class (to obtain the information to use to connect to the server to read the 104 * schema), as well as the following additional arguments: 105 * <UL> 106 * <LI>"--schemaDirectory {path}" -- specifies the path to a directory 107 * containing files with schema definitions. If this argument is 108 * provided, then no attempt will be made to communicate with a directory 109 * server.</LI> 110 * <LI>"-f {path}" or "--ldifFile {path}" -- specifies the path to the LDIF 111 * file to be validated.</LI> 112 * <LI>"-c" or "--isCompressed" -- indicates that the LDIF file is 113 * compressed.</LI> 114 * <LI>"-R {path}" or "--rejectFile {path}" -- specifies the path to the file 115 * to be written with information about all entries that failed 116 * validation.</LI> 117 * <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of 118 * concurrent threads to use when processing the LDIF. If this is not 119 * provided, then a default of one thread will be used.</LI> 120 * <LI>"--ignoreUndefinedObjectClasses" -- indicates that the validation 121 * process should ignore validation failures due to entries that contain 122 * object classes not defined in the server schema.</LI> 123 * <LI>"--ignoreUndefinedAttributes" -- indicates that the validation process 124 * should ignore validation failures due to entries that contain 125 * attributes not defined in the server schema.</LI> 126 * <LI>"--ignoreMalformedDNs" -- indicates that the validation process should 127 * ignore validation failures due to entries with malformed DNs.</LI> 128 * <LI>"--ignoreMissingRDNValues" -- indicates that the validation process 129 * should ignore validation failures due to entries that contain an RDN 130 * attribute value that is not present in the set of entry 131 * attributes.</LI> 132 * <LI>"--ignoreStructuralObjectClasses" -- indicates that the validation 133 * process should ignore validation failures due to entries that either do 134 * not have a structural object class or that have multiple structural 135 * object classes.</LI> 136 * <LI>"--ignoreProhibitedObjectClasses" -- indicates that the validation 137 * process should ignore validation failures due to entries containing 138 * auxiliary classes that are not allowed by a DIT content rule, or 139 * abstract classes that are not subclassed by an auxiliary or structural 140 * class contained in the entry.</LI> 141 * <LI>"--ignoreProhibitedAttributes" -- indicates that the validation process 142 * should ignore validation failures due to entries including attributes 143 * that are not allowed or are explicitly prohibited by a DIT content 144 * rule.</LI> 145 * <LI>"--ignoreMissingAttributes" -- indicates that the validation process 146 * should ignore validation failures due to entries missing required 147 * attributes.</LI> 148 * <LI>"--ignoreSingleValuedAttributes" -- indicates that the validation 149 * process should ignore validation failures due to single-valued 150 * attributes containing multiple values.</LI> 151 * <LI>"--ignoreAttributeSyntax" -- indicates that the validation process 152 * should ignore validation failures due to attribute values which violate 153 * the associated attribute syntax.</LI> 154 * <LI>"--ignoreSyntaxViolationsForAttribute" -- indicates that the validation 155 * process should ignore validation failures due to attribute values which 156 * violate the associated attribute syntax, but only for the specified 157 * attribute types.</LI> 158 * <LI>"--ignoreNameForms" -- indicates that the validation process should 159 * ignore validation failures due to name form violations (in which the 160 * entry's RDN does not comply with the associated name form).</LI> 161 * </UL> 162 */ 163@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 164public final class ValidateLDIF 165 extends LDAPCommandLineTool 166 implements LDIFReaderEntryTranslator 167{ 168 /** 169 * The end-of-line character for this platform. 170 */ 171 @NotNull private static final String EOL = 172 StaticUtils.getSystemProperty("line.separator", "\n"); 173 174 175 176 // The arguments used by this program. 177 @Nullable private BooleanArgument ignoreDuplicateValues; 178 @Nullable private BooleanArgument ignoreUndefinedObjectClasses; 179 @Nullable private BooleanArgument ignoreUndefinedAttributes; 180 @Nullable private BooleanArgument ignoreMalformedDNs; 181 @Nullable private BooleanArgument ignoreMissingRDNValues; 182 @Nullable private BooleanArgument ignoreMissingSuperiorObjectClasses; 183 @Nullable private BooleanArgument ignoreStructuralObjectClasses; 184 @Nullable private BooleanArgument ignoreProhibitedObjectClasses; 185 @Nullable private BooleanArgument ignoreProhibitedAttributes; 186 @Nullable private BooleanArgument ignoreMissingAttributes; 187 @Nullable private BooleanArgument ignoreSingleValuedAttributes; 188 @Nullable private BooleanArgument ignoreAttributeSyntax; 189 @Nullable private BooleanArgument ignoreNameForms; 190 @Nullable private BooleanArgument isCompressed; 191 @Nullable private FileArgument schemaDirectory; 192 @Nullable private FileArgument ldifFile; 193 @Nullable private FileArgument rejectFile; 194 @Nullable private FileArgument encryptionPassphraseFile; 195 @Nullable private IntegerArgument numThreads; 196 @Nullable private StringArgument ignoreSyntaxViolationsForAttribute; 197 198 // The counter used to keep track of the number of entries processed. 199 @NotNull private final AtomicLong entriesProcessed = new AtomicLong(0L); 200 201 // The counter used to keep track of the number of entries that could not be 202 // parsed as valid entries. 203 @NotNull private final AtomicLong malformedEntries = new AtomicLong(0L); 204 205 // The entry validator that will be used to validate the entries. 206 @Nullable private EntryValidator entryValidator; 207 208 // The LDIF writer that will be used to write rejected entries. 209 @Nullable private LDIFWriter rejectWriter; 210 211 212 213 /** 214 * Parse the provided command line arguments and make the appropriate set of 215 * changes. 216 * 217 * @param args The command line arguments provided to this program. 218 */ 219 public static void main(@NotNull final String[] args) 220 { 221 final ResultCode resultCode = main(args, System.out, System.err); 222 if (resultCode != ResultCode.SUCCESS) 223 { 224 System.exit(resultCode.intValue()); 225 } 226 } 227 228 229 230 /** 231 * Parse the provided command line arguments and make the appropriate set of 232 * changes. 233 * 234 * @param args The command line arguments provided to this program. 235 * @param outStream The output stream to which standard out should be 236 * written. It may be {@code null} if output should be 237 * suppressed. 238 * @param errStream The output stream to which standard error should be 239 * written. It may be {@code null} if error messages 240 * should be suppressed. 241 * 242 * @return A result code indicating whether the processing was successful. 243 */ 244 @NotNull() 245 public static ResultCode main(@NotNull final String[] args, 246 @Nullable final OutputStream outStream, 247 @Nullable final OutputStream errStream) 248 { 249 final ValidateLDIF validateLDIF = new ValidateLDIF(outStream, errStream); 250 return validateLDIF.runTool(args); 251 } 252 253 254 255 /** 256 * Creates a new instance of this tool. 257 * 258 * @param outStream The output stream to which standard out should be 259 * written. It may be {@code null} if output should be 260 * suppressed. 261 * @param errStream The output stream to which standard error should be 262 * written. It may be {@code null} if error messages 263 * should be suppressed. 264 */ 265 public ValidateLDIF(@Nullable final OutputStream outStream, 266 @Nullable final OutputStream errStream) 267 { 268 super(outStream, errStream); 269 } 270 271 272 273 /** 274 * Retrieves the name for this tool. 275 * 276 * @return The name for this tool. 277 */ 278 @Override() 279 @NotNull() 280 public String getToolName() 281 { 282 return "validate-ldif"; 283 } 284 285 286 287 /** 288 * Retrieves the description for this tool. 289 * 290 * @return The description for this tool. 291 */ 292 @Override() 293 @NotNull() 294 public String getToolDescription() 295 { 296 return "Validate the contents of an LDIF file " + 297 "against the server schema."; 298 } 299 300 301 302 /** 303 * Retrieves the version string for this tool. 304 * 305 * @return The version string for this tool. 306 */ 307 @Override() 308 @NotNull() 309 public String getToolVersion() 310 { 311 return Version.NUMERIC_VERSION_STRING; 312 } 313 314 315 316 /** 317 * Indicates whether this tool should provide support for an interactive mode, 318 * in which the tool offers a mode in which the arguments can be provided in 319 * a text-driven menu rather than requiring them to be given on the command 320 * line. If interactive mode is supported, it may be invoked using the 321 * "--interactive" argument. Alternately, if interactive mode is supported 322 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 323 * interactive mode may be invoked by simply launching the tool without any 324 * arguments. 325 * 326 * @return {@code true} if this tool supports interactive mode, or 327 * {@code false} if not. 328 */ 329 @Override() 330 public boolean supportsInteractiveMode() 331 { 332 return true; 333 } 334 335 336 337 /** 338 * Indicates whether this tool defaults to launching in interactive mode if 339 * the tool is invoked without any command-line arguments. This will only be 340 * used if {@link #supportsInteractiveMode()} returns {@code true}. 341 * 342 * @return {@code true} if this tool defaults to using interactive mode if 343 * launched without any command-line arguments, or {@code false} if 344 * not. 345 */ 346 @Override() 347 public boolean defaultsToInteractiveMode() 348 { 349 return true; 350 } 351 352 353 354 /** 355 * Indicates whether this tool should provide arguments for redirecting output 356 * to a file. If this method returns {@code true}, then the tool will offer 357 * an "--outputFile" argument that will specify the path to a file to which 358 * all standard output and standard error content will be written, and it will 359 * also offer a "--teeToStandardOut" argument that can only be used if the 360 * "--outputFile" argument is present and will cause all output to be written 361 * to both the specified output file and to standard output. 362 * 363 * @return {@code true} if this tool should provide arguments for redirecting 364 * output to a file, or {@code false} if not. 365 */ 366 @Override() 367 protected boolean supportsOutputFile() 368 { 369 return true; 370 } 371 372 373 374 /** 375 * Indicates whether this tool should default to interactively prompting for 376 * the bind password if a password is required but no argument was provided 377 * to indicate how to get the password. 378 * 379 * @return {@code true} if this tool should default to interactively 380 * prompting for the bind password, or {@code false} if not. 381 */ 382 @Override() 383 protected boolean defaultToPromptForBindPassword() 384 { 385 return true; 386 } 387 388 389 390 /** 391 * Indicates whether this tool supports the use of a properties file for 392 * specifying default values for arguments that aren't specified on the 393 * command line. 394 * 395 * @return {@code true} if this tool supports the use of a properties file 396 * for specifying default values for arguments that aren't specified 397 * on the command line, or {@code false} if not. 398 */ 399 @Override() 400 public boolean supportsPropertiesFile() 401 { 402 return true; 403 } 404 405 406 407 /** 408 * Indicates whether the LDAP-specific arguments should include alternate 409 * versions of all long identifiers that consist of multiple words so that 410 * they are available in both camelCase and dash-separated versions. 411 * 412 * @return {@code true} if this tool should provide multiple versions of 413 * long identifiers for LDAP-specific arguments, or {@code false} if 414 * not. 415 */ 416 @Override() 417 protected boolean includeAlternateLongIdentifiers() 418 { 419 return true; 420 } 421 422 423 424 /** 425 * Indicates whether this tool should provide a command-line argument that 426 * allows for low-level SSL debugging. If this returns {@code true}, then an 427 * "--enableSSLDebugging}" argument will be added that sets the 428 * "javax.net.debug" system property to "all" before attempting any 429 * communication. 430 * 431 * @return {@code true} if this tool should offer an "--enableSSLDebugging" 432 * argument, or {@code false} if not. 433 */ 434 @Override() 435 protected boolean supportsSSLDebugging() 436 { 437 return true; 438 } 439 440 441 442 /** 443 * Adds the arguments used by this program that aren't already provided by the 444 * generic {@code LDAPCommandLineTool} framework. 445 * 446 * @param parser The argument parser to which the arguments should be added. 447 * 448 * @throws ArgumentException If a problem occurs while adding the arguments. 449 */ 450 @Override() 451 public void addNonLDAPArguments(@NotNull final ArgumentParser parser) 452 throws ArgumentException 453 { 454 String description = "The path to the LDIF file to process. The tool " + 455 "will automatically attempt to detect whether the file is " + 456 "encrypted or compressed."; 457 ldifFile = new FileArgument('f', "ldifFile", true, 1, "{path}", description, 458 true, true, true, false); 459 ldifFile.addLongIdentifier("ldif-file", true); 460 parser.addArgument(ldifFile); 461 462 463 // Add an argument that makes it possible to read a compressed LDIF file. 464 // Note that this argument is no longer needed for dealing with compressed 465 // files, since the tool will automatically detect whether a file is 466 // compressed. However, the argument is still provided for the purpose of 467 // backward compatibility. 468 description = "Indicates that the specified LDIF file is compressed " + 469 "using gzip compression."; 470 isCompressed = new BooleanArgument('c', "isCompressed", description); 471 isCompressed.addLongIdentifier("is-compressed", true); 472 isCompressed.setHidden(true); 473 parser.addArgument(isCompressed); 474 475 476 // Add an argument that indicates that the tool should read the encryption 477 // passphrase from a file. 478 description = "Indicates that the specified LDIF file is encrypted and " + 479 "that the encryption passphrase is contained in the specified " + 480 "file. If the LDIF data is encrypted and this argument is not " + 481 "provided, then the tool will interactively prompt for the " + 482 "encryption passphrase."; 483 encryptionPassphraseFile = new FileArgument(null, 484 "encryptionPassphraseFile", false, 1, null, description, true, true, 485 true, false); 486 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 487 true); 488 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 489 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 490 true); 491 parser.addArgument(encryptionPassphraseFile); 492 493 494 description = "The path to the file to which rejected entries should be " + 495 "written."; 496 rejectFile = new FileArgument('R', "rejectFile", false, 1, "{path}", 497 description, false, true, true, false); 498 rejectFile.addLongIdentifier("reject-file", true); 499 parser.addArgument(rejectFile); 500 501 description = "The path to a directory containing one or more LDIF files " + 502 "with the schema information to use. If this is provided, " + 503 "then no LDAP communication will be performed."; 504 schemaDirectory = new FileArgument(null, "schemaDirectory", false, 1, 505 "{path}", description, true, true, false, true); 506 schemaDirectory.addLongIdentifier("schema-directory", true); 507 parser.addArgument(schemaDirectory); 508 509 description = "The number of threads to use when processing the LDIF file."; 510 numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}", 511 description, 1, Integer.MAX_VALUE, 1); 512 numThreads.addLongIdentifier("num-threads", true); 513 parser.addArgument(numThreads); 514 515 description = "Ignore validation failures due to entries containing " + 516 "duplicate values for the same attribute."; 517 ignoreDuplicateValues = 518 new BooleanArgument(null, "ignoreDuplicateValues", description); 519 ignoreDuplicateValues.setArgumentGroupName( 520 "Validation Strictness Arguments"); 521 ignoreDuplicateValues.addLongIdentifier("ignore-duplicate-values", true); 522 parser.addArgument(ignoreDuplicateValues); 523 524 description = "Ignore validation failures due to object classes not " + 525 "defined in the schema."; 526 ignoreUndefinedObjectClasses = 527 new BooleanArgument(null, "ignoreUndefinedObjectClasses", description); 528 ignoreUndefinedObjectClasses.setArgumentGroupName( 529 "Validation Strictness Arguments"); 530 ignoreUndefinedObjectClasses.addLongIdentifier( 531 "ignore-undefined-object-classes", true); 532 parser.addArgument(ignoreUndefinedObjectClasses); 533 534 description = "Ignore validation failures due to attributes not defined " + 535 "in the schema."; 536 ignoreUndefinedAttributes = 537 new BooleanArgument(null, "ignoreUndefinedAttributes", description); 538 ignoreUndefinedAttributes.setArgumentGroupName( 539 "Validation Strictness Arguments"); 540 ignoreUndefinedAttributes.addLongIdentifier("ignore-undefined-attributes", 541 true); 542 parser.addArgument(ignoreUndefinedAttributes); 543 544 description = "Ignore validation failures due to entries with malformed " + 545 "DNs."; 546 ignoreMalformedDNs = 547 new BooleanArgument(null, "ignoreMalformedDNs", description); 548 ignoreMalformedDNs.setArgumentGroupName("Validation Strictness Arguments"); 549 ignoreMalformedDNs.addLongIdentifier("ignore-malformed-dns", true); 550 parser.addArgument(ignoreMalformedDNs); 551 552 description = "Ignore validation failures due to entries with RDN " + 553 "attribute values that are missing from the set of entry " + 554 "attributes."; 555 ignoreMissingRDNValues = 556 new BooleanArgument(null, "ignoreMissingRDNValues", description); 557 ignoreMissingRDNValues.setArgumentGroupName( 558 "Validation Strictness Arguments"); 559 ignoreMissingRDNValues.addLongIdentifier("ignore-missing-rdn-values", true); 560 parser.addArgument(ignoreMissingRDNValues); 561 562 description = "Ignore validation failures due to entries without exactly " + 563 "structural object class."; 564 ignoreStructuralObjectClasses = 565 new BooleanArgument(null, "ignoreStructuralObjectClasses", 566 description); 567 ignoreStructuralObjectClasses.setArgumentGroupName( 568 "Validation Strictness Arguments"); 569 ignoreStructuralObjectClasses.addLongIdentifier( 570 "ignore-structural-object-classes", true); 571 parser.addArgument(ignoreStructuralObjectClasses); 572 573 description = "Ignore validation failures due to entries with object " + 574 "classes that are not allowed."; 575 ignoreProhibitedObjectClasses = 576 new BooleanArgument(null, "ignoreProhibitedObjectClasses", 577 description); 578 ignoreProhibitedObjectClasses.setArgumentGroupName( 579 "Validation Strictness Arguments"); 580 ignoreProhibitedObjectClasses.addLongIdentifier( 581 "ignore-prohibited-object-classes", true); 582 parser.addArgument(ignoreProhibitedObjectClasses); 583 584 description = "Ignore validation failures due to entries that are " + 585 "one or more superior object classes."; 586 ignoreMissingSuperiorObjectClasses = 587 new BooleanArgument(null, "ignoreMissingSuperiorObjectClasses", 588 description); 589 ignoreMissingSuperiorObjectClasses.setArgumentGroupName( 590 "Validation Strictness Arguments"); 591 ignoreMissingSuperiorObjectClasses.addLongIdentifier( 592 "ignore-missing-superior-object-classes", true); 593 parser.addArgument(ignoreMissingSuperiorObjectClasses); 594 595 description = "Ignore validation failures due to entries with attributes " + 596 "that are not allowed."; 597 ignoreProhibitedAttributes = 598 new BooleanArgument(null, "ignoreProhibitedAttributes", description); 599 ignoreProhibitedAttributes.setArgumentGroupName( 600 "Validation Strictness Arguments"); 601 ignoreProhibitedAttributes.addLongIdentifier( 602 "ignore-prohibited-attributes", true); 603 parser.addArgument(ignoreProhibitedAttributes); 604 605 description = "Ignore validation failures due to entries missing " + 606 "required attributes."; 607 ignoreMissingAttributes = 608 new BooleanArgument(null, "ignoreMissingAttributes", description); 609 ignoreMissingAttributes.setArgumentGroupName( 610 "Validation Strictness Arguments"); 611 ignoreMissingAttributes.addLongIdentifier("ignore-missing-attributes", 612 true); 613 parser.addArgument(ignoreMissingAttributes); 614 615 description = "Ignore validation failures due to entries with multiple " + 616 "values for single-valued attributes."; 617 ignoreSingleValuedAttributes = 618 new BooleanArgument(null, "ignoreSingleValuedAttributes", description); 619 ignoreSingleValuedAttributes.setArgumentGroupName( 620 "Validation Strictness Arguments"); 621 ignoreSingleValuedAttributes.addLongIdentifier( 622 "ignore-single-valued-attributes", true); 623 parser.addArgument(ignoreSingleValuedAttributes); 624 625 description = "Ignore validation failures due to entries with attribute " + 626 "values that violate their associated syntax. If this is " + 627 "provided, then no attribute syntax violations will be " + 628 "flagged. If this is not provided, then all attribute " + 629 "syntax violations will be flagged except for violations " + 630 "in those attributes excluded by the " + 631 "--ignoreSyntaxViolationsForAttribute argument."; 632 ignoreAttributeSyntax = 633 new BooleanArgument(null, "ignoreAttributeSyntax", description); 634 ignoreAttributeSyntax.setArgumentGroupName( 635 "Validation Strictness Arguments"); 636 ignoreAttributeSyntax.addLongIdentifier("ignore-attribute-syntax", true); 637 parser.addArgument(ignoreAttributeSyntax); 638 639 description = "The name or OID of an attribute for which to ignore " + 640 "validation failures due to violations of the associated " + 641 "attribute syntax. This argument can only be used if the " + 642 "--ignoreAttributeSyntax argument is not provided."; 643 ignoreSyntaxViolationsForAttribute = new StringArgument(null, 644 "ignoreSyntaxViolationsForAttribute", false, 0, "{attr}", description); 645 ignoreSyntaxViolationsForAttribute.setArgumentGroupName( 646 "Validation Strictness Arguments"); 647 ignoreSyntaxViolationsForAttribute.addLongIdentifier( 648 "ignore-syntax-violations-for-attribute", true); 649 parser.addArgument(ignoreSyntaxViolationsForAttribute); 650 651 description = "Ignore validation failures due to entries with RDNs " + 652 "that violate the associated name form definition."; 653 ignoreNameForms = new BooleanArgument(null, "ignoreNameForms", description); 654 ignoreNameForms.setArgumentGroupName("Validation Strictness Arguments"); 655 ignoreNameForms.addLongIdentifier("ignore-name-forms", true); 656 parser.addArgument(ignoreNameForms); 657 658 659 // The ignoreAttributeSyntax and ignoreAttributeSyntaxForAttribute arguments 660 // cannot be used together. 661 parser.addExclusiveArgumentSet(ignoreAttributeSyntax, 662 ignoreSyntaxViolationsForAttribute); 663 } 664 665 666 667 /** 668 * Performs the actual processing for this tool. In this case, it gets a 669 * connection to the directory server and uses it to retrieve the server 670 * schema. It then reads the LDIF file and validates each entry accordingly. 671 * 672 * @return The result code for the processing that was performed. 673 */ 674 @Override() 675 @NotNull() 676 public ResultCode doToolProcessing() 677 { 678 // Get the connection to the directory server and use it to read the schema. 679 final Schema schema; 680 if (schemaDirectory.isPresent()) 681 { 682 final File schemaDir = schemaDirectory.getValue(); 683 684 try 685 { 686 final TreeMap<String,File> fileMap = new TreeMap<>(); 687 for (final File f : schemaDir.listFiles()) 688 { 689 final String name = f.getName(); 690 if (f.isFile() && name.endsWith(".ldif")) 691 { 692 fileMap.put(name, f); 693 } 694 } 695 696 if (fileMap.isEmpty()) 697 { 698 err("No LDIF files found in directory " + 699 schemaDir.getAbsolutePath()); 700 return ResultCode.PARAM_ERROR; 701 } 702 703 final ArrayList<File> fileList = new ArrayList<>(fileMap.values()); 704 schema = Schema.getSchema(fileList); 705 } 706 catch (final Exception e) 707 { 708 Debug.debugException(e); 709 err("Unable to read schema from files in directory " + 710 schemaDir.getAbsolutePath() + ": " + 711 StaticUtils.getExceptionMessage(e)); 712 return ResultCode.LOCAL_ERROR; 713 } 714 } 715 else 716 { 717 try 718 { 719 final LDAPConnection connection = getConnection(); 720 schema = connection.getSchema(); 721 connection.close(); 722 } 723 catch (final LDAPException le) 724 { 725 Debug.debugException(le); 726 err("Unable to connect to the directory server and read the schema: ", 727 le.getMessage()); 728 return le.getResultCode(); 729 } 730 } 731 732 733 // Get the encryption passphrase, if it was provided. 734 String encryptionPassphrase = null; 735 if (encryptionPassphraseFile.isPresent()) 736 { 737 try 738 { 739 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 740 encryptionPassphraseFile.getValue()); 741 } 742 catch (final LDAPException e) 743 { 744 Debug.debugException(e); 745 err(e.getMessage()); 746 return e.getResultCode(); 747 } 748 } 749 750 751 // Create the entry validator and initialize its configuration. 752 entryValidator = new EntryValidator(schema); 753 entryValidator.setCheckAttributeSyntax(!ignoreAttributeSyntax.isPresent()); 754 entryValidator.setCheckMalformedDNs(!ignoreMalformedDNs.isPresent()); 755 entryValidator.setCheckEntryMissingRDNValues( 756 !ignoreMissingRDNValues.isPresent()); 757 entryValidator.setCheckMissingAttributes( 758 !ignoreMissingAttributes.isPresent()); 759 entryValidator.setCheckNameForms(!ignoreNameForms.isPresent()); 760 entryValidator.setCheckProhibitedAttributes( 761 !ignoreProhibitedAttributes.isPresent()); 762 entryValidator.setCheckProhibitedObjectClasses( 763 !ignoreProhibitedObjectClasses.isPresent()); 764 entryValidator.setCheckMissingSuperiorObjectClasses( 765 !ignoreMissingSuperiorObjectClasses.isPresent()); 766 entryValidator.setCheckSingleValuedAttributes( 767 !ignoreSingleValuedAttributes.isPresent()); 768 entryValidator.setCheckStructuralObjectClasses( 769 !ignoreStructuralObjectClasses.isPresent()); 770 entryValidator.setCheckUndefinedAttributes( 771 !ignoreUndefinedAttributes.isPresent()); 772 entryValidator.setCheckUndefinedObjectClasses( 773 !ignoreUndefinedObjectClasses.isPresent()); 774 775 if (ignoreSyntaxViolationsForAttribute.isPresent()) 776 { 777 entryValidator.setIgnoreSyntaxViolationAttributeTypes( 778 ignoreSyntaxViolationsForAttribute.getValues()); 779 } 780 781 782 // Create an LDIF reader that can be used to read through the LDIF file. 783 final LDIFReader ldifReader; 784 rejectWriter = null; 785 try 786 { 787 InputStream inputStream = new FileInputStream(ldifFile.getValue()); 788 789 inputStream = ToolUtils.getPossiblyPassphraseEncryptedInputStream( 790 inputStream, encryptionPassphrase, false, 791 "LDIF file '" + ldifFile.getValue().getPath() + 792 "' is encrypted. Please enter the encryption passphrase:", 793 "ERROR: The provided passphrase was incorrect.", 794 getOut(), getErr()).getFirst(); 795 796 if (isCompressed.isPresent()) 797 { 798 inputStream = new GZIPInputStream(inputStream); 799 } 800 else 801 { 802 inputStream = 803 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 804 } 805 806 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), this); 807 } 808 catch (final Exception e) 809 { 810 Debug.debugException(e); 811 err("Unable to open the LDIF reader: ", 812 StaticUtils.getExceptionMessage(e)); 813 return ResultCode.LOCAL_ERROR; 814 } 815 816 ldifReader.setSchema(schema); 817 if (ignoreDuplicateValues.isPresent()) 818 { 819 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.STRIP); 820 } 821 else 822 { 823 ldifReader.setDuplicateValueBehavior(DuplicateValueBehavior.REJECT); 824 } 825 826 try 827 { 828 // Create an LDIF writer that can be used to write information about 829 // rejected entries. 830 try 831 { 832 if (rejectFile.isPresent()) 833 { 834 rejectWriter = new LDIFWriter(rejectFile.getValue()); 835 } 836 } 837 catch (final Exception e) 838 { 839 Debug.debugException(e); 840 err("Unable to create the reject writer: ", 841 StaticUtils.getExceptionMessage(e)); 842 return ResultCode.LOCAL_ERROR; 843 } 844 845 ResultCode resultCode = ResultCode.SUCCESS; 846 while (true) 847 { 848 try 849 { 850 final Entry e = ldifReader.readEntry(); 851 if (e == null) 852 { 853 // Because we're performing parallel processing and returning null 854 // from the translate method, LDIFReader.readEntry() should never 855 // return a non-null value. However, it can throw an LDIFException 856 // if it encounters an invalid entry, or an IOException if there's 857 // a problem reading from the file, so we should still iterate 858 // through all of the entries to catch and report on those problems. 859 break; 860 } 861 } 862 catch (final LDIFException le) 863 { 864 Debug.debugException(le); 865 malformedEntries.incrementAndGet(); 866 867 if (resultCode == ResultCode.SUCCESS) 868 { 869 resultCode = ResultCode.DECODING_ERROR; 870 } 871 872 if (rejectWriter != null) 873 { 874 try 875 { 876 rejectWriter.writeComment( 877 "Unable to parse an entry read from LDIF:", false, false); 878 if (le.mayContinueReading()) 879 { 880 rejectWriter.writeComment( 881 StaticUtils.getExceptionMessage(le), false, true); 882 } 883 else 884 { 885 rejectWriter.writeComment( 886 StaticUtils.getExceptionMessage(le), false, 887 false); 888 rejectWriter.writeComment("Unable to continue LDIF processing.", 889 false, true); 890 err("Aborting LDIF processing: ", 891 StaticUtils.getExceptionMessage(le)); 892 return ResultCode.LOCAL_ERROR; 893 } 894 } 895 catch (final IOException ioe) 896 { 897 Debug.debugException(ioe); 898 err("Unable to write to the reject file:", 899 StaticUtils.getExceptionMessage(ioe)); 900 err("LDIF parse failure that triggered the rejection: ", 901 StaticUtils.getExceptionMessage(le)); 902 return ResultCode.LOCAL_ERROR; 903 } 904 } 905 } 906 catch (final IOException ioe) 907 { 908 Debug.debugException(ioe); 909 910 if (rejectWriter != null) 911 { 912 try 913 { 914 rejectWriter.writeComment("I/O error reading from LDIF:", false, 915 false); 916 rejectWriter.writeComment(StaticUtils.getExceptionMessage(ioe), 917 false, true); 918 return ResultCode.LOCAL_ERROR; 919 } 920 catch (final Exception ex) 921 { 922 Debug.debugException(ex); 923 err("I/O error reading from LDIF:", 924 StaticUtils.getExceptionMessage(ioe)); 925 return ResultCode.LOCAL_ERROR; 926 } 927 } 928 } 929 } 930 931 if (malformedEntries.get() > 0) 932 { 933 out(malformedEntries.get() + " entries were malformed and could not " + 934 "be read from the LDIF file."); 935 } 936 937 if (entryValidator.getInvalidEntries() > 0) 938 { 939 if (resultCode == ResultCode.SUCCESS) 940 { 941 resultCode = ResultCode.OBJECT_CLASS_VIOLATION; 942 } 943 944 for (final String s : entryValidator.getInvalidEntrySummary(true)) 945 { 946 out(s); 947 } 948 } 949 else 950 { 951 if (malformedEntries.get() == 0) 952 { 953 out("No errors were encountered."); 954 } 955 } 956 957 return resultCode; 958 } 959 finally 960 { 961 try 962 { 963 ldifReader.close(); 964 } 965 catch (final Exception e) 966 { 967 Debug.debugException(e); 968 } 969 970 try 971 { 972 if (rejectWriter != null) 973 { 974 rejectWriter.close(); 975 } 976 } 977 catch (final Exception e) 978 { 979 Debug.debugException(e); 980 } 981 } 982 } 983 984 985 986 /** 987 * Examines the provided entry to determine whether it conforms to the 988 * server schema. 989 * 990 * @param entry The entry to be examined. 991 * @param firstLineNumber The line number of the LDIF source on which the 992 * provided entry begins. 993 * 994 * @return The updated entry. This method will always return {@code null} 995 * because all of the real processing needed for the entry is 996 * performed in this method and the entry isn't needed any more 997 * after this method is done. 998 */ 999 @Override() 1000 @Nullable() 1001 public Entry translate(@NotNull final Entry entry, final long firstLineNumber) 1002 { 1003 final ArrayList<String> invalidReasons = new ArrayList<>(5); 1004 if (! entryValidator.entryIsValid(entry, invalidReasons)) 1005 { 1006 if (rejectWriter != null) 1007 { 1008 synchronized (this) 1009 { 1010 try 1011 { 1012 rejectWriter.writeEntry(entry, listToString(invalidReasons)); 1013 } 1014 catch (final IOException ioe) 1015 { 1016 Debug.debugException(ioe); 1017 } 1018 } 1019 } 1020 } 1021 1022 final long numEntries = entriesProcessed.incrementAndGet(); 1023 if ((numEntries % 1000L) == 0L) 1024 { 1025 out("Processed ", numEntries, " entries."); 1026 } 1027 1028 return null; 1029 } 1030 1031 1032 1033 /** 1034 * Converts the provided list of strings into a single string. It will 1035 * contain line breaks after all but the last element. 1036 * 1037 * @param l The list of strings to convert to a single string. 1038 * 1039 * @return The string from the provided list, or {@code null} if the provided 1040 * list is empty or {@code null}. 1041 */ 1042 @Nullable() 1043 private static String listToString(@Nullable final List<String> l) 1044 { 1045 if ((l == null) || (l.isEmpty())) 1046 { 1047 return null; 1048 } 1049 1050 final StringBuilder buffer = new StringBuilder(); 1051 final Iterator<String> iterator = l.iterator(); 1052 while (iterator.hasNext()) 1053 { 1054 buffer.append(iterator.next()); 1055 if (iterator.hasNext()) 1056 { 1057 buffer.append(EOL); 1058 } 1059 } 1060 1061 return buffer.toString(); 1062 } 1063 1064 1065 1066 /** 1067 * {@inheritDoc} 1068 */ 1069 @Override() 1070 @NotNull() 1071 public LinkedHashMap<String[],String> getExampleUsages() 1072 { 1073 final LinkedHashMap<String[],String> examples = 1074 new LinkedHashMap<>(StaticUtils.computeMapCapacity(2)); 1075 1076 String[] args = 1077 { 1078 "--hostname", "server.example.com", 1079 "--port", "389", 1080 "--ldifFile", "data.ldif", 1081 "--rejectFile", "rejects.ldif", 1082 "--numThreads", "4" 1083 }; 1084 String description = 1085 "Validate the contents of the 'data.ldif' file using the schema " + 1086 "defined in the specified directory server using four concurrent " + 1087 "threads. All types of validation will be performed, and " + 1088 "information about any errors will be written to the 'rejects.ldif' " + 1089 "file."; 1090 examples.put(args, description); 1091 1092 1093 args = new String[] 1094 { 1095 "--schemaDirectory", "/ds/config/schema", 1096 "--ldifFile", "data.ldif", 1097 "--rejectFile", "rejects.ldif", 1098 "--ignoreStructuralObjectClasses", 1099 "--ignoreAttributeSyntax" 1100 }; 1101 description = 1102 "Validate the contents of the 'data.ldif' file using the schema " + 1103 "defined in LDIF files contained in the /ds/config/schema directory " + 1104 "using a single thread. Any errors resulting from entries that do " + 1105 "not have exactly one structural object class or from values which " + 1106 "violate the syntax for their associated attribute types will be " + 1107 "ignored. Information about any other failures will be written to " + 1108 "the 'rejects.ldif' file."; 1109 examples.put(args, description); 1110 1111 return examples; 1112 } 1113 1114 1115 1116 /** 1117 * @return EntryValidator 1118 * 1119 * Returns the EntryValidator 1120 */ 1121 @Nullable() 1122 public EntryValidator getEntryValidator() 1123 { 1124 return entryValidator; 1125 } 1126}