001/* 002 * Copyright 2016-2024 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2016-2024 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2016-2024 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.transformations; 037 038 039 040import java.io.File; 041import java.io.FileOutputStream; 042import java.io.InputStream; 043import java.io.OutputStream; 044import java.util.ArrayList; 045import java.util.EnumSet; 046import java.util.Iterator; 047import java.util.LinkedHashMap; 048import java.util.List; 049import java.util.Set; 050import java.util.TreeMap; 051import java.util.concurrent.atomic.AtomicLong; 052import java.util.zip.GZIPOutputStream; 053 054import com.unboundid.ldap.sdk.Attribute; 055import com.unboundid.ldap.sdk.ChangeType; 056import com.unboundid.ldap.sdk.DN; 057import com.unboundid.ldap.sdk.Entry; 058import com.unboundid.ldap.sdk.LDAPException; 059import com.unboundid.ldap.sdk.ResultCode; 060import com.unboundid.ldap.sdk.Version; 061import com.unboundid.ldap.sdk.schema.Schema; 062import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 063import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 064import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 065import com.unboundid.ldif.LDIFException; 066import com.unboundid.ldif.LDIFReader; 067import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 068import com.unboundid.ldif.LDIFReaderEntryTranslator; 069import com.unboundid.ldif.LDIFRecord; 070import com.unboundid.util.ByteStringBuffer; 071import com.unboundid.util.CommandLineTool; 072import com.unboundid.util.Debug; 073import com.unboundid.util.NotNull; 074import com.unboundid.util.Nullable; 075import com.unboundid.util.ObjectPair; 076import com.unboundid.util.PassphraseEncryptedOutputStream; 077import com.unboundid.util.StaticUtils; 078import com.unboundid.util.ThreadSafety; 079import com.unboundid.util.ThreadSafetyLevel; 080import com.unboundid.util.args.ArgumentException; 081import com.unboundid.util.args.ArgumentParser; 082import com.unboundid.util.args.BooleanArgument; 083import com.unboundid.util.args.DNArgument; 084import com.unboundid.util.args.FileArgument; 085import com.unboundid.util.args.FilterArgument; 086import com.unboundid.util.args.IntegerArgument; 087import com.unboundid.util.args.ScopeArgument; 088import com.unboundid.util.args.StringArgument; 089 090import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 091 092 093 094/** 095 * This class provides a command-line tool that can be used to apply a number of 096 * transformations to an LDIF file. The transformations that can be applied 097 * include: 098 * <UL> 099 * <LI> 100 * It can scramble the values of a specified set of attributes in a manner 101 * that attempts to preserve the syntax and consistently scrambles the same 102 * value to the same representation. 103 * </LI> 104 * <LI> 105 * It can strip a specified set of attributes out of entries. 106 * </LI> 107 * <LI> 108 * It can redact the values of a specified set of attributes, to indicate 109 * that the values are there but providing no information about what their 110 * values are. 111 * </LI> 112 * <LI> 113 * It can replace the values of a specified attribute with a given set of 114 * values. 115 * </LI> 116 * <LI> 117 * It can add an attribute with a given set of values to any entry that does 118 * not contain that attribute. 119 * </LI> 120 * <LI> 121 * It can replace the values of a specified attribute with a value that 122 * contains a sequentially-incrementing counter. 123 * </LI> 124 * <LI> 125 * It can strip entries matching a given base DN, scope, and filter out of 126 * the LDIF file. 127 * </LI> 128 * <LI> 129 * It can perform DN mapping, so that entries that exist below one base DN 130 * are moved below a different base DN. 131 * </LI> 132 * <LI> 133 * It can perform attribute mapping, to replace uses of one attribute name 134 * with another. 135 * </LI> 136 * </UL> 137 */ 138@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 139public final class TransformLDIF 140 extends CommandLineTool 141 implements LDIFReaderEntryTranslator 142{ 143 /** 144 * The maximum length of any message to write to standard output or standard 145 * error. 146 */ 147 private static final int MAX_OUTPUT_LINE_LENGTH = 148 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 149 150 151 152 // The arguments for use by this program. 153 @Nullable private BooleanArgument addToExistingValues = null; 154 @Nullable private BooleanArgument appendToTargetLDIF = null; 155 @Nullable private BooleanArgument compressTarget = null; 156 @Nullable private BooleanArgument encryptTarget = null; 157 @Nullable private BooleanArgument excludeRecordsWithoutChangeType = null; 158 @Nullable private BooleanArgument excludeNonMatchingEntries = null; 159 @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToEntry = 160 null; 161 @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 162 @Nullable private BooleanArgument hideRedactedValueCount = null; 163 @Nullable private BooleanArgument processDNs = null; 164 @Nullable private BooleanArgument sourceCompressed = null; 165 @Nullable private BooleanArgument sourceContainsChangeRecords = null; 166 @Nullable private BooleanArgument sourceFromStandardInput = null; 167 @Nullable private BooleanArgument targetToStandardOutput = null; 168 @Nullable private DNArgument addAttributeBaseDN = null; 169 @Nullable private DNArgument excludeEntryBaseDN = null; 170 @Nullable private DNArgument flattenBaseDN = null; 171 @Nullable private DNArgument moveSubtreeFrom = null; 172 @Nullable private DNArgument moveSubtreeTo = null; 173 @Nullable private FileArgument encryptionPassphraseFile = null; 174 @Nullable private FileArgument schemaPath = null; 175 @Nullable private FileArgument sourceLDIF = null; 176 @Nullable private FileArgument targetLDIF = null; 177 @Nullable private FilterArgument addAttributeFilter = null; 178 @Nullable private FilterArgument excludeEntryFilter = null; 179 @Nullable private FilterArgument flattenExcludeFilter = null; 180 @Nullable private IntegerArgument initialSequentialValue = null; 181 @Nullable private IntegerArgument numThreads = null; 182 @Nullable private IntegerArgument randomSeed = null; 183 @Nullable private IntegerArgument sequentialValueIncrement = null; 184 @Nullable private IntegerArgument wrapColumn = null; 185 @Nullable private ScopeArgument addAttributeScope = null; 186 @Nullable private ScopeArgument excludeEntryScope = null; 187 @Nullable private StringArgument addAttributeName = null; 188 @Nullable private StringArgument addAttributeValue = null; 189 @Nullable private StringArgument excludeAttribute = null; 190 @Nullable private StringArgument excludeChangeType = null; 191 @Nullable private StringArgument redactAttribute = null; 192 @Nullable private StringArgument renameAttributeFrom = null; 193 @Nullable private StringArgument renameAttributeTo = null; 194 @Nullable private StringArgument replaceValuesAttribute = null; 195 @Nullable private StringArgument replacementValue = null; 196 @Nullable private StringArgument scrambleAttribute = null; 197 @Nullable private StringArgument scrambleJSONField = null; 198 @Nullable private StringArgument sequentialAttribute = null; 199 @Nullable private StringArgument textAfterSequentialValue = null; 200 @Nullable private StringArgument textBeforeSequentialValue = null; 201 202 // A set of thread-local byte stream buffers that will be used to construct 203 // the LDIF representations of records. 204 @NotNull private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 205 new ThreadLocal<>(); 206 207 208 209 /** 210 * Invokes this tool with the provided set of arguments. 211 * 212 * @param args The command-line arguments provided to this program. 213 */ 214 public static void main(@NotNull final String... args) 215 { 216 final ResultCode resultCode = main(System.out, System.err, args); 217 if (resultCode != ResultCode.SUCCESS) 218 { 219 System.exit(resultCode.intValue()); 220 } 221 } 222 223 224 225 /** 226 * Invokes this tool with the provided set of arguments. 227 * 228 * @param out The output stream to use for standard output. It may be 229 * {@code null} if standard output should be suppressed. 230 * @param err The output stream to use for standard error. It may be 231 * {@code null} if standard error should be suppressed. 232 * @param args The command-line arguments provided to this program. 233 * 234 * @return A result code indicating whether processing completed 235 * successfully. 236 */ 237 @NotNull() 238 public static ResultCode main(@Nullable final OutputStream out, 239 @Nullable final OutputStream err, 240 @NotNull final String... args) 241 { 242 final TransformLDIF tool = new TransformLDIF(out, err); 243 return tool.runTool(args); 244 } 245 246 247 248 /** 249 * Creates a new instance of this tool with the provided information. 250 * 251 * @param out The output stream to use for standard output. It may be 252 * {@code null} if standard output should be suppressed. 253 * @param err The output stream to use for standard error. It may be 254 * {@code null} if standard error should be suppressed. 255 */ 256 public TransformLDIF(@Nullable final OutputStream out, 257 @Nullable final OutputStream err) 258 { 259 super(out, err); 260 } 261 262 263 264 /** 265 * {@inheritDoc} 266 */ 267 @Override() 268 @NotNull() 269 public String getToolName() 270 { 271 return "transform-ldif"; 272 } 273 274 275 276 /** 277 * {@inheritDoc} 278 */ 279 @Override() 280 @NotNull() 281 public String getToolDescription() 282 { 283 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 284 } 285 286 287 288 /** 289 * {@inheritDoc} 290 */ 291 @Override() 292 @NotNull() 293 public String getToolVersion() 294 { 295 return Version.NUMERIC_VERSION_STRING; 296 } 297 298 299 300 /** 301 * {@inheritDoc} 302 */ 303 @Override() 304 public boolean supportsInteractiveMode() 305 { 306 return true; 307 } 308 309 310 311 /** 312 * {@inheritDoc} 313 */ 314 @Override() 315 public boolean defaultsToInteractiveMode() 316 { 317 return true; 318 } 319 320 321 322 /** 323 * {@inheritDoc} 324 */ 325 @Override() 326 public boolean supportsPropertiesFile() 327 { 328 return true; 329 } 330 331 332 333 /** 334 * {@inheritDoc} 335 */ 336 @Override() 337 protected boolean supportsDebugLogging() 338 { 339 return true; 340 } 341 342 343 344 /** 345 * {@inheritDoc} 346 */ 347 @Override() 348 public void addToolArguments(@NotNull final ArgumentParser parser) 349 throws ArgumentException 350 { 351 // Add arguments pertaining to the source and target LDIF files. 352 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 353 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 354 false); 355 sourceLDIF.addLongIdentifier("inputLDIF", true); 356 sourceLDIF.addLongIdentifier("source-ldif", true); 357 sourceLDIF.addLongIdentifier("input-ldif", true); 358 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 359 parser.addArgument(sourceLDIF); 360 361 sourceFromStandardInput = new BooleanArgument(null, 362 "sourceFromStandardInput", 1, 363 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 364 sourceFromStandardInput.addLongIdentifier("source-from-standard-input", 365 true); 366 sourceFromStandardInput.setArgumentGroupName( 367 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 368 parser.addArgument(sourceFromStandardInput); 369 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 370 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 371 372 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 373 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 374 false); 375 targetLDIF.addLongIdentifier("outputLDIF", true); 376 targetLDIF.addLongIdentifier("target-ldif", true); 377 targetLDIF.addLongIdentifier("output-ldif", true); 378 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 379 parser.addArgument(targetLDIF); 380 381 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 382 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 383 targetToStandardOutput.addLongIdentifier("target-to-standard-output", true); 384 targetToStandardOutput.setArgumentGroupName( 385 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 386 parser.addArgument(targetToStandardOutput); 387 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 388 389 sourceContainsChangeRecords = new BooleanArgument(null, 390 "sourceContainsChangeRecords", 391 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 392 sourceContainsChangeRecords.addLongIdentifier( 393 "source-contains-change-records", true); 394 sourceContainsChangeRecords.setArgumentGroupName( 395 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 396 parser.addArgument(sourceContainsChangeRecords); 397 398 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 399 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 400 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true); 401 appendToTargetLDIF.setArgumentGroupName( 402 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 403 parser.addArgument(appendToTargetLDIF); 404 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 405 406 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 407 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 408 wrapColumn.addLongIdentifier("wrap-column", true); 409 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 410 parser.addArgument(wrapColumn); 411 412 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 413 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 414 sourceCompressed.addLongIdentifier("inputCompressed", true); 415 sourceCompressed.addLongIdentifier("source-compressed", true); 416 sourceCompressed.addLongIdentifier("input-compressed", true); 417 sourceCompressed.setArgumentGroupName( 418 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 419 parser.addArgument(sourceCompressed); 420 421 compressTarget = new BooleanArgument('c', "compressTarget", 422 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 423 compressTarget.addLongIdentifier("compressOutput", true); 424 compressTarget.addLongIdentifier("compress", true); 425 compressTarget.addLongIdentifier("compress-target", true); 426 compressTarget.addLongIdentifier("compress-output", true); 427 compressTarget.setArgumentGroupName( 428 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 429 parser.addArgument(compressTarget); 430 431 encryptTarget = new BooleanArgument(null, "encryptTarget", 432 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get()); 433 encryptTarget.addLongIdentifier("encryptOutput", true); 434 encryptTarget.addLongIdentifier("encrypt", true); 435 encryptTarget.addLongIdentifier("encrypt-target", true); 436 encryptTarget.addLongIdentifier("encrypt-output", true); 437 encryptTarget.setArgumentGroupName( 438 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 439 parser.addArgument(encryptTarget); 440 441 encryptionPassphraseFile = new FileArgument(null, 442 "encryptionPassphraseFile", false, 1, null, 443 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true, 444 true, false); 445 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 446 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 447 true); 448 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 449 true); 450 encryptionPassphraseFile.setArgumentGroupName( 451 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 452 parser.addArgument(encryptionPassphraseFile); 453 454 455 // Add arguments pertaining to attribute scrambling. 456 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 457 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 458 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 459 scrambleAttribute.addLongIdentifier("attributeName", true); 460 scrambleAttribute.addLongIdentifier("scramble-attribute", true); 461 scrambleAttribute.addLongIdentifier("attribute-name", true); 462 scrambleAttribute.setArgumentGroupName( 463 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 464 parser.addArgument(scrambleAttribute); 465 466 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 467 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 468 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 469 scrambleAttribute.getIdentifierString())); 470 scrambleJSONField.addLongIdentifier("scramble-json-field", true); 471 scrambleJSONField.setArgumentGroupName( 472 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 473 parser.addArgument(scrambleJSONField); 474 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 475 476 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 477 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 478 randomSeed.addLongIdentifier("random-seed", true); 479 randomSeed.setArgumentGroupName( 480 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 481 parser.addArgument(randomSeed); 482 483 484 // Add arguments pertaining to replacing attribute values with a generated 485 // value using a sequential counter. 486 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 487 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 488 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 489 sourceContainsChangeRecords.getIdentifierString())); 490 sequentialAttribute.addLongIdentifier("sequentialAttributeName", true); 491 sequentialAttribute.addLongIdentifier("sequential-attribute", true); 492 sequentialAttribute.addLongIdentifier("sequential-attribute-name", true); 493 sequentialAttribute.setArgumentGroupName( 494 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 495 parser.addArgument(sequentialAttribute); 496 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 497 sequentialAttribute); 498 499 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 500 false, 1, null, 501 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 502 sequentialAttribute.getIdentifierString())); 503 initialSequentialValue.addLongIdentifier("initial-sequential-value", true); 504 initialSequentialValue.setArgumentGroupName( 505 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 506 parser.addArgument(initialSequentialValue); 507 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 508 509 sequentialValueIncrement = new IntegerArgument(null, 510 "sequentialValueIncrement", false, 1, null, 511 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 512 sequentialAttribute.getIdentifierString())); 513 sequentialValueIncrement.addLongIdentifier("sequential-value-increment", 514 true); 515 sequentialValueIncrement.setArgumentGroupName( 516 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 517 parser.addArgument(sequentialValueIncrement); 518 parser.addDependentArgumentSet(sequentialValueIncrement, 519 sequentialAttribute); 520 521 textBeforeSequentialValue = new StringArgument(null, 522 "textBeforeSequentialValue", false, 1, null, 523 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 524 sequentialAttribute.getIdentifierString())); 525 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value", 526 true); 527 textBeforeSequentialValue.setArgumentGroupName( 528 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 529 parser.addArgument(textBeforeSequentialValue); 530 parser.addDependentArgumentSet(textBeforeSequentialValue, 531 sequentialAttribute); 532 533 textAfterSequentialValue = new StringArgument(null, 534 "textAfterSequentialValue", false, 1, null, 535 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 536 sequentialAttribute.getIdentifierString())); 537 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value", 538 true); 539 textAfterSequentialValue.setArgumentGroupName( 540 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 541 parser.addArgument(textAfterSequentialValue); 542 parser.addDependentArgumentSet(textAfterSequentialValue, 543 sequentialAttribute); 544 545 546 // Add arguments pertaining to attribute value replacement. 547 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 548 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 549 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 550 sourceContainsChangeRecords.getIdentifierString())); 551 replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true); 552 replaceValuesAttribute.setArgumentGroupName( 553 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 554 parser.addArgument(replaceValuesAttribute); 555 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 556 replaceValuesAttribute); 557 558 replacementValue = new StringArgument(null, "replacementValue", false, 0, 559 null, 560 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 561 replaceValuesAttribute.getIdentifierString())); 562 replacementValue.addLongIdentifier("replacement-value", true); 563 replacementValue.setArgumentGroupName( 564 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 565 parser.addArgument(replacementValue); 566 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 567 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 568 569 570 // Add arguments pertaining to adding missing attributes. 571 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 572 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 573 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 574 "--addAttributeValue", 575 sourceContainsChangeRecords.getIdentifierString())); 576 addAttributeName.addLongIdentifier("add-attribute-name", true); 577 addAttributeName.setArgumentGroupName( 578 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 579 parser.addArgument(addAttributeName); 580 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 581 addAttributeName); 582 583 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 584 null, 585 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 586 addAttributeName.getIdentifierString())); 587 addAttributeValue.addLongIdentifier("add-attribute-value", true); 588 addAttributeValue.setArgumentGroupName( 589 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 590 parser.addArgument(addAttributeValue); 591 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 592 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 593 594 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 595 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 596 addAttributeName.getIdentifierString(), 597 addAttributeValue.getIdentifierString())); 598 addToExistingValues.addLongIdentifier("add-to-existing-values", true); 599 addToExistingValues.setArgumentGroupName( 600 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 601 parser.addArgument(addToExistingValues); 602 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 603 604 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 605 null, 606 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 607 addAttributeName.getIdentifierString())); 608 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true); 609 addAttributeBaseDN.setArgumentGroupName( 610 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 611 parser.addArgument(addAttributeBaseDN); 612 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 613 614 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 615 null, 616 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 617 addAttributeBaseDN.getIdentifierString(), 618 addAttributeName.getIdentifierString())); 619 addAttributeScope.addLongIdentifier("add-attribute-scope", true); 620 addAttributeScope.setArgumentGroupName( 621 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 622 parser.addArgument(addAttributeScope); 623 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 624 625 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 626 1, null, 627 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 628 addAttributeName.getIdentifierString())); 629 addAttributeFilter.addLongIdentifier("add-attribute-filter", true); 630 addAttributeFilter.setArgumentGroupName( 631 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 632 parser.addArgument(addAttributeFilter); 633 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 634 635 636 // Add arguments pertaining to renaming attributes. 637 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 638 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 639 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 640 renameAttributeFrom.addLongIdentifier("rename-attribute-from", true); 641 renameAttributeFrom.setArgumentGroupName( 642 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 643 parser.addArgument(renameAttributeFrom); 644 645 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 646 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 647 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 648 renameAttributeFrom.getIdentifierString())); 649 renameAttributeTo.addLongIdentifier("rename-attribute-to", true); 650 renameAttributeTo.setArgumentGroupName( 651 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 652 parser.addArgument(renameAttributeTo); 653 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 654 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 655 656 657 // Add arguments pertaining to flattening subtrees. 658 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 659 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 660 flattenBaseDN.addLongIdentifier("flatten-base-dn", true); 661 flattenBaseDN.setArgumentGroupName( 662 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 663 parser.addArgument(flattenBaseDN); 664 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 665 flattenBaseDN); 666 667 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 668 "flattenAddOmittedRDNAttributesToEntry", 1, 669 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 670 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 671 "flatten-add-omitted-rdn-attributes-to-entry", true); 672 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 673 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 674 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 675 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 676 flattenBaseDN); 677 678 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 679 "flattenAddOmittedRDNAttributesToRDN", 1, 680 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 681 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 682 "flatten-add-omitted-rdn-attributes-to-rdn", true); 683 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 684 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 685 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 686 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 687 flattenBaseDN); 688 689 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 690 false, 1, null, 691 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 692 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true); 693 flattenExcludeFilter.setArgumentGroupName( 694 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 695 parser.addArgument(flattenExcludeFilter); 696 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 697 698 699 // Add arguments pertaining to moving subtrees. 700 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 701 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 702 moveSubtreeFrom.addLongIdentifier("move-subtree-from", true); 703 moveSubtreeFrom.setArgumentGroupName( 704 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 705 parser.addArgument(moveSubtreeFrom); 706 707 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 708 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 709 moveSubtreeFrom.getIdentifierString())); 710 moveSubtreeTo.addLongIdentifier("move-subtree-to", true); 711 moveSubtreeTo.setArgumentGroupName( 712 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 713 parser.addArgument(moveSubtreeTo); 714 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 715 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 716 717 718 // Add arguments pertaining to redacting attribute values. 719 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 720 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 721 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 722 redactAttribute.addLongIdentifier("redact-attribute", true); 723 redactAttribute.setArgumentGroupName( 724 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 725 parser.addArgument(redactAttribute); 726 727 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 728 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 729 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count", 730 true); 731 hideRedactedValueCount.setArgumentGroupName( 732 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 733 parser.addArgument(hideRedactedValueCount); 734 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 735 736 737 // Add arguments pertaining to excluding attributes and entries. 738 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 739 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 740 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 741 excludeAttribute.addLongIdentifier("suppressAttribute", true); 742 excludeAttribute.addLongIdentifier("exclude-attribute", true); 743 excludeAttribute.addLongIdentifier("suppress-attribute", true); 744 excludeAttribute.setArgumentGroupName( 745 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 746 parser.addArgument(excludeAttribute); 747 748 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 749 null, 750 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 751 sourceContainsChangeRecords.getIdentifierString())); 752 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true); 753 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true); 754 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true); 755 excludeEntryBaseDN.setArgumentGroupName( 756 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 757 parser.addArgument(excludeEntryBaseDN); 758 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 759 excludeEntryBaseDN); 760 761 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 762 null, 763 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 764 sourceContainsChangeRecords.getIdentifierString())); 765 excludeEntryScope.addLongIdentifier("suppressEntryScope", true); 766 excludeEntryScope.addLongIdentifier("exclude-entry-scope", true); 767 excludeEntryScope.addLongIdentifier("suppress-entry-scope", true); 768 excludeEntryScope.setArgumentGroupName( 769 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 770 parser.addArgument(excludeEntryScope); 771 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 772 excludeEntryScope); 773 774 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 775 1, null, 776 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 777 sourceContainsChangeRecords.getIdentifierString())); 778 excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true); 779 excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true); 780 excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true); 781 excludeEntryFilter.setArgumentGroupName( 782 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 783 parser.addArgument(excludeEntryFilter); 784 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 785 excludeEntryFilter); 786 787 excludeNonMatchingEntries = new BooleanArgument(null, 788 "excludeNonMatchingEntries", 789 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 790 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries", 791 true); 792 excludeNonMatchingEntries.setArgumentGroupName( 793 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 794 parser.addArgument(excludeNonMatchingEntries); 795 parser.addDependentArgumentSet(excludeNonMatchingEntries, 796 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 797 798 799 // Add arguments for excluding records based on their change types. 800 excludeChangeType = new StringArgument(null, "excludeChangeType", 801 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(), 802 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(), 803 StaticUtils.setOf("add", "delete", "modify", "moddn")); 804 excludeChangeType.addLongIdentifier("exclude-change-type", true); 805 excludeChangeType.addLongIdentifier("exclude-changetype", true); 806 excludeChangeType.setArgumentGroupName( 807 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 808 parser.addArgument(excludeChangeType); 809 810 811 // Add arguments for excluding records that don't have a change type. 812 excludeRecordsWithoutChangeType = new BooleanArgument(null, 813 "excludeRecordsWithoutChangeType", 1, 814 INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get()); 815 excludeRecordsWithoutChangeType.addLongIdentifier( 816 "exclude-records-without-change-type", true); 817 excludeRecordsWithoutChangeType.addLongIdentifier( 818 "exclude-records-without-changetype", true); 819 excludeRecordsWithoutChangeType.setArgumentGroupName( 820 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 821 parser.addArgument(excludeRecordsWithoutChangeType); 822 823 824 // Add the remaining arguments. 825 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 826 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 827 true, true, false, false); 828 schemaPath.addLongIdentifier("schemaFile", true); 829 schemaPath.addLongIdentifier("schemaDirectory", true); 830 schemaPath.addLongIdentifier("schema-path", true); 831 schemaPath.addLongIdentifier("schema-file", true); 832 schemaPath.addLongIdentifier("schema-directory", true); 833 parser.addArgument(schemaPath); 834 835 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 836 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 837 1); 838 numThreads.addLongIdentifier("num-threads", true); 839 parser.addArgument(numThreads); 840 841 processDNs = new BooleanArgument('d', "processDNs", 842 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 843 processDNs.addLongIdentifier("process-dns", true); 844 parser.addArgument(processDNs); 845 846 847 // Ensure that at least one kind of transformation was requested. 848 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 849 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 850 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 851 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter, 852 excludeChangeType, excludeRecordsWithoutChangeType); 853 } 854 855 856 857 /** 858 * {@inheritDoc} 859 */ 860 @Override() 861 public void doExtendedArgumentValidation() 862 throws ArgumentException 863 { 864 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 865 // arguments should always be provided. But in order to preserve backward 866 // compatibility with a legacy scramble-ldif tool, we will allow both to be 867 // omitted if either --scrambleAttribute or --sequentialArgument is 868 // provided. In that case, the path of the output file will be the path of 869 // the first input file with ".scrambled" appended to it. 870 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 871 { 872 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 873 { 874 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 875 targetLDIF.getIdentifierString(), 876 targetToStandardOutput.getIdentifierString())); 877 } 878 } 879 880 881 // Make sure that the --renameAttributeFrom and --renameAttributeTo 882 // arguments were provided an equal number of times. 883 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 884 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 885 if (renameFromOccurrences != renameToOccurrences) 886 { 887 throw new ArgumentException( 888 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 889 renameAttributeFrom.getIdentifierString(), 890 renameAttributeTo.getIdentifierString())); 891 } 892 893 894 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 895 // provided an equal number of times. 896 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 897 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 898 if (moveFromOccurrences != moveToOccurrences) 899 { 900 throw new ArgumentException( 901 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 902 moveSubtreeFrom.getIdentifierString(), 903 moveSubtreeTo.getIdentifierString())); 904 } 905 } 906 907 908 909 /** 910 * {@inheritDoc} 911 */ 912 @Override() 913 @NotNull() 914 public ResultCode doToolProcessing() 915 { 916 final Schema schema; 917 try 918 { 919 schema = getSchema(); 920 } 921 catch (final LDAPException le) 922 { 923 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 924 return le.getResultCode(); 925 } 926 927 928 // If an encryption passphrase file is provided, then get the passphrase 929 // from it. 930 String encryptionPassphrase = null; 931 if (encryptionPassphraseFile.isPresent()) 932 { 933 try 934 { 935 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 936 encryptionPassphraseFile.getValue()); 937 } 938 catch (final LDAPException e) 939 { 940 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage()); 941 return e.getResultCode(); 942 } 943 } 944 945 946 // Create the translators to use to apply the transformations. 947 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 948 new ArrayList<>(10); 949 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 950 new ArrayList<>(10); 951 952 final AtomicLong excludedEntryCount = new AtomicLong(0L); 953 createTranslators(entryTranslators, changeRecordTranslators, 954 schema, excludedEntryCount); 955 956 final AggregateLDIFReaderEntryTranslator entryTranslator = 957 new AggregateLDIFReaderEntryTranslator(entryTranslators); 958 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 959 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 960 961 962 // Determine the path to the target file to be written. 963 final File targetFile; 964 if (targetLDIF.isPresent()) 965 { 966 targetFile = targetLDIF.getValue(); 967 } 968 else if (targetToStandardOutput.isPresent()) 969 { 970 targetFile = null; 971 } 972 else 973 { 974 targetFile = 975 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 976 } 977 978 979 // Create the LDIF reader. 980 final LDIFReader ldifReader; 981 try 982 { 983 final InputStream inputStream; 984 if (sourceLDIF.isPresent()) 985 { 986 final ObjectPair<InputStream,String> p = 987 ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(), 988 encryptionPassphrase, getOut(), getErr()); 989 inputStream = p.getFirst(); 990 if ((encryptionPassphrase == null) && (p.getSecond() != null)) 991 { 992 encryptionPassphrase = p.getSecond(); 993 } 994 } 995 else 996 { 997 inputStream = System.in; 998 } 999 1000 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 1001 entryTranslator, changeRecordTranslator); 1002 if (schema != null) 1003 { 1004 ldifReader.setSchema(schema); 1005 } 1006 } 1007 catch (final Exception e) 1008 { 1009 Debug.debugException(e); 1010 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1011 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 1012 StaticUtils.getExceptionMessage(e))); 1013 return ResultCode.LOCAL_ERROR; 1014 } 1015 1016 1017 ResultCode resultCode = ResultCode.SUCCESS; 1018 OutputStream outputStream = null; 1019processingBlock: 1020 try 1021 { 1022 // Create the output stream to use to write the transformed data. 1023 try 1024 { 1025 if (targetFile == null) 1026 { 1027 outputStream = getOut(); 1028 } 1029 else 1030 { 1031 outputStream = 1032 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 1033 } 1034 1035 if (encryptTarget.isPresent()) 1036 { 1037 if (encryptionPassphrase == null) 1038 { 1039 encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase( 1040 false, true, getOut(), getErr()); 1041 } 1042 1043 outputStream = new PassphraseEncryptedOutputStream( 1044 encryptionPassphrase, outputStream); 1045 } 1046 1047 if (compressTarget.isPresent()) 1048 { 1049 outputStream = new GZIPOutputStream(outputStream); 1050 } 1051 } 1052 catch (final Exception e) 1053 { 1054 Debug.debugException(e); 1055 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1056 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 1057 targetFile.getAbsolutePath(), 1058 StaticUtils.getExceptionMessage(e))); 1059 resultCode = ResultCode.LOCAL_ERROR; 1060 break processingBlock; 1061 } 1062 1063 1064 // Read the source data one record at a time. The transformations will 1065 // automatically be applied by the LDIF reader's translators, and even if 1066 // there are multiple reader threads, we're guaranteed to get the results 1067 // in the right order. 1068 long entriesWritten = 0L; 1069 while (true) 1070 { 1071 final LDIFRecord ldifRecord; 1072 try 1073 { 1074 ldifRecord = ldifReader.readLDIFRecord(); 1075 } 1076 catch (final LDIFException le) 1077 { 1078 Debug.debugException(le); 1079 if (le.mayContinueReading()) 1080 { 1081 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1082 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 1083 StaticUtils.getExceptionMessage(le))); 1084 if (resultCode == ResultCode.SUCCESS) 1085 { 1086 resultCode = ResultCode.PARAM_ERROR; 1087 } 1088 continue; 1089 } 1090 else 1091 { 1092 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1093 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 1094 StaticUtils.getExceptionMessage(le))); 1095 if (resultCode == ResultCode.SUCCESS) 1096 { 1097 resultCode = ResultCode.PARAM_ERROR; 1098 } 1099 break processingBlock; 1100 } 1101 } 1102 catch (final Exception e) 1103 { 1104 Debug.debugException(e); 1105 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1106 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1107 StaticUtils.getExceptionMessage(e))); 1108 resultCode = ResultCode.LOCAL_ERROR; 1109 break processingBlock; 1110 } 1111 1112 1113 // If the LDIF record is null, then we've run out of records so we're 1114 // done. 1115 if (ldifRecord == null) 1116 { 1117 break; 1118 } 1119 1120 1121 // Write the record to the output stream. 1122 try 1123 { 1124 if (ldifRecord instanceof PreEncodedLDIFEntry) 1125 { 1126 outputStream.write( 1127 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1128 } 1129 else 1130 { 1131 final ByteStringBuffer buffer = getBuffer(); 1132 if (wrapColumn.isPresent()) 1133 { 1134 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1135 } 1136 else 1137 { 1138 ldifRecord.toLDIF(buffer, 0); 1139 } 1140 buffer.append(StaticUtils.EOL_BYTES); 1141 buffer.write(outputStream); 1142 } 1143 } 1144 catch (final Exception e) 1145 { 1146 Debug.debugException(e); 1147 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1148 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1149 StaticUtils.getExceptionMessage(e))); 1150 resultCode = ResultCode.LOCAL_ERROR; 1151 break processingBlock; 1152 } 1153 1154 1155 // If we've written a multiple of 1000 entries, print a progress 1156 // message. 1157 entriesWritten++; 1158 if ((! targetToStandardOutput.isPresent()) && 1159 ((entriesWritten % 1000L) == 0)) 1160 { 1161 final long numExcluded = excludedEntryCount.get(); 1162 if (numExcluded > 0L) 1163 { 1164 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1165 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1166 entriesWritten, numExcluded)); 1167 } 1168 else 1169 { 1170 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1171 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1172 entriesWritten)); 1173 } 1174 } 1175 } 1176 1177 1178 if (! targetToStandardOutput.isPresent()) 1179 { 1180 final long numExcluded = excludedEntryCount.get(); 1181 if (numExcluded > 0L) 1182 { 1183 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1184 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1185 numExcluded)); 1186 } 1187 else 1188 { 1189 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1190 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1191 } 1192 } 1193 } 1194 finally 1195 { 1196 if (outputStream != null) 1197 { 1198 try 1199 { 1200 outputStream.close(); 1201 } 1202 catch (final Exception e) 1203 { 1204 Debug.debugException(e); 1205 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1206 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1207 targetFile.getAbsolutePath(), 1208 StaticUtils.getExceptionMessage(e))); 1209 if (resultCode == ResultCode.SUCCESS) 1210 { 1211 resultCode = ResultCode.LOCAL_ERROR; 1212 } 1213 } 1214 } 1215 1216 try 1217 { 1218 ldifReader.close(); 1219 } 1220 catch (final Exception e) 1221 { 1222 Debug.debugException(e); 1223 // We can ignore this. 1224 } 1225 } 1226 1227 1228 return resultCode; 1229 } 1230 1231 1232 1233 /** 1234 * Retrieves the schema that should be used for processing. 1235 * 1236 * @return The schema that was created. 1237 * 1238 * @throws LDAPException If a problem is encountered while retrieving the 1239 * schema. 1240 */ 1241 @Nullable() 1242 private Schema getSchema() 1243 throws LDAPException 1244 { 1245 // If any schema paths were specified, then load the schema only from those 1246 // paths. 1247 if (schemaPath.isPresent()) 1248 { 1249 final ArrayList<File> schemaFiles = new ArrayList<>(10); 1250 for (final File path : schemaPath.getValues()) 1251 { 1252 if (path.isFile()) 1253 { 1254 schemaFiles.add(path); 1255 } 1256 else 1257 { 1258 final TreeMap<String,File> fileMap = new TreeMap<>(); 1259 for (final File schemaDirFile : path.listFiles()) 1260 { 1261 final String name = schemaDirFile.getName(); 1262 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1263 { 1264 fileMap.put(name, schemaDirFile); 1265 } 1266 } 1267 schemaFiles.addAll(fileMap.values()); 1268 } 1269 } 1270 1271 if (schemaFiles.isEmpty()) 1272 { 1273 throw new LDAPException(ResultCode.PARAM_ERROR, 1274 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1275 schemaPath.getIdentifierString())); 1276 } 1277 else 1278 { 1279 try 1280 { 1281 return Schema.getSchema(schemaFiles); 1282 } 1283 catch (final Exception e) 1284 { 1285 Debug.debugException(e); 1286 throw new LDAPException(ResultCode.LOCAL_ERROR, 1287 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1288 StaticUtils.getExceptionMessage(e))); 1289 } 1290 } 1291 } 1292 else 1293 { 1294 // If the INSTANCE_ROOT environment variable is set and it refers to a 1295 // directory that has a config/schema subdirectory that has one or more 1296 // schema files in it, then read the schema from that directory. 1297 try 1298 { 1299 final String instanceRootStr = 1300 StaticUtils.getEnvironmentVariable("INSTANCE_ROOT"); 1301 if (instanceRootStr != null) 1302 { 1303 final File instanceRoot = new File(instanceRootStr); 1304 final File configDir = new File(instanceRoot, "config"); 1305 final File schemaDir = new File(configDir, "schema"); 1306 if (schemaDir.exists()) 1307 { 1308 final TreeMap<String,File> fileMap = new TreeMap<>(); 1309 for (final File schemaDirFile : schemaDir.listFiles()) 1310 { 1311 final String name = schemaDirFile.getName(); 1312 if (schemaDirFile.isFile() && 1313 name.toLowerCase().endsWith(".ldif")) 1314 { 1315 fileMap.put(name, schemaDirFile); 1316 } 1317 } 1318 1319 if (! fileMap.isEmpty()) 1320 { 1321 return Schema.getSchema(new ArrayList<>(fileMap.values())); 1322 } 1323 } 1324 } 1325 } 1326 catch (final Exception e) 1327 { 1328 Debug.debugException(e); 1329 } 1330 } 1331 1332 1333 // If we've gotten here, then just return null and the tool will try to use 1334 // the default standard schema. 1335 return null; 1336 } 1337 1338 1339 1340 /** 1341 * Creates the entry and change record translators that will be used to 1342 * perform the transformations. 1343 * 1344 * @param entryTranslators A list to which all created entry 1345 * translators should be written. 1346 * @param changeRecordTranslators A list to which all created change record 1347 * translators should be written. 1348 * @param schema The schema to use when processing. 1349 * @param excludedEntryCount A counter used to keep track of the number 1350 * of entries that have been excluded from 1351 * the result set. 1352 */ 1353 private void createTranslators( 1354 @NotNull final List<LDIFReaderEntryTranslator> entryTranslators, 1355 @NotNull final List<LDIFReaderChangeRecordTranslator> 1356 changeRecordTranslators, 1357 @Nullable final Schema schema, 1358 @NotNull final AtomicLong excludedEntryCount) 1359 { 1360 if (scrambleAttribute.isPresent()) 1361 { 1362 final Long seed; 1363 if (randomSeed.isPresent()) 1364 { 1365 seed = randomSeed.getValue().longValue(); 1366 } 1367 else 1368 { 1369 seed = null; 1370 } 1371 1372 final ScrambleAttributeTransformation t = 1373 new ScrambleAttributeTransformation(schema, seed, 1374 processDNs.isPresent(), scrambleAttribute.getValues(), 1375 scrambleJSONField.getValues()); 1376 entryTranslators.add(t); 1377 changeRecordTranslators.add(t); 1378 } 1379 1380 if (sequentialAttribute.isPresent()) 1381 { 1382 final long initialValue; 1383 if (initialSequentialValue.isPresent()) 1384 { 1385 initialValue = initialSequentialValue.getValue().longValue(); 1386 } 1387 else 1388 { 1389 initialValue = 0L; 1390 } 1391 1392 final long incrementAmount; 1393 if (sequentialValueIncrement.isPresent()) 1394 { 1395 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1396 } 1397 else 1398 { 1399 incrementAmount = 1L; 1400 } 1401 1402 for (final String attrName : sequentialAttribute.getValues()) 1403 { 1404 1405 1406 final ReplaceWithCounterTransformation t = 1407 new ReplaceWithCounterTransformation(schema, attrName, 1408 initialValue, incrementAmount, 1409 textBeforeSequentialValue.getValue(), 1410 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1411 entryTranslators.add(t); 1412 } 1413 } 1414 1415 if (replaceValuesAttribute.isPresent()) 1416 { 1417 final ReplaceAttributeTransformation t = 1418 new ReplaceAttributeTransformation(schema, 1419 replaceValuesAttribute.getValue(), 1420 replacementValue.getValues()); 1421 entryTranslators.add(t); 1422 } 1423 1424 if (addAttributeName.isPresent()) 1425 { 1426 final AddAttributeTransformation t = new AddAttributeTransformation( 1427 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1428 addAttributeFilter.getValue(), 1429 new Attribute(addAttributeName.getValue(), schema, 1430 addAttributeValue.getValues()), 1431 (! addToExistingValues.isPresent())); 1432 entryTranslators.add(t); 1433 } 1434 1435 if (renameAttributeFrom.isPresent()) 1436 { 1437 final Iterator<String> renameFromIterator = 1438 renameAttributeFrom.getValues().iterator(); 1439 final Iterator<String> renameToIterator = 1440 renameAttributeTo.getValues().iterator(); 1441 while (renameFromIterator.hasNext()) 1442 { 1443 final RenameAttributeTransformation t = 1444 new RenameAttributeTransformation(schema, 1445 renameFromIterator.next(), renameToIterator.next(), 1446 processDNs.isPresent()); 1447 entryTranslators.add(t); 1448 changeRecordTranslators.add(t); 1449 } 1450 } 1451 1452 if (flattenBaseDN.isPresent()) 1453 { 1454 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1455 schema, flattenBaseDN.getValue(), 1456 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1457 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1458 flattenExcludeFilter.getValue()); 1459 entryTranslators.add(t); 1460 } 1461 1462 if (moveSubtreeFrom.isPresent()) 1463 { 1464 final Iterator<DN> moveFromIterator = 1465 moveSubtreeFrom.getValues().iterator(); 1466 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1467 while (moveFromIterator.hasNext()) 1468 { 1469 final MoveSubtreeTransformation t = 1470 new MoveSubtreeTransformation(moveFromIterator.next(), 1471 moveToIterator.next()); 1472 entryTranslators.add(t); 1473 changeRecordTranslators.add(t); 1474 } 1475 } 1476 1477 if (redactAttribute.isPresent()) 1478 { 1479 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1480 schema, processDNs.isPresent(), 1481 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1482 entryTranslators.add(t); 1483 changeRecordTranslators.add(t); 1484 } 1485 1486 if (excludeAttribute.isPresent()) 1487 { 1488 final ExcludeAttributeTransformation t = 1489 new ExcludeAttributeTransformation(schema, 1490 excludeAttribute.getValues()); 1491 entryTranslators.add(t); 1492 changeRecordTranslators.add(t); 1493 } 1494 1495 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1496 excludeEntryFilter.isPresent()) 1497 { 1498 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1499 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1500 excludeEntryFilter.getValue(), 1501 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1502 entryTranslators.add(t); 1503 } 1504 1505 if (excludeChangeType.isPresent()) 1506 { 1507 final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class); 1508 for (final String changeTypeName : excludeChangeType.getValues()) 1509 { 1510 changeTypes.add(ChangeType.forName(changeTypeName)); 1511 } 1512 1513 changeRecordTranslators.add( 1514 new ExcludeChangeTypeTransformation(changeTypes)); 1515 } 1516 1517 if (excludeRecordsWithoutChangeType.isPresent()) 1518 { 1519 entryTranslators.add(new ExcludeAllEntriesTransformation()); 1520 } 1521 1522 entryTranslators.add(this); 1523 } 1524 1525 1526 1527 /** 1528 * {@inheritDoc} 1529 */ 1530 @Override() 1531 @NotNull() 1532 public LinkedHashMap<String[],String> getExampleUsages() 1533 { 1534 final LinkedHashMap<String[],String> examples = 1535 new LinkedHashMap<>(StaticUtils.computeMapCapacity(4)); 1536 1537 examples.put( 1538 new String[] 1539 { 1540 "--sourceLDIF", "input.ldif", 1541 "--targetLDIF", "scrambled.ldif", 1542 "--scrambleAttribute", "givenName", 1543 "--scrambleAttribute", "sn", 1544 "--scrambleAttribute", "cn", 1545 "--numThreads", "10", 1546 "--schemaPath", "/ds/config/schema", 1547 "--processDNs" 1548 }, 1549 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1550 1551 examples.put( 1552 new String[] 1553 { 1554 "--sourceLDIF", "input.ldif", 1555 "--targetLDIF", "sequential.ldif", 1556 "--sequentialAttribute", "uid", 1557 "--initialSequentialValue", "1", 1558 "--sequentialValueIncrement", "1", 1559 "--textBeforeSequentialValue", "user.", 1560 "--numThreads", "10", 1561 "--schemaPath", "/ds/config/schema", 1562 "--processDNs" 1563 }, 1564 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1565 1566 examples.put( 1567 new String[] 1568 { 1569 "--sourceLDIF", "input.ldif", 1570 "--targetLDIF", "added-organization.ldif", 1571 "--addAttributeName", "o", 1572 "--addAttributeValue", "Example Corp.", 1573 "--addAttributeFilter", "(objectClass=person)", 1574 "--numThreads", "10", 1575 "--schemaPath", "/ds/config/schema" 1576 }, 1577 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1578 1579 examples.put( 1580 new String[] 1581 { 1582 "--sourceLDIF", "input.ldif", 1583 "--targetLDIF", "rebased.ldif", 1584 "--moveSubtreeFrom", "o=example.com", 1585 "--moveSubtreeTo", "dc=example,dc=com", 1586 "--numThreads", "10", 1587 "--schemaPath", "/ds/config/schema" 1588 }, 1589 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1590 1591 return examples; 1592 } 1593 1594 1595 1596 /** 1597 * {@inheritDoc} 1598 */ 1599 @Override() 1600 @Nullable() 1601 public Entry translate(@NotNull final Entry original, 1602 final long firstLineNumber) 1603 throws LDIFException 1604 { 1605 final ByteStringBuffer buffer = getBuffer(); 1606 if (wrapColumn.isPresent()) 1607 { 1608 original.toLDIF(buffer, wrapColumn.getValue()); 1609 } 1610 else 1611 { 1612 original.toLDIF(buffer, 0); 1613 } 1614 buffer.append(StaticUtils.EOL_BYTES); 1615 1616 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1617 } 1618 1619 1620 1621 /** 1622 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1623 * 1624 * @return A byte string buffer that can be used to perform LDIF encoding. 1625 */ 1626 @NotNull() 1627 private ByteStringBuffer getBuffer() 1628 { 1629 ByteStringBuffer buffer = byteStringBuffers.get(); 1630 if (buffer == null) 1631 { 1632 buffer = new ByteStringBuffer(); 1633 byteStringBuffers.set(buffer); 1634 } 1635 else 1636 { 1637 buffer.clear(); 1638 } 1639 1640 return buffer; 1641 } 1642}