001/* 002 * Copyright 2016-2024 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2016-2024 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2016-2024 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.transformations; 037 038 039 040import java.io.File; 041import java.io.FileOutputStream; 042import java.io.InputStream; 043import java.io.OutputStream; 044import java.util.ArrayList; 045import java.util.EnumSet; 046import java.util.Iterator; 047import java.util.LinkedHashMap; 048import java.util.List; 049import java.util.Set; 050import java.util.TreeMap; 051import java.util.concurrent.atomic.AtomicLong; 052import java.util.zip.GZIPOutputStream; 053 054import com.unboundid.ldap.sdk.Attribute; 055import com.unboundid.ldap.sdk.ChangeType; 056import com.unboundid.ldap.sdk.DN; 057import com.unboundid.ldap.sdk.Entry; 058import com.unboundid.ldap.sdk.LDAPException; 059import com.unboundid.ldap.sdk.ResultCode; 060import com.unboundid.ldap.sdk.Version; 061import com.unboundid.ldap.sdk.schema.Schema; 062import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 063import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator; 064import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator; 065import com.unboundid.ldif.LDIFException; 066import com.unboundid.ldif.LDIFReader; 067import com.unboundid.ldif.LDIFReaderChangeRecordTranslator; 068import com.unboundid.ldif.LDIFReaderEntryTranslator; 069import com.unboundid.ldif.LDIFRecord; 070import com.unboundid.util.ByteStringBuffer; 071import com.unboundid.util.CommandLineTool; 072import com.unboundid.util.Debug; 073import com.unboundid.util.NotNull; 074import com.unboundid.util.Nullable; 075import com.unboundid.util.ObjectPair; 076import com.unboundid.util.PassphraseEncryptedOutputStream; 077import com.unboundid.util.StaticUtils; 078import com.unboundid.util.ThreadSafety; 079import com.unboundid.util.ThreadSafetyLevel; 080import com.unboundid.util.args.ArgumentException; 081import com.unboundid.util.args.ArgumentParser; 082import com.unboundid.util.args.BooleanArgument; 083import com.unboundid.util.args.DNArgument; 084import com.unboundid.util.args.FileArgument; 085import com.unboundid.util.args.FilterArgument; 086import com.unboundid.util.args.IntegerArgument; 087import com.unboundid.util.args.ScopeArgument; 088import com.unboundid.util.args.StringArgument; 089 090import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*; 091 092 093 094/** 095 * This class provides a command-line tool that can be used to apply a number of 096 * transformations to an LDIF file. The transformations that can be applied 097 * include: 098 * <UL> 099 * <LI> 100 * It can scramble the values of a specified set of attributes in a manner 101 * that attempts to preserve the syntax and consistently scrambles the same 102 * value to the same representation. 103 * </LI> 104 * <LI> 105 * It can strip a specified set of attributes out of entries. 106 * </LI> 107 * <LI> 108 * It can redact the values of a specified set of attributes, to indicate 109 * that the values are there but providing no information about what their 110 * values are. 111 * </LI> 112 * <LI> 113 * It can replace the values of a specified attribute with a given set of 114 * values. 115 * </LI> 116 * <LI> 117 * It can add an attribute with a given set of values to any entry that does 118 * not contain that attribute. 119 * </LI> 120 * <LI> 121 * It can replace the values of a specified attribute with a value that 122 * contains a sequentially-incrementing counter. 123 * </LI> 124 * <LI> 125 * It can strip entries matching a given base DN, scope, and filter out of 126 * the LDIF file. 127 * </LI> 128 * <LI> 129 * It can perform DN mapping, so that entries that exist below one base DN 130 * are moved below a different base DN. 131 * </LI> 132 * <LI> 133 * It can perform attribute mapping, to replace uses of one attribute name 134 * with another. 135 * </LI> 136 * </UL> 137 */ 138@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 139public final class TransformLDIF 140 extends CommandLineTool 141 implements LDIFReaderEntryTranslator 142{ 143 /** 144 * The maximum length of any message to write to standard output or standard 145 * error. 146 */ 147 private static final int MAX_OUTPUT_LINE_LENGTH = 148 StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 149 150 151 152 // The arguments for use by this program. 153 @Nullable private BooleanArgument addToExistingValues = null; 154 @Nullable private BooleanArgument appendToTargetLDIF = null; 155 @Nullable private BooleanArgument compressTarget = null; 156 @Nullable private BooleanArgument encryptTarget = null; 157 @Nullable private BooleanArgument excludeRecordsWithoutChangeType = null; 158 @Nullable private BooleanArgument excludeNonMatchingEntries = null; 159 @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToEntry = 160 null; 161 @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null; 162 @Nullable private BooleanArgument hideRedactedValueCount = null; 163 @Nullable private BooleanArgument processDNs = null; 164 @Nullable private BooleanArgument sourceCompressed = null; 165 @Nullable private BooleanArgument sourceContainsChangeRecords = null; 166 @Nullable private BooleanArgument sourceFromStandardInput = null; 167 @Nullable private BooleanArgument targetToStandardOutput = null; 168 @Nullable private DNArgument addAttributeBaseDN = null; 169 @Nullable private DNArgument excludeEntryBaseDN = null; 170 @Nullable private DNArgument flattenBaseDN = null; 171 @Nullable private DNArgument moveSubtreeFrom = null; 172 @Nullable private DNArgument moveSubtreeTo = null; 173 @Nullable private FileArgument encryptionPassphraseFile = null; 174 @Nullable private FileArgument schemaPath = null; 175 @Nullable private FileArgument sourceLDIF = null; 176 @Nullable private FileArgument targetLDIF = null; 177 @Nullable private FilterArgument addAttributeFilter = null; 178 @Nullable private FilterArgument excludeEntryFilter = null; 179 @Nullable private FilterArgument flattenExcludeFilter = null; 180 @Nullable private IntegerArgument initialSequentialValue = null; 181 @Nullable private IntegerArgument numThreads = null; 182 @Nullable private IntegerArgument randomSeed = null; 183 @Nullable private IntegerArgument sequentialValueIncrement = null; 184 @Nullable private IntegerArgument wrapColumn = null; 185 @Nullable private ScopeArgument addAttributeScope = null; 186 @Nullable private ScopeArgument excludeEntryScope = null; 187 @Nullable private StringArgument addAttributeName = null; 188 @Nullable private StringArgument addAttributeValue = null; 189 @Nullable private StringArgument excludeAttribute = null; 190 @Nullable private StringArgument excludeChangeType = null; 191 @Nullable private StringArgument redactAttribute = null; 192 @Nullable private StringArgument renameAttributeFrom = null; 193 @Nullable private StringArgument renameAttributeTo = null; 194 @Nullable private StringArgument replaceValuesAttribute = null; 195 @Nullable private StringArgument replacementValue = null; 196 @Nullable private StringArgument scrambleAttribute = null; 197 @Nullable private StringArgument scrambleJSONField = null; 198 @Nullable private StringArgument sequentialAttribute = null; 199 @Nullable private StringArgument textAfterSequentialValue = null; 200 @Nullable private StringArgument textBeforeSequentialValue = null; 201 202 // A set of thread-local byte stream buffers that will be used to construct 203 // the LDIF representations of records. 204 @NotNull private final ThreadLocal<ByteStringBuffer> byteStringBuffers = 205 new ThreadLocal<>(); 206 207 208 209 /** 210 * Invokes this tool with the provided set of arguments. 211 * 212 * @param args The command-line arguments provided to this program. 213 */ 214 public static void main(@NotNull final String... args) 215 { 216 final ResultCode resultCode = main(System.out, System.err, args); 217 if (resultCode != ResultCode.SUCCESS) 218 { 219 System.exit(resultCode.intValue()); 220 } 221 } 222 223 224 225 /** 226 * Invokes this tool with the provided set of arguments. 227 * 228 * @param out The output stream to use for standard output. It may be 229 * {@code null} if standard output should be suppressed. 230 * @param err The output stream to use for standard error. It may be 231 * {@code null} if standard error should be suppressed. 232 * @param args The command-line arguments provided to this program. 233 * 234 * @return A result code indicating whether processing completed 235 * successfully. 236 */ 237 @NotNull() 238 public static ResultCode main(@Nullable final OutputStream out, 239 @Nullable final OutputStream err, 240 @NotNull final String... args) 241 { 242 final TransformLDIF tool = new TransformLDIF(out, err); 243 return tool.runTool(args); 244 } 245 246 247 248 /** 249 * Creates a new instance of this tool with the provided information. 250 * 251 * @param out The output stream to use for standard output. It may be 252 * {@code null} if standard output should be suppressed. 253 * @param err The output stream to use for standard error. It may be 254 * {@code null} if standard error should be suppressed. 255 */ 256 public TransformLDIF(@Nullable final OutputStream out, 257 @Nullable final OutputStream err) 258 { 259 super(out, err); 260 } 261 262 263 264 /** 265 * {@inheritDoc} 266 */ 267 @Override() 268 @NotNull() 269 public String getToolName() 270 { 271 return "transform-ldif"; 272 } 273 274 275 276 /** 277 * {@inheritDoc} 278 */ 279 @Override() 280 @NotNull() 281 public String getToolDescription() 282 { 283 return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get(); 284 } 285 286 287 288 /** 289 * {@inheritDoc} 290 */ 291 @Override() 292 @NotNull() 293 public String getToolVersion() 294 { 295 return Version.NUMERIC_VERSION_STRING; 296 } 297 298 299 300 /** 301 * {@inheritDoc} 302 */ 303 @Override() 304 public boolean supportsInteractiveMode() 305 { 306 return true; 307 } 308 309 310 311 /** 312 * {@inheritDoc} 313 */ 314 @Override() 315 public boolean defaultsToInteractiveMode() 316 { 317 return true; 318 } 319 320 321 322 /** 323 * {@inheritDoc} 324 */ 325 @Override() 326 public boolean supportsPropertiesFile() 327 { 328 return true; 329 } 330 331 332 333 /** 334 * {@inheritDoc} 335 */ 336 @Override() 337 public void addToolArguments(@NotNull final ArgumentParser parser) 338 throws ArgumentException 339 { 340 // Add arguments pertaining to the source and target LDIF files. 341 sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null, 342 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true, 343 false); 344 sourceLDIF.addLongIdentifier("inputLDIF", true); 345 sourceLDIF.addLongIdentifier("source-ldif", true); 346 sourceLDIF.addLongIdentifier("input-ldif", true); 347 sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 348 parser.addArgument(sourceLDIF); 349 350 sourceFromStandardInput = new BooleanArgument(null, 351 "sourceFromStandardInput", 1, 352 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get()); 353 sourceFromStandardInput.addLongIdentifier("source-from-standard-input", 354 true); 355 sourceFromStandardInput.setArgumentGroupName( 356 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 357 parser.addArgument(sourceFromStandardInput); 358 parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput); 359 parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput); 360 361 targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null, 362 INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true, 363 false); 364 targetLDIF.addLongIdentifier("outputLDIF", true); 365 targetLDIF.addLongIdentifier("target-ldif", true); 366 targetLDIF.addLongIdentifier("output-ldif", true); 367 targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 368 parser.addArgument(targetLDIF); 369 370 targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput", 371 1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get()); 372 targetToStandardOutput.addLongIdentifier("target-to-standard-output", true); 373 targetToStandardOutput.setArgumentGroupName( 374 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 375 parser.addArgument(targetToStandardOutput); 376 parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput); 377 378 sourceContainsChangeRecords = new BooleanArgument(null, 379 "sourceContainsChangeRecords", 380 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get()); 381 sourceContainsChangeRecords.addLongIdentifier( 382 "source-contains-change-records", true); 383 sourceContainsChangeRecords.setArgumentGroupName( 384 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 385 parser.addArgument(sourceContainsChangeRecords); 386 387 appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF", 388 INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get()); 389 appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true); 390 appendToTargetLDIF.setArgumentGroupName( 391 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 392 parser.addArgument(appendToTargetLDIF); 393 parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF); 394 395 wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null, 396 INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE); 397 wrapColumn.addLongIdentifier("wrap-column", true); 398 wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 399 parser.addArgument(wrapColumn); 400 401 sourceCompressed = new BooleanArgument('C', "sourceCompressed", 402 INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get()); 403 sourceCompressed.addLongIdentifier("inputCompressed", true); 404 sourceCompressed.addLongIdentifier("source-compressed", true); 405 sourceCompressed.addLongIdentifier("input-compressed", true); 406 sourceCompressed.setArgumentGroupName( 407 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 408 parser.addArgument(sourceCompressed); 409 410 compressTarget = new BooleanArgument('c', "compressTarget", 411 INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get()); 412 compressTarget.addLongIdentifier("compressOutput", true); 413 compressTarget.addLongIdentifier("compress", true); 414 compressTarget.addLongIdentifier("compress-target", true); 415 compressTarget.addLongIdentifier("compress-output", true); 416 compressTarget.setArgumentGroupName( 417 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 418 parser.addArgument(compressTarget); 419 420 encryptTarget = new BooleanArgument(null, "encryptTarget", 421 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get()); 422 encryptTarget.addLongIdentifier("encryptOutput", true); 423 encryptTarget.addLongIdentifier("encrypt", true); 424 encryptTarget.addLongIdentifier("encrypt-target", true); 425 encryptTarget.addLongIdentifier("encrypt-output", true); 426 encryptTarget.setArgumentGroupName( 427 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 428 parser.addArgument(encryptTarget); 429 430 encryptionPassphraseFile = new FileArgument(null, 431 "encryptionPassphraseFile", false, 1, null, 432 INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true, 433 true, false); 434 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 435 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 436 true); 437 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 438 true); 439 encryptionPassphraseFile.setArgumentGroupName( 440 INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get()); 441 parser.addArgument(encryptionPassphraseFile); 442 443 444 // Add arguments pertaining to attribute scrambling. 445 scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0, 446 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 447 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get()); 448 scrambleAttribute.addLongIdentifier("attributeName", true); 449 scrambleAttribute.addLongIdentifier("scramble-attribute", true); 450 scrambleAttribute.addLongIdentifier("attribute-name", true); 451 scrambleAttribute.setArgumentGroupName( 452 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 453 parser.addArgument(scrambleAttribute); 454 455 scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0, 456 INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(), 457 INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get( 458 scrambleAttribute.getIdentifierString())); 459 scrambleJSONField.addLongIdentifier("scramble-json-field", true); 460 scrambleJSONField.setArgumentGroupName( 461 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 462 parser.addArgument(scrambleJSONField); 463 parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute); 464 465 randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null, 466 INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get()); 467 randomSeed.addLongIdentifier("random-seed", true); 468 randomSeed.setArgumentGroupName( 469 INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get()); 470 parser.addArgument(randomSeed); 471 472 473 // Add arguments pertaining to replacing attribute values with a generated 474 // value using a sequential counter. 475 sequentialAttribute = new StringArgument('S', "sequentialAttribute", 476 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 477 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get( 478 sourceContainsChangeRecords.getIdentifierString())); 479 sequentialAttribute.addLongIdentifier("sequentialAttributeName", true); 480 sequentialAttribute.addLongIdentifier("sequential-attribute", true); 481 sequentialAttribute.addLongIdentifier("sequential-attribute-name", true); 482 sequentialAttribute.setArgumentGroupName( 483 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 484 parser.addArgument(sequentialAttribute); 485 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 486 sequentialAttribute); 487 488 initialSequentialValue = new IntegerArgument('i', "initialSequentialValue", 489 false, 1, null, 490 INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get( 491 sequentialAttribute.getIdentifierString())); 492 initialSequentialValue.addLongIdentifier("initial-sequential-value", true); 493 initialSequentialValue.setArgumentGroupName( 494 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 495 parser.addArgument(initialSequentialValue); 496 parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute); 497 498 sequentialValueIncrement = new IntegerArgument(null, 499 "sequentialValueIncrement", false, 1, null, 500 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get( 501 sequentialAttribute.getIdentifierString())); 502 sequentialValueIncrement.addLongIdentifier("sequential-value-increment", 503 true); 504 sequentialValueIncrement.setArgumentGroupName( 505 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 506 parser.addArgument(sequentialValueIncrement); 507 parser.addDependentArgumentSet(sequentialValueIncrement, 508 sequentialAttribute); 509 510 textBeforeSequentialValue = new StringArgument(null, 511 "textBeforeSequentialValue", false, 1, null, 512 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get( 513 sequentialAttribute.getIdentifierString())); 514 textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value", 515 true); 516 textBeforeSequentialValue.setArgumentGroupName( 517 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 518 parser.addArgument(textBeforeSequentialValue); 519 parser.addDependentArgumentSet(textBeforeSequentialValue, 520 sequentialAttribute); 521 522 textAfterSequentialValue = new StringArgument(null, 523 "textAfterSequentialValue", false, 1, null, 524 INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get( 525 sequentialAttribute.getIdentifierString())); 526 textAfterSequentialValue.addLongIdentifier("text-after-sequential-value", 527 true); 528 textAfterSequentialValue.setArgumentGroupName( 529 INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get()); 530 parser.addArgument(textAfterSequentialValue); 531 parser.addDependentArgumentSet(textAfterSequentialValue, 532 sequentialAttribute); 533 534 535 // Add arguments pertaining to attribute value replacement. 536 replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute", 537 false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 538 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get( 539 sourceContainsChangeRecords.getIdentifierString())); 540 replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true); 541 replaceValuesAttribute.setArgumentGroupName( 542 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 543 parser.addArgument(replaceValuesAttribute); 544 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 545 replaceValuesAttribute); 546 547 replacementValue = new StringArgument(null, "replacementValue", false, 0, 548 null, 549 INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get( 550 replaceValuesAttribute.getIdentifierString())); 551 replacementValue.addLongIdentifier("replacement-value", true); 552 replacementValue.setArgumentGroupName( 553 INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get()); 554 parser.addArgument(replacementValue); 555 parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue); 556 parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute); 557 558 559 // Add arguments pertaining to adding missing attributes. 560 addAttributeName = new StringArgument(null, "addAttributeName", false, 1, 561 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 562 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get( 563 "--addAttributeValue", 564 sourceContainsChangeRecords.getIdentifierString())); 565 addAttributeName.addLongIdentifier("add-attribute-name", true); 566 addAttributeName.setArgumentGroupName( 567 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 568 parser.addArgument(addAttributeName); 569 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 570 addAttributeName); 571 572 addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0, 573 null, 574 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get( 575 addAttributeName.getIdentifierString())); 576 addAttributeValue.addLongIdentifier("add-attribute-value", true); 577 addAttributeValue.setArgumentGroupName( 578 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 579 parser.addArgument(addAttributeValue); 580 parser.addDependentArgumentSet(addAttributeName, addAttributeValue); 581 parser.addDependentArgumentSet(addAttributeValue, addAttributeName); 582 583 addToExistingValues = new BooleanArgument(null, "addToExistingValues", 584 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get( 585 addAttributeName.getIdentifierString(), 586 addAttributeValue.getIdentifierString())); 587 addToExistingValues.addLongIdentifier("add-to-existing-values", true); 588 addToExistingValues.setArgumentGroupName( 589 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 590 parser.addArgument(addToExistingValues); 591 parser.addDependentArgumentSet(addToExistingValues, addAttributeName); 592 593 addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1, 594 null, 595 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get( 596 addAttributeName.getIdentifierString())); 597 addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true); 598 addAttributeBaseDN.setArgumentGroupName( 599 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 600 parser.addArgument(addAttributeBaseDN); 601 parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName); 602 603 addAttributeScope = new ScopeArgument(null, "addAttributeScope", false, 604 null, 605 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get( 606 addAttributeBaseDN.getIdentifierString(), 607 addAttributeName.getIdentifierString())); 608 addAttributeScope.addLongIdentifier("add-attribute-scope", true); 609 addAttributeScope.setArgumentGroupName( 610 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 611 parser.addArgument(addAttributeScope); 612 parser.addDependentArgumentSet(addAttributeScope, addAttributeName); 613 614 addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false, 615 1, null, 616 INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get( 617 addAttributeName.getIdentifierString())); 618 addAttributeFilter.addLongIdentifier("add-attribute-filter", true); 619 addAttributeFilter.setArgumentGroupName( 620 INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get()); 621 parser.addArgument(addAttributeFilter); 622 parser.addDependentArgumentSet(addAttributeFilter, addAttributeName); 623 624 625 // Add arguments pertaining to renaming attributes. 626 renameAttributeFrom = new StringArgument(null, "renameAttributeFrom", 627 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 628 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get()); 629 renameAttributeFrom.addLongIdentifier("rename-attribute-from", true); 630 renameAttributeFrom.setArgumentGroupName( 631 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 632 parser.addArgument(renameAttributeFrom); 633 634 renameAttributeTo = new StringArgument(null, "renameAttributeTo", 635 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 636 INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get( 637 renameAttributeFrom.getIdentifierString())); 638 renameAttributeTo.addLongIdentifier("rename-attribute-to", true); 639 renameAttributeTo.setArgumentGroupName( 640 INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get()); 641 parser.addArgument(renameAttributeTo); 642 parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo); 643 parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom); 644 645 646 // Add arguments pertaining to flattening subtrees. 647 flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null, 648 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get()); 649 flattenBaseDN.addLongIdentifier("flatten-base-dn", true); 650 flattenBaseDN.setArgumentGroupName( 651 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 652 parser.addArgument(flattenBaseDN); 653 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 654 flattenBaseDN); 655 656 flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null, 657 "flattenAddOmittedRDNAttributesToEntry", 1, 658 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get()); 659 flattenAddOmittedRDNAttributesToEntry.addLongIdentifier( 660 "flatten-add-omitted-rdn-attributes-to-entry", true); 661 flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName( 662 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 663 parser.addArgument(flattenAddOmittedRDNAttributesToEntry); 664 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry, 665 flattenBaseDN); 666 667 flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null, 668 "flattenAddOmittedRDNAttributesToRDN", 1, 669 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get()); 670 flattenAddOmittedRDNAttributesToRDN.addLongIdentifier( 671 "flatten-add-omitted-rdn-attributes-to-rdn", true); 672 flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName( 673 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 674 parser.addArgument(flattenAddOmittedRDNAttributesToRDN); 675 parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN, 676 flattenBaseDN); 677 678 flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter", 679 false, 1, null, 680 INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get()); 681 flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true); 682 flattenExcludeFilter.setArgumentGroupName( 683 INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get()); 684 parser.addArgument(flattenExcludeFilter); 685 parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN); 686 687 688 // Add arguments pertaining to moving subtrees. 689 moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null, 690 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get()); 691 moveSubtreeFrom.addLongIdentifier("move-subtree-from", true); 692 moveSubtreeFrom.setArgumentGroupName( 693 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 694 parser.addArgument(moveSubtreeFrom); 695 696 moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null, 697 INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get( 698 moveSubtreeFrom.getIdentifierString())); 699 moveSubtreeTo.addLongIdentifier("move-subtree-to", true); 700 moveSubtreeTo.setArgumentGroupName( 701 INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get()); 702 parser.addArgument(moveSubtreeTo); 703 parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo); 704 parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom); 705 706 707 // Add arguments pertaining to redacting attribute values. 708 redactAttribute = new StringArgument(null, "redactAttribute", false, 0, 709 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 710 INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get()); 711 redactAttribute.addLongIdentifier("redact-attribute", true); 712 redactAttribute.setArgumentGroupName( 713 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 714 parser.addArgument(redactAttribute); 715 716 hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount", 717 INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get()); 718 hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count", 719 true); 720 hideRedactedValueCount.setArgumentGroupName( 721 INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get()); 722 parser.addArgument(hideRedactedValueCount); 723 parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute); 724 725 726 // Add arguments pertaining to excluding attributes and entries. 727 excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0, 728 INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(), 729 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get()); 730 excludeAttribute.addLongIdentifier("suppressAttribute", true); 731 excludeAttribute.addLongIdentifier("exclude-attribute", true); 732 excludeAttribute.addLongIdentifier("suppress-attribute", true); 733 excludeAttribute.setArgumentGroupName( 734 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 735 parser.addArgument(excludeAttribute); 736 737 excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1, 738 null, 739 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get( 740 sourceContainsChangeRecords.getIdentifierString())); 741 excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true); 742 excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true); 743 excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true); 744 excludeEntryBaseDN.setArgumentGroupName( 745 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 746 parser.addArgument(excludeEntryBaseDN); 747 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 748 excludeEntryBaseDN); 749 750 excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false, 751 null, 752 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get( 753 sourceContainsChangeRecords.getIdentifierString())); 754 excludeEntryScope.addLongIdentifier("suppressEntryScope", true); 755 excludeEntryScope.addLongIdentifier("exclude-entry-scope", true); 756 excludeEntryScope.addLongIdentifier("suppress-entry-scope", true); 757 excludeEntryScope.setArgumentGroupName( 758 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 759 parser.addArgument(excludeEntryScope); 760 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 761 excludeEntryScope); 762 763 excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false, 764 1, null, 765 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get( 766 sourceContainsChangeRecords.getIdentifierString())); 767 excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true); 768 excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true); 769 excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true); 770 excludeEntryFilter.setArgumentGroupName( 771 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 772 parser.addArgument(excludeEntryFilter); 773 parser.addExclusiveArgumentSet(sourceContainsChangeRecords, 774 excludeEntryFilter); 775 776 excludeNonMatchingEntries = new BooleanArgument(null, 777 "excludeNonMatchingEntries", 778 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get()); 779 excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries", 780 true); 781 excludeNonMatchingEntries.setArgumentGroupName( 782 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 783 parser.addArgument(excludeNonMatchingEntries); 784 parser.addDependentArgumentSet(excludeNonMatchingEntries, 785 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter); 786 787 788 // Add arguments for excluding records based on their change types. 789 excludeChangeType = new StringArgument(null, "excludeChangeType", 790 false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(), 791 INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(), 792 StaticUtils.setOf("add", "delete", "modify", "moddn")); 793 excludeChangeType.addLongIdentifier("exclude-change-type", true); 794 excludeChangeType.addLongIdentifier("exclude-changetype", true); 795 excludeChangeType.setArgumentGroupName( 796 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 797 parser.addArgument(excludeChangeType); 798 799 800 // Add arguments for excluding records that don't have a change type. 801 excludeRecordsWithoutChangeType = new BooleanArgument(null, 802 "excludeRecordsWithoutChangeType", 1, 803 INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get()); 804 excludeRecordsWithoutChangeType.addLongIdentifier( 805 "exclude-records-without-change-type", true); 806 excludeRecordsWithoutChangeType.addLongIdentifier( 807 "exclude-records-without-changetype", true); 808 excludeRecordsWithoutChangeType.setArgumentGroupName( 809 INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get()); 810 parser.addArgument(excludeRecordsWithoutChangeType); 811 812 813 // Add the remaining arguments. 814 schemaPath = new FileArgument(null, "schemaPath", false, 0, null, 815 INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(), 816 true, true, false, false); 817 schemaPath.addLongIdentifier("schemaFile", true); 818 schemaPath.addLongIdentifier("schemaDirectory", true); 819 schemaPath.addLongIdentifier("schema-path", true); 820 schemaPath.addLongIdentifier("schema-file", true); 821 schemaPath.addLongIdentifier("schema-directory", true); 822 parser.addArgument(schemaPath); 823 824 numThreads = new IntegerArgument('t', "numThreads", false, 1, null, 825 INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE, 826 1); 827 numThreads.addLongIdentifier("num-threads", true); 828 parser.addArgument(numThreads); 829 830 processDNs = new BooleanArgument('d', "processDNs", 831 INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get()); 832 processDNs.addLongIdentifier("process-dns", true); 833 parser.addArgument(processDNs); 834 835 836 // Ensure that at least one kind of transformation was requested. 837 parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute, 838 replaceValuesAttribute, addAttributeName, renameAttributeFrom, 839 flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute, 840 excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter, 841 excludeChangeType, excludeRecordsWithoutChangeType); 842 } 843 844 845 846 /** 847 * {@inheritDoc} 848 */ 849 @Override() 850 public void doExtendedArgumentValidation() 851 throws ArgumentException 852 { 853 // Ideally, exactly one of the targetLDIF and targetToStandardOutput 854 // arguments should always be provided. But in order to preserve backward 855 // compatibility with a legacy scramble-ldif tool, we will allow both to be 856 // omitted if either --scrambleAttribute or --sequentialArgument is 857 // provided. In that case, the path of the output file will be the path of 858 // the first input file with ".scrambled" appended to it. 859 if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent())) 860 { 861 if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent())) 862 { 863 throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get( 864 targetLDIF.getIdentifierString(), 865 targetToStandardOutput.getIdentifierString())); 866 } 867 } 868 869 870 // Make sure that the --renameAttributeFrom and --renameAttributeTo 871 // arguments were provided an equal number of times. 872 final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences(); 873 final int renameToOccurrences = renameAttributeTo.getNumOccurrences(); 874 if (renameFromOccurrences != renameToOccurrences) 875 { 876 throw new ArgumentException( 877 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 878 renameAttributeFrom.getIdentifierString(), 879 renameAttributeTo.getIdentifierString())); 880 } 881 882 883 // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were 884 // provided an equal number of times. 885 final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences(); 886 final int moveToOccurrences = moveSubtreeTo.getNumOccurrences(); 887 if (moveFromOccurrences != moveToOccurrences) 888 { 889 throw new ArgumentException( 890 ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get( 891 moveSubtreeFrom.getIdentifierString(), 892 moveSubtreeTo.getIdentifierString())); 893 } 894 } 895 896 897 898 /** 899 * {@inheritDoc} 900 */ 901 @Override() 902 @NotNull() 903 public ResultCode doToolProcessing() 904 { 905 final Schema schema; 906 try 907 { 908 schema = getSchema(); 909 } 910 catch (final LDAPException le) 911 { 912 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage()); 913 return le.getResultCode(); 914 } 915 916 917 // If an encryption passphrase file is provided, then get the passphrase 918 // from it. 919 String encryptionPassphrase = null; 920 if (encryptionPassphraseFile.isPresent()) 921 { 922 try 923 { 924 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 925 encryptionPassphraseFile.getValue()); 926 } 927 catch (final LDAPException e) 928 { 929 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage()); 930 return e.getResultCode(); 931 } 932 } 933 934 935 // Create the translators to use to apply the transformations. 936 final ArrayList<LDIFReaderEntryTranslator> entryTranslators = 937 new ArrayList<>(10); 938 final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators = 939 new ArrayList<>(10); 940 941 final AtomicLong excludedEntryCount = new AtomicLong(0L); 942 createTranslators(entryTranslators, changeRecordTranslators, 943 schema, excludedEntryCount); 944 945 final AggregateLDIFReaderEntryTranslator entryTranslator = 946 new AggregateLDIFReaderEntryTranslator(entryTranslators); 947 final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator = 948 new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators); 949 950 951 // Determine the path to the target file to be written. 952 final File targetFile; 953 if (targetLDIF.isPresent()) 954 { 955 targetFile = targetLDIF.getValue(); 956 } 957 else if (targetToStandardOutput.isPresent()) 958 { 959 targetFile = null; 960 } 961 else 962 { 963 targetFile = 964 new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled"); 965 } 966 967 968 // Create the LDIF reader. 969 final LDIFReader ldifReader; 970 try 971 { 972 final InputStream inputStream; 973 if (sourceLDIF.isPresent()) 974 { 975 final ObjectPair<InputStream,String> p = 976 ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(), 977 encryptionPassphrase, getOut(), getErr()); 978 inputStream = p.getFirst(); 979 if ((encryptionPassphrase == null) && (p.getSecond() != null)) 980 { 981 encryptionPassphrase = p.getSecond(); 982 } 983 } 984 else 985 { 986 inputStream = System.in; 987 } 988 989 ldifReader = new LDIFReader(inputStream, numThreads.getValue(), 990 entryTranslator, changeRecordTranslator); 991 if (schema != null) 992 { 993 ldifReader.setSchema(schema); 994 } 995 } 996 catch (final Exception e) 997 { 998 Debug.debugException(e); 999 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1000 ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get( 1001 StaticUtils.getExceptionMessage(e))); 1002 return ResultCode.LOCAL_ERROR; 1003 } 1004 1005 1006 ResultCode resultCode = ResultCode.SUCCESS; 1007 OutputStream outputStream = null; 1008processingBlock: 1009 try 1010 { 1011 // Create the output stream to use to write the transformed data. 1012 try 1013 { 1014 if (targetFile == null) 1015 { 1016 outputStream = getOut(); 1017 } 1018 else 1019 { 1020 outputStream = 1021 new FileOutputStream(targetFile, appendToTargetLDIF.isPresent()); 1022 } 1023 1024 if (encryptTarget.isPresent()) 1025 { 1026 if (encryptionPassphrase == null) 1027 { 1028 encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase( 1029 false, true, getOut(), getErr()); 1030 } 1031 1032 outputStream = new PassphraseEncryptedOutputStream( 1033 encryptionPassphrase, outputStream); 1034 } 1035 1036 if (compressTarget.isPresent()) 1037 { 1038 outputStream = new GZIPOutputStream(outputStream); 1039 } 1040 } 1041 catch (final Exception e) 1042 { 1043 Debug.debugException(e); 1044 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1045 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get( 1046 targetFile.getAbsolutePath(), 1047 StaticUtils.getExceptionMessage(e))); 1048 resultCode = ResultCode.LOCAL_ERROR; 1049 break processingBlock; 1050 } 1051 1052 1053 // Read the source data one record at a time. The transformations will 1054 // automatically be applied by the LDIF reader's translators, and even if 1055 // there are multiple reader threads, we're guaranteed to get the results 1056 // in the right order. 1057 long entriesWritten = 0L; 1058 while (true) 1059 { 1060 final LDIFRecord ldifRecord; 1061 try 1062 { 1063 ldifRecord = ldifReader.readLDIFRecord(); 1064 } 1065 catch (final LDIFException le) 1066 { 1067 Debug.debugException(le); 1068 if (le.mayContinueReading()) 1069 { 1070 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1071 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get( 1072 StaticUtils.getExceptionMessage(le))); 1073 if (resultCode == ResultCode.SUCCESS) 1074 { 1075 resultCode = ResultCode.PARAM_ERROR; 1076 } 1077 continue; 1078 } 1079 else 1080 { 1081 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1082 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get( 1083 StaticUtils.getExceptionMessage(le))); 1084 if (resultCode == ResultCode.SUCCESS) 1085 { 1086 resultCode = ResultCode.PARAM_ERROR; 1087 } 1088 break processingBlock; 1089 } 1090 } 1091 catch (final Exception e) 1092 { 1093 Debug.debugException(e); 1094 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1095 ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get( 1096 StaticUtils.getExceptionMessage(e))); 1097 resultCode = ResultCode.LOCAL_ERROR; 1098 break processingBlock; 1099 } 1100 1101 1102 // If the LDIF record is null, then we've run out of records so we're 1103 // done. 1104 if (ldifRecord == null) 1105 { 1106 break; 1107 } 1108 1109 1110 // Write the record to the output stream. 1111 try 1112 { 1113 if (ldifRecord instanceof PreEncodedLDIFEntry) 1114 { 1115 outputStream.write( 1116 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes()); 1117 } 1118 else 1119 { 1120 final ByteStringBuffer buffer = getBuffer(); 1121 if (wrapColumn.isPresent()) 1122 { 1123 ldifRecord.toLDIF(buffer, wrapColumn.getValue()); 1124 } 1125 else 1126 { 1127 ldifRecord.toLDIF(buffer, 0); 1128 } 1129 buffer.append(StaticUtils.EOL_BYTES); 1130 buffer.write(outputStream); 1131 } 1132 } 1133 catch (final Exception e) 1134 { 1135 Debug.debugException(e); 1136 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1137 ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(), 1138 StaticUtils.getExceptionMessage(e))); 1139 resultCode = ResultCode.LOCAL_ERROR; 1140 break processingBlock; 1141 } 1142 1143 1144 // If we've written a multiple of 1000 entries, print a progress 1145 // message. 1146 entriesWritten++; 1147 if ((! targetToStandardOutput.isPresent()) && 1148 ((entriesWritten % 1000L) == 0)) 1149 { 1150 final long numExcluded = excludedEntryCount.get(); 1151 if (numExcluded > 0L) 1152 { 1153 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1154 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get( 1155 entriesWritten, numExcluded)); 1156 } 1157 else 1158 { 1159 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1160 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get( 1161 entriesWritten)); 1162 } 1163 } 1164 } 1165 1166 1167 if (! targetToStandardOutput.isPresent()) 1168 { 1169 final long numExcluded = excludedEntryCount.get(); 1170 if (numExcluded > 0L) 1171 { 1172 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1173 INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten, 1174 numExcluded)); 1175 } 1176 else 1177 { 1178 wrapOut(0, MAX_OUTPUT_LINE_LENGTH, 1179 INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten)); 1180 } 1181 } 1182 } 1183 finally 1184 { 1185 if (outputStream != null) 1186 { 1187 try 1188 { 1189 outputStream.close(); 1190 } 1191 catch (final Exception e) 1192 { 1193 Debug.debugException(e); 1194 wrapErr(0, MAX_OUTPUT_LINE_LENGTH, 1195 ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get( 1196 targetFile.getAbsolutePath(), 1197 StaticUtils.getExceptionMessage(e))); 1198 if (resultCode == ResultCode.SUCCESS) 1199 { 1200 resultCode = ResultCode.LOCAL_ERROR; 1201 } 1202 } 1203 } 1204 1205 try 1206 { 1207 ldifReader.close(); 1208 } 1209 catch (final Exception e) 1210 { 1211 Debug.debugException(e); 1212 // We can ignore this. 1213 } 1214 } 1215 1216 1217 return resultCode; 1218 } 1219 1220 1221 1222 /** 1223 * Retrieves the schema that should be used for processing. 1224 * 1225 * @return The schema that was created. 1226 * 1227 * @throws LDAPException If a problem is encountered while retrieving the 1228 * schema. 1229 */ 1230 @Nullable() 1231 private Schema getSchema() 1232 throws LDAPException 1233 { 1234 // If any schema paths were specified, then load the schema only from those 1235 // paths. 1236 if (schemaPath.isPresent()) 1237 { 1238 final ArrayList<File> schemaFiles = new ArrayList<>(10); 1239 for (final File path : schemaPath.getValues()) 1240 { 1241 if (path.isFile()) 1242 { 1243 schemaFiles.add(path); 1244 } 1245 else 1246 { 1247 final TreeMap<String,File> fileMap = new TreeMap<>(); 1248 for (final File schemaDirFile : path.listFiles()) 1249 { 1250 final String name = schemaDirFile.getName(); 1251 if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif")) 1252 { 1253 fileMap.put(name, schemaDirFile); 1254 } 1255 } 1256 schemaFiles.addAll(fileMap.values()); 1257 } 1258 } 1259 1260 if (schemaFiles.isEmpty()) 1261 { 1262 throw new LDAPException(ResultCode.PARAM_ERROR, 1263 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get( 1264 schemaPath.getIdentifierString())); 1265 } 1266 else 1267 { 1268 try 1269 { 1270 return Schema.getSchema(schemaFiles); 1271 } 1272 catch (final Exception e) 1273 { 1274 Debug.debugException(e); 1275 throw new LDAPException(ResultCode.LOCAL_ERROR, 1276 ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get( 1277 StaticUtils.getExceptionMessage(e))); 1278 } 1279 } 1280 } 1281 else 1282 { 1283 // If the INSTANCE_ROOT environment variable is set and it refers to a 1284 // directory that has a config/schema subdirectory that has one or more 1285 // schema files in it, then read the schema from that directory. 1286 try 1287 { 1288 final String instanceRootStr = 1289 StaticUtils.getEnvironmentVariable("INSTANCE_ROOT"); 1290 if (instanceRootStr != null) 1291 { 1292 final File instanceRoot = new File(instanceRootStr); 1293 final File configDir = new File(instanceRoot, "config"); 1294 final File schemaDir = new File(configDir, "schema"); 1295 if (schemaDir.exists()) 1296 { 1297 final TreeMap<String,File> fileMap = new TreeMap<>(); 1298 for (final File schemaDirFile : schemaDir.listFiles()) 1299 { 1300 final String name = schemaDirFile.getName(); 1301 if (schemaDirFile.isFile() && 1302 name.toLowerCase().endsWith(".ldif")) 1303 { 1304 fileMap.put(name, schemaDirFile); 1305 } 1306 } 1307 1308 if (! fileMap.isEmpty()) 1309 { 1310 return Schema.getSchema(new ArrayList<>(fileMap.values())); 1311 } 1312 } 1313 } 1314 } 1315 catch (final Exception e) 1316 { 1317 Debug.debugException(e); 1318 } 1319 } 1320 1321 1322 // If we've gotten here, then just return null and the tool will try to use 1323 // the default standard schema. 1324 return null; 1325 } 1326 1327 1328 1329 /** 1330 * Creates the entry and change record translators that will be used to 1331 * perform the transformations. 1332 * 1333 * @param entryTranslators A list to which all created entry 1334 * translators should be written. 1335 * @param changeRecordTranslators A list to which all created change record 1336 * translators should be written. 1337 * @param schema The schema to use when processing. 1338 * @param excludedEntryCount A counter used to keep track of the number 1339 * of entries that have been excluded from 1340 * the result set. 1341 */ 1342 private void createTranslators( 1343 @NotNull final List<LDIFReaderEntryTranslator> entryTranslators, 1344 @NotNull final List<LDIFReaderChangeRecordTranslator> 1345 changeRecordTranslators, 1346 @Nullable final Schema schema, 1347 @NotNull final AtomicLong excludedEntryCount) 1348 { 1349 if (scrambleAttribute.isPresent()) 1350 { 1351 final Long seed; 1352 if (randomSeed.isPresent()) 1353 { 1354 seed = randomSeed.getValue().longValue(); 1355 } 1356 else 1357 { 1358 seed = null; 1359 } 1360 1361 final ScrambleAttributeTransformation t = 1362 new ScrambleAttributeTransformation(schema, seed, 1363 processDNs.isPresent(), scrambleAttribute.getValues(), 1364 scrambleJSONField.getValues()); 1365 entryTranslators.add(t); 1366 changeRecordTranslators.add(t); 1367 } 1368 1369 if (sequentialAttribute.isPresent()) 1370 { 1371 final long initialValue; 1372 if (initialSequentialValue.isPresent()) 1373 { 1374 initialValue = initialSequentialValue.getValue().longValue(); 1375 } 1376 else 1377 { 1378 initialValue = 0L; 1379 } 1380 1381 final long incrementAmount; 1382 if (sequentialValueIncrement.isPresent()) 1383 { 1384 incrementAmount = sequentialValueIncrement.getValue().longValue(); 1385 } 1386 else 1387 { 1388 incrementAmount = 1L; 1389 } 1390 1391 for (final String attrName : sequentialAttribute.getValues()) 1392 { 1393 1394 1395 final ReplaceWithCounterTransformation t = 1396 new ReplaceWithCounterTransformation(schema, attrName, 1397 initialValue, incrementAmount, 1398 textBeforeSequentialValue.getValue(), 1399 textAfterSequentialValue.getValue(), processDNs.isPresent()); 1400 entryTranslators.add(t); 1401 } 1402 } 1403 1404 if (replaceValuesAttribute.isPresent()) 1405 { 1406 final ReplaceAttributeTransformation t = 1407 new ReplaceAttributeTransformation(schema, 1408 replaceValuesAttribute.getValue(), 1409 replacementValue.getValues()); 1410 entryTranslators.add(t); 1411 } 1412 1413 if (addAttributeName.isPresent()) 1414 { 1415 final AddAttributeTransformation t = new AddAttributeTransformation( 1416 schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(), 1417 addAttributeFilter.getValue(), 1418 new Attribute(addAttributeName.getValue(), schema, 1419 addAttributeValue.getValues()), 1420 (! addToExistingValues.isPresent())); 1421 entryTranslators.add(t); 1422 } 1423 1424 if (renameAttributeFrom.isPresent()) 1425 { 1426 final Iterator<String> renameFromIterator = 1427 renameAttributeFrom.getValues().iterator(); 1428 final Iterator<String> renameToIterator = 1429 renameAttributeTo.getValues().iterator(); 1430 while (renameFromIterator.hasNext()) 1431 { 1432 final RenameAttributeTransformation t = 1433 new RenameAttributeTransformation(schema, 1434 renameFromIterator.next(), renameToIterator.next(), 1435 processDNs.isPresent()); 1436 entryTranslators.add(t); 1437 changeRecordTranslators.add(t); 1438 } 1439 } 1440 1441 if (flattenBaseDN.isPresent()) 1442 { 1443 final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation( 1444 schema, flattenBaseDN.getValue(), 1445 flattenAddOmittedRDNAttributesToEntry.isPresent(), 1446 flattenAddOmittedRDNAttributesToRDN.isPresent(), 1447 flattenExcludeFilter.getValue()); 1448 entryTranslators.add(t); 1449 } 1450 1451 if (moveSubtreeFrom.isPresent()) 1452 { 1453 final Iterator<DN> moveFromIterator = 1454 moveSubtreeFrom.getValues().iterator(); 1455 final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator(); 1456 while (moveFromIterator.hasNext()) 1457 { 1458 final MoveSubtreeTransformation t = 1459 new MoveSubtreeTransformation(moveFromIterator.next(), 1460 moveToIterator.next()); 1461 entryTranslators.add(t); 1462 changeRecordTranslators.add(t); 1463 } 1464 } 1465 1466 if (redactAttribute.isPresent()) 1467 { 1468 final RedactAttributeTransformation t = new RedactAttributeTransformation( 1469 schema, processDNs.isPresent(), 1470 (! hideRedactedValueCount.isPresent()), redactAttribute.getValues()); 1471 entryTranslators.add(t); 1472 changeRecordTranslators.add(t); 1473 } 1474 1475 if (excludeAttribute.isPresent()) 1476 { 1477 final ExcludeAttributeTransformation t = 1478 new ExcludeAttributeTransformation(schema, 1479 excludeAttribute.getValues()); 1480 entryTranslators.add(t); 1481 changeRecordTranslators.add(t); 1482 } 1483 1484 if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() || 1485 excludeEntryFilter.isPresent()) 1486 { 1487 final ExcludeEntryTransformation t = new ExcludeEntryTransformation( 1488 schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(), 1489 excludeEntryFilter.getValue(), 1490 (! excludeNonMatchingEntries.isPresent()), excludedEntryCount); 1491 entryTranslators.add(t); 1492 } 1493 1494 if (excludeChangeType.isPresent()) 1495 { 1496 final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class); 1497 for (final String changeTypeName : excludeChangeType.getValues()) 1498 { 1499 changeTypes.add(ChangeType.forName(changeTypeName)); 1500 } 1501 1502 changeRecordTranslators.add( 1503 new ExcludeChangeTypeTransformation(changeTypes)); 1504 } 1505 1506 if (excludeRecordsWithoutChangeType.isPresent()) 1507 { 1508 entryTranslators.add(new ExcludeAllEntriesTransformation()); 1509 } 1510 1511 entryTranslators.add(this); 1512 } 1513 1514 1515 1516 /** 1517 * {@inheritDoc} 1518 */ 1519 @Override() 1520 @NotNull() 1521 public LinkedHashMap<String[],String> getExampleUsages() 1522 { 1523 final LinkedHashMap<String[],String> examples = 1524 new LinkedHashMap<>(StaticUtils.computeMapCapacity(4)); 1525 1526 examples.put( 1527 new String[] 1528 { 1529 "--sourceLDIF", "input.ldif", 1530 "--targetLDIF", "scrambled.ldif", 1531 "--scrambleAttribute", "givenName", 1532 "--scrambleAttribute", "sn", 1533 "--scrambleAttribute", "cn", 1534 "--numThreads", "10", 1535 "--schemaPath", "/ds/config/schema", 1536 "--processDNs" 1537 }, 1538 INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get()); 1539 1540 examples.put( 1541 new String[] 1542 { 1543 "--sourceLDIF", "input.ldif", 1544 "--targetLDIF", "sequential.ldif", 1545 "--sequentialAttribute", "uid", 1546 "--initialSequentialValue", "1", 1547 "--sequentialValueIncrement", "1", 1548 "--textBeforeSequentialValue", "user.", 1549 "--numThreads", "10", 1550 "--schemaPath", "/ds/config/schema", 1551 "--processDNs" 1552 }, 1553 INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get()); 1554 1555 examples.put( 1556 new String[] 1557 { 1558 "--sourceLDIF", "input.ldif", 1559 "--targetLDIF", "added-organization.ldif", 1560 "--addAttributeName", "o", 1561 "--addAttributeValue", "Example Corp.", 1562 "--addAttributeFilter", "(objectClass=person)", 1563 "--numThreads", "10", 1564 "--schemaPath", "/ds/config/schema" 1565 }, 1566 INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get()); 1567 1568 examples.put( 1569 new String[] 1570 { 1571 "--sourceLDIF", "input.ldif", 1572 "--targetLDIF", "rebased.ldif", 1573 "--moveSubtreeFrom", "o=example.com", 1574 "--moveSubtreeTo", "dc=example,dc=com", 1575 "--numThreads", "10", 1576 "--schemaPath", "/ds/config/schema" 1577 }, 1578 INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get()); 1579 1580 return examples; 1581 } 1582 1583 1584 1585 /** 1586 * {@inheritDoc} 1587 */ 1588 @Override() 1589 @Nullable() 1590 public Entry translate(@NotNull final Entry original, 1591 final long firstLineNumber) 1592 throws LDIFException 1593 { 1594 final ByteStringBuffer buffer = getBuffer(); 1595 if (wrapColumn.isPresent()) 1596 { 1597 original.toLDIF(buffer, wrapColumn.getValue()); 1598 } 1599 else 1600 { 1601 original.toLDIF(buffer, 0); 1602 } 1603 buffer.append(StaticUtils.EOL_BYTES); 1604 1605 return new PreEncodedLDIFEntry(original, buffer.toByteArray()); 1606 } 1607 1608 1609 1610 /** 1611 * Retrieves a byte string buffer that can be used to perform LDIF encoding. 1612 * 1613 * @return A byte string buffer that can be used to perform LDIF encoding. 1614 */ 1615 @NotNull() 1616 private ByteStringBuffer getBuffer() 1617 { 1618 ByteStringBuffer buffer = byteStringBuffers.get(); 1619 if (buffer == null) 1620 { 1621 buffer = new ByteStringBuffer(); 1622 byteStringBuffers.set(buffer); 1623 } 1624 else 1625 { 1626 buffer.clear(); 1627 } 1628 1629 return buffer; 1630 } 1631}