001/*
002 * Copyright 2016-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2016-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2016-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.transformations;
037
038
039
040import java.io.File;
041import java.io.FileOutputStream;
042import java.io.InputStream;
043import java.io.OutputStream;
044import java.util.ArrayList;
045import java.util.EnumSet;
046import java.util.Iterator;
047import java.util.LinkedHashMap;
048import java.util.List;
049import java.util.Set;
050import java.util.TreeMap;
051import java.util.concurrent.atomic.AtomicLong;
052import java.util.zip.GZIPOutputStream;
053
054import com.unboundid.ldap.sdk.Attribute;
055import com.unboundid.ldap.sdk.ChangeType;
056import com.unboundid.ldap.sdk.DN;
057import com.unboundid.ldap.sdk.Entry;
058import com.unboundid.ldap.sdk.LDAPException;
059import com.unboundid.ldap.sdk.ResultCode;
060import com.unboundid.ldap.sdk.Version;
061import com.unboundid.ldap.sdk.schema.Schema;
062import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
063import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
064import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
065import com.unboundid.ldif.LDIFException;
066import com.unboundid.ldif.LDIFReader;
067import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
068import com.unboundid.ldif.LDIFReaderEntryTranslator;
069import com.unboundid.ldif.LDIFRecord;
070import com.unboundid.util.ByteStringBuffer;
071import com.unboundid.util.CommandLineTool;
072import com.unboundid.util.Debug;
073import com.unboundid.util.NotNull;
074import com.unboundid.util.Nullable;
075import com.unboundid.util.ObjectPair;
076import com.unboundid.util.PassphraseEncryptedOutputStream;
077import com.unboundid.util.StaticUtils;
078import com.unboundid.util.ThreadSafety;
079import com.unboundid.util.ThreadSafetyLevel;
080import com.unboundid.util.args.ArgumentException;
081import com.unboundid.util.args.ArgumentParser;
082import com.unboundid.util.args.BooleanArgument;
083import com.unboundid.util.args.DNArgument;
084import com.unboundid.util.args.FileArgument;
085import com.unboundid.util.args.FilterArgument;
086import com.unboundid.util.args.IntegerArgument;
087import com.unboundid.util.args.ScopeArgument;
088import com.unboundid.util.args.StringArgument;
089
090import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
091
092
093
094/**
095 * This class provides a command-line tool that can be used to apply a number of
096 * transformations to an LDIF file.  The transformations that can be applied
097 * include:
098 * <UL>
099 *   <LI>
100 *     It can scramble the values of a specified set of attributes in a manner
101 *     that attempts to preserve the syntax and consistently scrambles the same
102 *     value to the same representation.
103 *   </LI>
104 *   <LI>
105 *     It can strip a specified set of attributes out of entries.
106 *   </LI>
107 *   <LI>
108 *     It can redact the values of a specified set of attributes, to indicate
109 *     that the values are there but providing no information about what their
110 *     values are.
111 *   </LI>
112 *   <LI>
113 *     It can replace the values of a specified attribute with a given set of
114 *     values.
115 *   </LI>
116 *   <LI>
117 *     It can add an attribute with a given set of values to any entry that does
118 *     not contain that attribute.
119 *   </LI>
120 *   <LI>
121 *     It can replace the values of a specified attribute with a value that
122 *     contains a sequentially-incrementing counter.
123 *   </LI>
124 *   <LI>
125 *     It can strip entries matching a given base DN, scope, and filter out of
126 *     the LDIF file.
127 *   </LI>
128 *   <LI>
129 *     It can perform DN mapping, so that entries that exist below one base DN
130 *     are moved below a different base DN.
131 *   </LI>
132 *   <LI>
133 *     It can perform attribute mapping, to replace uses of one attribute name
134 *     with another.
135 *   </LI>
136 * </UL>
137 */
138@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
139public final class TransformLDIF
140       extends CommandLineTool
141       implements LDIFReaderEntryTranslator
142{
143  /**
144   * The maximum length of any message to write to standard output or standard
145   * error.
146   */
147  private static final int MAX_OUTPUT_LINE_LENGTH =
148       StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
149
150
151
152  // The arguments for use by this program.
153  @Nullable private BooleanArgument addToExistingValues = null;
154  @Nullable private BooleanArgument appendToTargetLDIF = null;
155  @Nullable private BooleanArgument compressTarget = null;
156  @Nullable private BooleanArgument encryptTarget = null;
157  @Nullable private BooleanArgument excludeRecordsWithoutChangeType = null;
158  @Nullable private BooleanArgument excludeNonMatchingEntries = null;
159  @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToEntry =
160       null;
161  @Nullable private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
162  @Nullable private BooleanArgument hideRedactedValueCount = null;
163  @Nullable private BooleanArgument processDNs = null;
164  @Nullable private BooleanArgument sourceCompressed = null;
165  @Nullable private BooleanArgument sourceContainsChangeRecords = null;
166  @Nullable private BooleanArgument sourceFromStandardInput = null;
167  @Nullable private BooleanArgument targetToStandardOutput = null;
168  @Nullable private DNArgument addAttributeBaseDN = null;
169  @Nullable private DNArgument excludeEntryBaseDN = null;
170  @Nullable private DNArgument flattenBaseDN = null;
171  @Nullable private DNArgument moveSubtreeFrom = null;
172  @Nullable private DNArgument moveSubtreeTo = null;
173  @Nullable private FileArgument encryptionPassphraseFile = null;
174  @Nullable private FileArgument schemaPath = null;
175  @Nullable private FileArgument sourceLDIF = null;
176  @Nullable private FileArgument targetLDIF = null;
177  @Nullable private FilterArgument addAttributeFilter = null;
178  @Nullable private FilterArgument excludeEntryFilter = null;
179  @Nullable private FilterArgument flattenExcludeFilter = null;
180  @Nullable private IntegerArgument initialSequentialValue = null;
181  @Nullable private IntegerArgument numThreads = null;
182  @Nullable private IntegerArgument randomSeed = null;
183  @Nullable private IntegerArgument sequentialValueIncrement = null;
184  @Nullable private IntegerArgument wrapColumn = null;
185  @Nullable private ScopeArgument addAttributeScope = null;
186  @Nullable private ScopeArgument excludeEntryScope = null;
187  @Nullable private StringArgument addAttributeName = null;
188  @Nullable private StringArgument addAttributeValue = null;
189  @Nullable private StringArgument excludeAttribute = null;
190  @Nullable private StringArgument excludeChangeType  = null;
191  @Nullable private StringArgument redactAttribute = null;
192  @Nullable private StringArgument renameAttributeFrom = null;
193  @Nullable private StringArgument renameAttributeTo = null;
194  @Nullable private StringArgument replaceValuesAttribute = null;
195  @Nullable private StringArgument replacementValue = null;
196  @Nullable private StringArgument scrambleAttribute = null;
197  @Nullable private StringArgument scrambleJSONField = null;
198  @Nullable private StringArgument sequentialAttribute = null;
199  @Nullable private StringArgument textAfterSequentialValue = null;
200  @Nullable private StringArgument textBeforeSequentialValue = null;
201
202  // A set of thread-local byte stream buffers that will be used to construct
203  // the LDIF representations of records.
204  @NotNull private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
205       new ThreadLocal<>();
206
207
208
209  /**
210   * Invokes this tool with the provided set of arguments.
211   *
212   * @param  args  The command-line arguments provided to this program.
213   */
214  public static void main(@NotNull final String... args)
215  {
216    final ResultCode resultCode = main(System.out, System.err, args);
217    if (resultCode != ResultCode.SUCCESS)
218    {
219      System.exit(resultCode.intValue());
220    }
221  }
222
223
224
225  /**
226   * Invokes this tool with the provided set of arguments.
227   *
228   * @param  out   The output stream to use for standard output.  It may be
229   *               {@code null} if standard output should be suppressed.
230   * @param  err   The output stream to use for standard error.  It may be
231   *               {@code null} if standard error should be suppressed.
232   * @param  args  The command-line arguments provided to this program.
233   *
234   * @return  A result code indicating whether processing completed
235   *          successfully.
236   */
237  @NotNull()
238  public static ResultCode main(@Nullable final OutputStream out,
239                                @Nullable final OutputStream err,
240                                @NotNull final String... args)
241  {
242    final TransformLDIF tool = new TransformLDIF(out, err);
243    return tool.runTool(args);
244  }
245
246
247
248  /**
249   * Creates a new instance of this tool with the provided information.
250   *
251   * @param  out  The output stream to use for standard output.  It may be
252   *              {@code null} if standard output should be suppressed.
253   * @param  err  The output stream to use for standard error.  It may be
254   *              {@code null} if standard error should be suppressed.
255   */
256  public TransformLDIF(@Nullable final OutputStream out,
257                       @Nullable final OutputStream err)
258  {
259    super(out, err);
260  }
261
262
263
264  /**
265   * {@inheritDoc}
266   */
267  @Override()
268  @NotNull()
269  public String getToolName()
270  {
271    return "transform-ldif";
272  }
273
274
275
276  /**
277   * {@inheritDoc}
278   */
279  @Override()
280  @NotNull()
281  public String getToolDescription()
282  {
283    return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
284  }
285
286
287
288  /**
289   * {@inheritDoc}
290   */
291  @Override()
292  @NotNull()
293  public String getToolVersion()
294  {
295    return Version.NUMERIC_VERSION_STRING;
296  }
297
298
299
300  /**
301   * {@inheritDoc}
302   */
303  @Override()
304  public boolean supportsInteractiveMode()
305  {
306    return true;
307  }
308
309
310
311  /**
312   * {@inheritDoc}
313   */
314  @Override()
315  public boolean defaultsToInteractiveMode()
316  {
317    return true;
318  }
319
320
321
322  /**
323   * {@inheritDoc}
324   */
325  @Override()
326  public boolean supportsPropertiesFile()
327  {
328    return true;
329  }
330
331
332
333  /**
334   * {@inheritDoc}
335   */
336  @Override()
337  public void addToolArguments(@NotNull final ArgumentParser parser)
338         throws ArgumentException
339  {
340    // Add arguments pertaining to the source and target LDIF files.
341    sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
342         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
343         false);
344    sourceLDIF.addLongIdentifier("inputLDIF", true);
345    sourceLDIF.addLongIdentifier("source-ldif", true);
346    sourceLDIF.addLongIdentifier("input-ldif", true);
347    sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
348    parser.addArgument(sourceLDIF);
349
350    sourceFromStandardInput = new BooleanArgument(null,
351         "sourceFromStandardInput", 1,
352         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
353    sourceFromStandardInput.addLongIdentifier("source-from-standard-input",
354         true);
355    sourceFromStandardInput.setArgumentGroupName(
356         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
357    parser.addArgument(sourceFromStandardInput);
358    parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
359    parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
360
361    targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
362         INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
363         false);
364    targetLDIF.addLongIdentifier("outputLDIF", true);
365    targetLDIF.addLongIdentifier("target-ldif", true);
366    targetLDIF.addLongIdentifier("output-ldif", true);
367    targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
368    parser.addArgument(targetLDIF);
369
370    targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
371         1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
372    targetToStandardOutput.addLongIdentifier("target-to-standard-output", true);
373    targetToStandardOutput.setArgumentGroupName(
374         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
375    parser.addArgument(targetToStandardOutput);
376    parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
377
378    sourceContainsChangeRecords = new BooleanArgument(null,
379         "sourceContainsChangeRecords",
380         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
381    sourceContainsChangeRecords.addLongIdentifier(
382         "source-contains-change-records", true);
383    sourceContainsChangeRecords.setArgumentGroupName(
384         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
385    parser.addArgument(sourceContainsChangeRecords);
386
387    appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
388         INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
389    appendToTargetLDIF.addLongIdentifier("append-to-target-ldif", true);
390    appendToTargetLDIF.setArgumentGroupName(
391         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
392    parser.addArgument(appendToTargetLDIF);
393    parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
394
395    wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
396         INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
397    wrapColumn.addLongIdentifier("wrap-column", true);
398    wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
399    parser.addArgument(wrapColumn);
400
401    sourceCompressed = new BooleanArgument('C', "sourceCompressed",
402         INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
403    sourceCompressed.addLongIdentifier("inputCompressed", true);
404    sourceCompressed.addLongIdentifier("source-compressed", true);
405    sourceCompressed.addLongIdentifier("input-compressed", true);
406    sourceCompressed.setArgumentGroupName(
407         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
408    parser.addArgument(sourceCompressed);
409
410    compressTarget = new BooleanArgument('c', "compressTarget",
411         INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
412    compressTarget.addLongIdentifier("compressOutput", true);
413    compressTarget.addLongIdentifier("compress", true);
414    compressTarget.addLongIdentifier("compress-target", true);
415    compressTarget.addLongIdentifier("compress-output", true);
416    compressTarget.setArgumentGroupName(
417         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
418    parser.addArgument(compressTarget);
419
420    encryptTarget = new BooleanArgument(null, "encryptTarget",
421         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPT_TARGET.get());
422    encryptTarget.addLongIdentifier("encryptOutput", true);
423    encryptTarget.addLongIdentifier("encrypt", true);
424    encryptTarget.addLongIdentifier("encrypt-target", true);
425    encryptTarget.addLongIdentifier("encrypt-output", true);
426    encryptTarget.setArgumentGroupName(
427         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
428    parser.addArgument(encryptTarget);
429
430    encryptionPassphraseFile = new FileArgument(null,
431         "encryptionPassphraseFile", false, 1, null,
432         INFO_TRANSFORM_LDIF_ARG_DESC_ENCRYPTION_PW_FILE.get(), true, true,
433         true, false);
434    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
435    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
436         true);
437    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
438         true);
439    encryptionPassphraseFile.setArgumentGroupName(
440         INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
441    parser.addArgument(encryptionPassphraseFile);
442
443
444    // Add arguments pertaining to attribute scrambling.
445    scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
446         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
447         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
448    scrambleAttribute.addLongIdentifier("attributeName", true);
449    scrambleAttribute.addLongIdentifier("scramble-attribute", true);
450    scrambleAttribute.addLongIdentifier("attribute-name", true);
451    scrambleAttribute.setArgumentGroupName(
452         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
453    parser.addArgument(scrambleAttribute);
454
455    scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
456         INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
457         INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
458              scrambleAttribute.getIdentifierString()));
459    scrambleJSONField.addLongIdentifier("scramble-json-field", true);
460    scrambleJSONField.setArgumentGroupName(
461         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
462    parser.addArgument(scrambleJSONField);
463    parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
464
465    randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
466         INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
467    randomSeed.addLongIdentifier("random-seed", true);
468    randomSeed.setArgumentGroupName(
469         INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
470    parser.addArgument(randomSeed);
471
472
473    // Add arguments pertaining to replacing attribute values with a generated
474    // value using a sequential counter.
475    sequentialAttribute = new StringArgument('S', "sequentialAttribute",
476         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
477         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
478              sourceContainsChangeRecords.getIdentifierString()));
479    sequentialAttribute.addLongIdentifier("sequentialAttributeName", true);
480    sequentialAttribute.addLongIdentifier("sequential-attribute", true);
481    sequentialAttribute.addLongIdentifier("sequential-attribute-name", true);
482    sequentialAttribute.setArgumentGroupName(
483         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
484    parser.addArgument(sequentialAttribute);
485    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
486         sequentialAttribute);
487
488    initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
489         false, 1, null,
490         INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
491              sequentialAttribute.getIdentifierString()));
492    initialSequentialValue.addLongIdentifier("initial-sequential-value", true);
493    initialSequentialValue.setArgumentGroupName(
494         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
495    parser.addArgument(initialSequentialValue);
496    parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
497
498    sequentialValueIncrement = new IntegerArgument(null,
499         "sequentialValueIncrement", false, 1, null,
500         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
501              sequentialAttribute.getIdentifierString()));
502    sequentialValueIncrement.addLongIdentifier("sequential-value-increment",
503         true);
504    sequentialValueIncrement.setArgumentGroupName(
505         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
506    parser.addArgument(sequentialValueIncrement);
507    parser.addDependentArgumentSet(sequentialValueIncrement,
508         sequentialAttribute);
509
510    textBeforeSequentialValue = new StringArgument(null,
511         "textBeforeSequentialValue", false, 1, null,
512         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
513              sequentialAttribute.getIdentifierString()));
514    textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value",
515         true);
516    textBeforeSequentialValue.setArgumentGroupName(
517         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
518    parser.addArgument(textBeforeSequentialValue);
519    parser.addDependentArgumentSet(textBeforeSequentialValue,
520         sequentialAttribute);
521
522    textAfterSequentialValue = new StringArgument(null,
523         "textAfterSequentialValue", false, 1, null,
524         INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
525              sequentialAttribute.getIdentifierString()));
526    textAfterSequentialValue.addLongIdentifier("text-after-sequential-value",
527         true);
528    textAfterSequentialValue.setArgumentGroupName(
529         INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
530    parser.addArgument(textAfterSequentialValue);
531    parser.addDependentArgumentSet(textAfterSequentialValue,
532         sequentialAttribute);
533
534
535    // Add arguments pertaining to attribute value replacement.
536    replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
537         false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
538         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
539              sourceContainsChangeRecords.getIdentifierString()));
540    replaceValuesAttribute.addLongIdentifier("replace-values-attribute", true);
541    replaceValuesAttribute.setArgumentGroupName(
542         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
543    parser.addArgument(replaceValuesAttribute);
544    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
545         replaceValuesAttribute);
546
547    replacementValue = new StringArgument(null, "replacementValue", false, 0,
548         null,
549         INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
550              replaceValuesAttribute.getIdentifierString()));
551    replacementValue.addLongIdentifier("replacement-value", true);
552    replacementValue.setArgumentGroupName(
553         INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
554    parser.addArgument(replacementValue);
555    parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
556    parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
557
558
559    // Add arguments pertaining to adding missing attributes.
560    addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
561         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
562         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
563              "--addAttributeValue",
564              sourceContainsChangeRecords.getIdentifierString()));
565    addAttributeName.addLongIdentifier("add-attribute-name", true);
566    addAttributeName.setArgumentGroupName(
567         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
568    parser.addArgument(addAttributeName);
569    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
570         addAttributeName);
571
572    addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
573         null,
574         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
575              addAttributeName.getIdentifierString()));
576    addAttributeValue.addLongIdentifier("add-attribute-value", true);
577    addAttributeValue.setArgumentGroupName(
578         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
579    parser.addArgument(addAttributeValue);
580    parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
581    parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
582
583    addToExistingValues = new BooleanArgument(null, "addToExistingValues",
584         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
585              addAttributeName.getIdentifierString(),
586              addAttributeValue.getIdentifierString()));
587    addToExistingValues.addLongIdentifier("add-to-existing-values", true);
588    addToExistingValues.setArgumentGroupName(
589         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
590    parser.addArgument(addToExistingValues);
591    parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
592
593    addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
594         null,
595         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
596              addAttributeName.getIdentifierString()));
597    addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn", true);
598    addAttributeBaseDN.setArgumentGroupName(
599         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
600    parser.addArgument(addAttributeBaseDN);
601    parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
602
603    addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
604         null,
605         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
606              addAttributeBaseDN.getIdentifierString(),
607              addAttributeName.getIdentifierString()));
608    addAttributeScope.addLongIdentifier("add-attribute-scope", true);
609    addAttributeScope.setArgumentGroupName(
610         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
611    parser.addArgument(addAttributeScope);
612    parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
613
614    addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
615         1, null,
616         INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
617              addAttributeName.getIdentifierString()));
618    addAttributeFilter.addLongIdentifier("add-attribute-filter", true);
619    addAttributeFilter.setArgumentGroupName(
620         INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
621    parser.addArgument(addAttributeFilter);
622    parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
623
624
625    // Add arguments pertaining to renaming attributes.
626    renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
627         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
628         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
629    renameAttributeFrom.addLongIdentifier("rename-attribute-from", true);
630    renameAttributeFrom.setArgumentGroupName(
631         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
632    parser.addArgument(renameAttributeFrom);
633
634    renameAttributeTo = new StringArgument(null, "renameAttributeTo",
635         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
636         INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
637              renameAttributeFrom.getIdentifierString()));
638    renameAttributeTo.addLongIdentifier("rename-attribute-to", true);
639    renameAttributeTo.setArgumentGroupName(
640         INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
641    parser.addArgument(renameAttributeTo);
642    parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
643    parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
644
645
646    // Add arguments pertaining to flattening subtrees.
647    flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
648         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
649    flattenBaseDN.addLongIdentifier("flatten-base-dn", true);
650    flattenBaseDN.setArgumentGroupName(
651         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
652    parser.addArgument(flattenBaseDN);
653    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
654         flattenBaseDN);
655
656    flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
657         "flattenAddOmittedRDNAttributesToEntry", 1,
658         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
659    flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
660         "flatten-add-omitted-rdn-attributes-to-entry", true);
661    flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
662         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
663    parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
664    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
665         flattenBaseDN);
666
667    flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
668         "flattenAddOmittedRDNAttributesToRDN", 1,
669         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
670    flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
671         "flatten-add-omitted-rdn-attributes-to-rdn", true);
672    flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
673         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
674    parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
675    parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
676         flattenBaseDN);
677
678    flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
679         false, 1, null,
680         INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
681    flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter", true);
682    flattenExcludeFilter.setArgumentGroupName(
683         INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
684    parser.addArgument(flattenExcludeFilter);
685    parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
686
687
688    // Add arguments pertaining to moving subtrees.
689    moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
690         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
691    moveSubtreeFrom.addLongIdentifier("move-subtree-from", true);
692    moveSubtreeFrom.setArgumentGroupName(
693         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
694    parser.addArgument(moveSubtreeFrom);
695
696    moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
697         INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
698              moveSubtreeFrom.getIdentifierString()));
699    moveSubtreeTo.addLongIdentifier("move-subtree-to", true);
700    moveSubtreeTo.setArgumentGroupName(
701         INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
702    parser.addArgument(moveSubtreeTo);
703    parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
704    parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
705
706
707    // Add arguments pertaining to redacting attribute values.
708    redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
709         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
710         INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
711    redactAttribute.addLongIdentifier("redact-attribute", true);
712    redactAttribute.setArgumentGroupName(
713         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
714    parser.addArgument(redactAttribute);
715
716    hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
717         INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
718    hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count",
719         true);
720    hideRedactedValueCount.setArgumentGroupName(
721         INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
722    parser.addArgument(hideRedactedValueCount);
723    parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
724
725
726    // Add arguments pertaining to excluding attributes and entries.
727    excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
728         INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
729         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
730    excludeAttribute.addLongIdentifier("suppressAttribute", true);
731    excludeAttribute.addLongIdentifier("exclude-attribute", true);
732    excludeAttribute.addLongIdentifier("suppress-attribute", true);
733    excludeAttribute.setArgumentGroupName(
734         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
735    parser.addArgument(excludeAttribute);
736
737    excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
738         null,
739         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
740              sourceContainsChangeRecords.getIdentifierString()));
741    excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN", true);
742    excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn", true);
743    excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn", true);
744    excludeEntryBaseDN.setArgumentGroupName(
745         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
746    parser.addArgument(excludeEntryBaseDN);
747    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
748         excludeEntryBaseDN);
749
750    excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
751         null,
752         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
753              sourceContainsChangeRecords.getIdentifierString()));
754    excludeEntryScope.addLongIdentifier("suppressEntryScope", true);
755    excludeEntryScope.addLongIdentifier("exclude-entry-scope", true);
756    excludeEntryScope.addLongIdentifier("suppress-entry-scope", true);
757    excludeEntryScope.setArgumentGroupName(
758         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
759    parser.addArgument(excludeEntryScope);
760    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
761         excludeEntryScope);
762
763    excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
764         1, null,
765         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
766              sourceContainsChangeRecords.getIdentifierString()));
767    excludeEntryFilter.addLongIdentifier("suppressEntryFilter", true);
768    excludeEntryFilter.addLongIdentifier("exclude-entry-filter", true);
769    excludeEntryFilter.addLongIdentifier("suppress-entry-filter", true);
770    excludeEntryFilter.setArgumentGroupName(
771         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
772    parser.addArgument(excludeEntryFilter);
773    parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
774         excludeEntryFilter);
775
776    excludeNonMatchingEntries = new BooleanArgument(null,
777         "excludeNonMatchingEntries",
778         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
779    excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries",
780         true);
781    excludeNonMatchingEntries.setArgumentGroupName(
782         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
783    parser.addArgument(excludeNonMatchingEntries);
784    parser.addDependentArgumentSet(excludeNonMatchingEntries,
785         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
786
787
788    // Add arguments for excluding records based on their change types.
789    excludeChangeType = new StringArgument(null, "excludeChangeType",
790         false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_CHANGE_TYPES.get(),
791         INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_CHANGE_TYPE.get(),
792         StaticUtils.setOf("add", "delete", "modify", "moddn"));
793    excludeChangeType.addLongIdentifier("exclude-change-type", true);
794    excludeChangeType.addLongIdentifier("exclude-changetype", true);
795    excludeChangeType.setArgumentGroupName(
796         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
797    parser.addArgument(excludeChangeType);
798
799
800    // Add arguments for excluding records that don't have a change type.
801    excludeRecordsWithoutChangeType = new BooleanArgument(null,
802         "excludeRecordsWithoutChangeType", 1,
803         INFO_TRANSFORM_LDIF_EXCLUDE_WITHOUT_CHANGETYPE.get());
804    excludeRecordsWithoutChangeType.addLongIdentifier(
805         "exclude-records-without-change-type", true);
806    excludeRecordsWithoutChangeType.addLongIdentifier(
807         "exclude-records-without-changetype", true);
808    excludeRecordsWithoutChangeType.setArgumentGroupName(
809         INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
810    parser.addArgument(excludeRecordsWithoutChangeType);
811
812
813    // Add the remaining arguments.
814    schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
815         INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
816         true, true, false, false);
817    schemaPath.addLongIdentifier("schemaFile", true);
818    schemaPath.addLongIdentifier("schemaDirectory", true);
819    schemaPath.addLongIdentifier("schema-path", true);
820    schemaPath.addLongIdentifier("schema-file", true);
821    schemaPath.addLongIdentifier("schema-directory", true);
822    parser.addArgument(schemaPath);
823
824    numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
825         INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
826         1);
827    numThreads.addLongIdentifier("num-threads", true);
828    parser.addArgument(numThreads);
829
830    processDNs = new BooleanArgument('d', "processDNs",
831         INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
832    processDNs.addLongIdentifier("process-dns", true);
833    parser.addArgument(processDNs);
834
835
836    // Ensure that at least one kind of transformation was requested.
837    parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
838         replaceValuesAttribute, addAttributeName, renameAttributeFrom,
839         flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
840         excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter,
841         excludeChangeType, excludeRecordsWithoutChangeType);
842  }
843
844
845
846  /**
847   * {@inheritDoc}
848   */
849  @Override()
850  public void doExtendedArgumentValidation()
851         throws ArgumentException
852  {
853    // Ideally, exactly one of the targetLDIF and targetToStandardOutput
854    // arguments should always be provided.  But in order to preserve backward
855    // compatibility with a legacy scramble-ldif tool, we will allow both to be
856    // omitted if either --scrambleAttribute or --sequentialArgument is
857    // provided.  In that case, the path of the output file will be the path of
858    // the first input file with ".scrambled" appended to it.
859    if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
860    {
861      if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
862      {
863        throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
864             targetLDIF.getIdentifierString(),
865             targetToStandardOutput.getIdentifierString()));
866      }
867    }
868
869
870    // Make sure that the --renameAttributeFrom and --renameAttributeTo
871    // arguments were provided an equal number of times.
872    final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
873    final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
874    if (renameFromOccurrences != renameToOccurrences)
875    {
876      throw new ArgumentException(
877           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
878                renameAttributeFrom.getIdentifierString(),
879                renameAttributeTo.getIdentifierString()));
880    }
881
882
883    // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
884    // provided an equal number of times.
885    final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
886    final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
887    if (moveFromOccurrences != moveToOccurrences)
888    {
889      throw new ArgumentException(
890           ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
891                moveSubtreeFrom.getIdentifierString(),
892                moveSubtreeTo.getIdentifierString()));
893    }
894  }
895
896
897
898  /**
899   * {@inheritDoc}
900   */
901  @Override()
902  @NotNull()
903  public ResultCode doToolProcessing()
904  {
905    final Schema schema;
906    try
907    {
908      schema = getSchema();
909    }
910    catch (final LDAPException le)
911    {
912      wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
913      return le.getResultCode();
914    }
915
916
917    // If an encryption passphrase file is provided, then get the passphrase
918    // from it.
919    String encryptionPassphrase = null;
920    if (encryptionPassphraseFile.isPresent())
921    {
922      try
923      {
924        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
925             encryptionPassphraseFile.getValue());
926      }
927      catch (final LDAPException e)
928      {
929        wrapErr(0, MAX_OUTPUT_LINE_LENGTH, e.getMessage());
930        return e.getResultCode();
931      }
932    }
933
934
935    // Create the translators to use to apply the transformations.
936    final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
937         new ArrayList<>(10);
938    final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
939         new ArrayList<>(10);
940
941    final AtomicLong excludedEntryCount = new AtomicLong(0L);
942    createTranslators(entryTranslators, changeRecordTranslators,
943         schema, excludedEntryCount);
944
945    final AggregateLDIFReaderEntryTranslator entryTranslator =
946         new AggregateLDIFReaderEntryTranslator(entryTranslators);
947    final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
948         new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
949
950
951    // Determine the path to the target file to be written.
952    final File targetFile;
953    if (targetLDIF.isPresent())
954    {
955      targetFile = targetLDIF.getValue();
956    }
957    else if (targetToStandardOutput.isPresent())
958    {
959      targetFile = null;
960    }
961    else
962    {
963      targetFile =
964           new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
965    }
966
967
968    // Create the LDIF reader.
969    final LDIFReader ldifReader;
970    try
971    {
972      final InputStream inputStream;
973      if (sourceLDIF.isPresent())
974      {
975        final ObjectPair<InputStream,String> p =
976             ToolUtils.getInputStreamForLDIFFiles(sourceLDIF.getValues(),
977                  encryptionPassphrase, getOut(), getErr());
978        inputStream = p.getFirst();
979        if ((encryptionPassphrase == null) && (p.getSecond() != null))
980        {
981          encryptionPassphrase = p.getSecond();
982        }
983      }
984      else
985      {
986        inputStream = System.in;
987      }
988
989      ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
990           entryTranslator, changeRecordTranslator);
991      if (schema != null)
992      {
993        ldifReader.setSchema(schema);
994      }
995    }
996    catch (final Exception e)
997    {
998      Debug.debugException(e);
999      wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1000           ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
1001                StaticUtils.getExceptionMessage(e)));
1002      return ResultCode.LOCAL_ERROR;
1003    }
1004
1005
1006    ResultCode resultCode = ResultCode.SUCCESS;
1007    OutputStream outputStream = null;
1008processingBlock:
1009    try
1010    {
1011      // Create the output stream to use to write the transformed data.
1012      try
1013      {
1014        if (targetFile == null)
1015        {
1016          outputStream = getOut();
1017        }
1018        else
1019        {
1020          outputStream =
1021               new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
1022        }
1023
1024        if (encryptTarget.isPresent())
1025        {
1026          if (encryptionPassphrase == null)
1027          {
1028            encryptionPassphrase = ToolUtils.promptForEncryptionPassphrase(
1029                 false, true, getOut(), getErr());
1030          }
1031
1032          outputStream = new PassphraseEncryptedOutputStream(
1033               encryptionPassphrase, outputStream);
1034        }
1035
1036        if (compressTarget.isPresent())
1037        {
1038          outputStream = new GZIPOutputStream(outputStream);
1039        }
1040      }
1041      catch (final Exception e)
1042      {
1043        Debug.debugException(e);
1044        wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1045             ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
1046                  targetFile.getAbsolutePath(),
1047                  StaticUtils.getExceptionMessage(e)));
1048        resultCode = ResultCode.LOCAL_ERROR;
1049        break processingBlock;
1050      }
1051
1052
1053      // Read the source data one record at a time.  The transformations will
1054      // automatically be applied by the LDIF reader's translators, and even if
1055      // there are multiple reader threads, we're guaranteed to get the results
1056      // in the right order.
1057      long entriesWritten = 0L;
1058      while (true)
1059      {
1060        final LDIFRecord ldifRecord;
1061        try
1062        {
1063          ldifRecord = ldifReader.readLDIFRecord();
1064        }
1065        catch (final LDIFException le)
1066        {
1067          Debug.debugException(le);
1068          if (le.mayContinueReading())
1069          {
1070            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1071                 ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
1072                      StaticUtils.getExceptionMessage(le)));
1073            if (resultCode == ResultCode.SUCCESS)
1074            {
1075              resultCode = ResultCode.PARAM_ERROR;
1076            }
1077            continue;
1078          }
1079          else
1080          {
1081            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1082                 ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
1083                      StaticUtils.getExceptionMessage(le)));
1084            if (resultCode == ResultCode.SUCCESS)
1085            {
1086              resultCode = ResultCode.PARAM_ERROR;
1087            }
1088            break processingBlock;
1089          }
1090        }
1091        catch (final Exception e)
1092        {
1093          Debug.debugException(e);
1094          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1095               ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1096                    StaticUtils.getExceptionMessage(e)));
1097          resultCode = ResultCode.LOCAL_ERROR;
1098          break processingBlock;
1099        }
1100
1101
1102        // If the LDIF record is null, then we've run out of records so we're
1103        // done.
1104        if (ldifRecord == null)
1105        {
1106          break;
1107        }
1108
1109
1110        // Write the record to the output stream.
1111        try
1112        {
1113          if (ldifRecord instanceof PreEncodedLDIFEntry)
1114          {
1115            outputStream.write(
1116                 ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1117          }
1118          else
1119          {
1120            final ByteStringBuffer buffer = getBuffer();
1121            if (wrapColumn.isPresent())
1122            {
1123              ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1124            }
1125            else
1126            {
1127              ldifRecord.toLDIF(buffer, 0);
1128            }
1129            buffer.append(StaticUtils.EOL_BYTES);
1130            buffer.write(outputStream);
1131          }
1132        }
1133        catch (final Exception e)
1134        {
1135          Debug.debugException(e);
1136          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1137               ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1138                    StaticUtils.getExceptionMessage(e)));
1139          resultCode = ResultCode.LOCAL_ERROR;
1140          break processingBlock;
1141        }
1142
1143
1144        // If we've written a multiple of 1000 entries, print a progress
1145        // message.
1146        entriesWritten++;
1147        if ((! targetToStandardOutput.isPresent()) &&
1148            ((entriesWritten % 1000L) == 0))
1149        {
1150          final long numExcluded = excludedEntryCount.get();
1151          if (numExcluded > 0L)
1152          {
1153            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1154                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1155                      entriesWritten, numExcluded));
1156          }
1157          else
1158          {
1159            wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1160                 INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1161                      entriesWritten));
1162          }
1163        }
1164      }
1165
1166
1167      if (! targetToStandardOutput.isPresent())
1168      {
1169        final long numExcluded = excludedEntryCount.get();
1170        if (numExcluded > 0L)
1171        {
1172          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1173               INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1174                    numExcluded));
1175        }
1176        else
1177        {
1178          wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1179               INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1180        }
1181      }
1182    }
1183    finally
1184    {
1185      if (outputStream != null)
1186      {
1187        try
1188        {
1189          outputStream.close();
1190        }
1191        catch (final Exception e)
1192        {
1193          Debug.debugException(e);
1194          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1195               ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1196                    targetFile.getAbsolutePath(),
1197                    StaticUtils.getExceptionMessage(e)));
1198          if (resultCode == ResultCode.SUCCESS)
1199          {
1200            resultCode = ResultCode.LOCAL_ERROR;
1201          }
1202        }
1203      }
1204
1205      try
1206      {
1207        ldifReader.close();
1208      }
1209      catch (final Exception e)
1210      {
1211        Debug.debugException(e);
1212        // We can ignore this.
1213      }
1214    }
1215
1216
1217    return resultCode;
1218  }
1219
1220
1221
1222  /**
1223   * Retrieves the schema that should be used for processing.
1224   *
1225   * @return  The schema that was created.
1226   *
1227   * @throws  LDAPException  If a problem is encountered while retrieving the
1228   *                         schema.
1229   */
1230  @Nullable()
1231  private Schema getSchema()
1232          throws LDAPException
1233  {
1234    // If any schema paths were specified, then load the schema only from those
1235    // paths.
1236    if (schemaPath.isPresent())
1237    {
1238      final ArrayList<File> schemaFiles = new ArrayList<>(10);
1239      for (final File path : schemaPath.getValues())
1240      {
1241        if (path.isFile())
1242        {
1243          schemaFiles.add(path);
1244        }
1245        else
1246        {
1247          final TreeMap<String,File> fileMap = new TreeMap<>();
1248          for (final File schemaDirFile : path.listFiles())
1249          {
1250            final String name = schemaDirFile.getName();
1251            if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1252            {
1253              fileMap.put(name, schemaDirFile);
1254            }
1255          }
1256          schemaFiles.addAll(fileMap.values());
1257        }
1258      }
1259
1260      if (schemaFiles.isEmpty())
1261      {
1262        throw new LDAPException(ResultCode.PARAM_ERROR,
1263             ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1264                  schemaPath.getIdentifierString()));
1265      }
1266      else
1267      {
1268        try
1269        {
1270          return Schema.getSchema(schemaFiles);
1271        }
1272        catch (final Exception e)
1273        {
1274          Debug.debugException(e);
1275          throw new LDAPException(ResultCode.LOCAL_ERROR,
1276               ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1277                    StaticUtils.getExceptionMessage(e)));
1278        }
1279      }
1280    }
1281    else
1282    {
1283      // If the INSTANCE_ROOT environment variable is set and it refers to a
1284      // directory that has a config/schema subdirectory that has one or more
1285      // schema files in it, then read the schema from that directory.
1286      try
1287      {
1288        final String instanceRootStr =
1289             StaticUtils.getEnvironmentVariable("INSTANCE_ROOT");
1290        if (instanceRootStr != null)
1291        {
1292          final File instanceRoot = new File(instanceRootStr);
1293          final File configDir = new File(instanceRoot, "config");
1294          final File schemaDir = new File(configDir, "schema");
1295          if (schemaDir.exists())
1296          {
1297            final TreeMap<String,File> fileMap = new TreeMap<>();
1298            for (final File schemaDirFile : schemaDir.listFiles())
1299            {
1300              final String name = schemaDirFile.getName();
1301              if (schemaDirFile.isFile() &&
1302                  name.toLowerCase().endsWith(".ldif"))
1303              {
1304                fileMap.put(name, schemaDirFile);
1305              }
1306            }
1307
1308            if (! fileMap.isEmpty())
1309            {
1310              return Schema.getSchema(new ArrayList<>(fileMap.values()));
1311            }
1312          }
1313        }
1314      }
1315      catch (final Exception e)
1316      {
1317        Debug.debugException(e);
1318      }
1319    }
1320
1321
1322    // If we've gotten here, then just return null and the tool will try to use
1323    // the default standard schema.
1324    return null;
1325  }
1326
1327
1328
1329  /**
1330   * Creates the entry and change record translators that will be used to
1331   * perform the transformations.
1332   *
1333   * @param  entryTranslators         A list to which all created entry
1334   *                                  translators should be written.
1335   * @param  changeRecordTranslators  A list to which all created change record
1336   *                                  translators should be written.
1337   * @param  schema                   The schema to use when processing.
1338   * @param  excludedEntryCount       A counter used to keep track of the number
1339   *                                  of entries that have been excluded from
1340   *                                  the result set.
1341   */
1342  private void createTranslators(
1343       @NotNull final List<LDIFReaderEntryTranslator> entryTranslators,
1344       @NotNull final List<LDIFReaderChangeRecordTranslator>
1345            changeRecordTranslators,
1346       @Nullable final Schema schema,
1347       @NotNull final AtomicLong excludedEntryCount)
1348  {
1349    if (scrambleAttribute.isPresent())
1350    {
1351      final Long seed;
1352      if (randomSeed.isPresent())
1353      {
1354        seed = randomSeed.getValue().longValue();
1355      }
1356      else
1357      {
1358        seed = null;
1359      }
1360
1361      final ScrambleAttributeTransformation t =
1362           new ScrambleAttributeTransformation(schema, seed,
1363                processDNs.isPresent(), scrambleAttribute.getValues(),
1364                scrambleJSONField.getValues());
1365      entryTranslators.add(t);
1366      changeRecordTranslators.add(t);
1367    }
1368
1369    if (sequentialAttribute.isPresent())
1370    {
1371      final long initialValue;
1372      if (initialSequentialValue.isPresent())
1373      {
1374        initialValue = initialSequentialValue.getValue().longValue();
1375      }
1376      else
1377      {
1378        initialValue = 0L;
1379      }
1380
1381      final long incrementAmount;
1382      if (sequentialValueIncrement.isPresent())
1383      {
1384        incrementAmount = sequentialValueIncrement.getValue().longValue();
1385      }
1386      else
1387      {
1388        incrementAmount = 1L;
1389      }
1390
1391      for (final String attrName : sequentialAttribute.getValues())
1392      {
1393
1394
1395        final ReplaceWithCounterTransformation t =
1396             new ReplaceWithCounterTransformation(schema, attrName,
1397                  initialValue, incrementAmount,
1398                  textBeforeSequentialValue.getValue(),
1399                  textAfterSequentialValue.getValue(), processDNs.isPresent());
1400        entryTranslators.add(t);
1401      }
1402    }
1403
1404    if (replaceValuesAttribute.isPresent())
1405    {
1406      final ReplaceAttributeTransformation t =
1407           new ReplaceAttributeTransformation(schema,
1408                replaceValuesAttribute.getValue(),
1409                replacementValue.getValues());
1410      entryTranslators.add(t);
1411    }
1412
1413    if (addAttributeName.isPresent())
1414    {
1415      final AddAttributeTransformation t = new AddAttributeTransformation(
1416           schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1417           addAttributeFilter.getValue(),
1418           new Attribute(addAttributeName.getValue(), schema,
1419                addAttributeValue.getValues()),
1420           (! addToExistingValues.isPresent()));
1421      entryTranslators.add(t);
1422    }
1423
1424    if (renameAttributeFrom.isPresent())
1425    {
1426      final Iterator<String> renameFromIterator =
1427           renameAttributeFrom.getValues().iterator();
1428      final Iterator<String> renameToIterator =
1429           renameAttributeTo.getValues().iterator();
1430      while (renameFromIterator.hasNext())
1431      {
1432        final RenameAttributeTransformation t =
1433             new RenameAttributeTransformation(schema,
1434                  renameFromIterator.next(), renameToIterator.next(),
1435                  processDNs.isPresent());
1436        entryTranslators.add(t);
1437        changeRecordTranslators.add(t);
1438      }
1439    }
1440
1441    if (flattenBaseDN.isPresent())
1442    {
1443      final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1444           schema, flattenBaseDN.getValue(),
1445           flattenAddOmittedRDNAttributesToEntry.isPresent(),
1446           flattenAddOmittedRDNAttributesToRDN.isPresent(),
1447           flattenExcludeFilter.getValue());
1448      entryTranslators.add(t);
1449    }
1450
1451    if (moveSubtreeFrom.isPresent())
1452    {
1453      final Iterator<DN> moveFromIterator =
1454           moveSubtreeFrom.getValues().iterator();
1455      final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1456      while (moveFromIterator.hasNext())
1457      {
1458        final MoveSubtreeTransformation t =
1459             new MoveSubtreeTransformation(moveFromIterator.next(),
1460                  moveToIterator.next());
1461        entryTranslators.add(t);
1462        changeRecordTranslators.add(t);
1463      }
1464    }
1465
1466    if (redactAttribute.isPresent())
1467    {
1468      final RedactAttributeTransformation t = new RedactAttributeTransformation(
1469           schema, processDNs.isPresent(),
1470           (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1471      entryTranslators.add(t);
1472      changeRecordTranslators.add(t);
1473    }
1474
1475    if (excludeAttribute.isPresent())
1476    {
1477      final ExcludeAttributeTransformation t =
1478           new ExcludeAttributeTransformation(schema,
1479                excludeAttribute.getValues());
1480      entryTranslators.add(t);
1481      changeRecordTranslators.add(t);
1482    }
1483
1484    if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1485        excludeEntryFilter.isPresent())
1486    {
1487      final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1488           schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1489           excludeEntryFilter.getValue(),
1490           (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1491      entryTranslators.add(t);
1492    }
1493
1494    if (excludeChangeType.isPresent())
1495    {
1496      final Set<ChangeType> changeTypes = EnumSet.noneOf(ChangeType.class);
1497      for (final String changeTypeName : excludeChangeType.getValues())
1498      {
1499        changeTypes.add(ChangeType.forName(changeTypeName));
1500      }
1501
1502      changeRecordTranslators.add(
1503           new ExcludeChangeTypeTransformation(changeTypes));
1504    }
1505
1506    if (excludeRecordsWithoutChangeType.isPresent())
1507    {
1508      entryTranslators.add(new ExcludeAllEntriesTransformation());
1509    }
1510
1511    entryTranslators.add(this);
1512  }
1513
1514
1515
1516  /**
1517   * {@inheritDoc}
1518   */
1519  @Override()
1520  @NotNull()
1521  public LinkedHashMap<String[],String> getExampleUsages()
1522  {
1523    final LinkedHashMap<String[],String> examples =
1524         new LinkedHashMap<>(StaticUtils.computeMapCapacity(4));
1525
1526    examples.put(
1527         new String[]
1528         {
1529           "--sourceLDIF", "input.ldif",
1530           "--targetLDIF", "scrambled.ldif",
1531           "--scrambleAttribute", "givenName",
1532           "--scrambleAttribute", "sn",
1533           "--scrambleAttribute", "cn",
1534           "--numThreads", "10",
1535           "--schemaPath", "/ds/config/schema",
1536           "--processDNs"
1537         },
1538         INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1539
1540    examples.put(
1541         new String[]
1542         {
1543           "--sourceLDIF", "input.ldif",
1544           "--targetLDIF", "sequential.ldif",
1545           "--sequentialAttribute", "uid",
1546           "--initialSequentialValue", "1",
1547           "--sequentialValueIncrement", "1",
1548           "--textBeforeSequentialValue", "user.",
1549           "--numThreads", "10",
1550           "--schemaPath", "/ds/config/schema",
1551           "--processDNs"
1552         },
1553         INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1554
1555    examples.put(
1556         new String[]
1557         {
1558           "--sourceLDIF", "input.ldif",
1559           "--targetLDIF", "added-organization.ldif",
1560           "--addAttributeName", "o",
1561           "--addAttributeValue", "Example Corp.",
1562           "--addAttributeFilter", "(objectClass=person)",
1563           "--numThreads", "10",
1564           "--schemaPath", "/ds/config/schema"
1565         },
1566         INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1567
1568    examples.put(
1569         new String[]
1570         {
1571           "--sourceLDIF", "input.ldif",
1572           "--targetLDIF", "rebased.ldif",
1573           "--moveSubtreeFrom", "o=example.com",
1574           "--moveSubtreeTo", "dc=example,dc=com",
1575           "--numThreads", "10",
1576           "--schemaPath", "/ds/config/schema"
1577         },
1578         INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1579
1580    return examples;
1581  }
1582
1583
1584
1585  /**
1586   * {@inheritDoc}
1587   */
1588  @Override()
1589  @Nullable()
1590  public Entry translate(@NotNull final Entry original,
1591                         final long firstLineNumber)
1592         throws LDIFException
1593  {
1594    final ByteStringBuffer buffer = getBuffer();
1595    if (wrapColumn.isPresent())
1596    {
1597      original.toLDIF(buffer, wrapColumn.getValue());
1598    }
1599    else
1600    {
1601      original.toLDIF(buffer, 0);
1602    }
1603    buffer.append(StaticUtils.EOL_BYTES);
1604
1605    return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1606  }
1607
1608
1609
1610  /**
1611   * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1612   *
1613   * @return  A byte string buffer that can be used to perform LDIF encoding.
1614   */
1615  @NotNull()
1616  private ByteStringBuffer getBuffer()
1617  {
1618    ByteStringBuffer buffer = byteStringBuffers.get();
1619    if (buffer == null)
1620    {
1621      buffer = new ByteStringBuffer();
1622      byteStringBuffers.set(buffer);
1623    }
1624    else
1625    {
1626      buffer.clear();
1627    }
1628
1629    return buffer;
1630  }
1631}