001    /*
002     * Copyright 2016 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2016 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.transformations;
022    
023    
024    
025    import java.io.ByteArrayInputStream;
026    import java.io.File;
027    import java.io.FileInputStream;
028    import java.io.FileOutputStream;
029    import java.io.InputStream;
030    import java.io.OutputStream;
031    import java.util.ArrayList;
032    import java.util.Iterator;
033    import java.util.LinkedHashMap;
034    import java.util.List;
035    import java.util.TreeMap;
036    import java.util.concurrent.atomic.AtomicLong;
037    import java.util.zip.GZIPInputStream;
038    import java.util.zip.GZIPOutputStream;
039    
040    import com.unboundid.ldap.sdk.Attribute;
041    import com.unboundid.ldap.sdk.DN;
042    import com.unboundid.ldap.sdk.Entry;
043    import com.unboundid.ldap.sdk.LDAPException;
044    import com.unboundid.ldap.sdk.ResultCode;
045    import com.unboundid.ldap.sdk.Version;
046    import com.unboundid.ldap.sdk.schema.Schema;
047    import com.unboundid.ldif.AggregateLDIFReaderChangeRecordTranslator;
048    import com.unboundid.ldif.AggregateLDIFReaderEntryTranslator;
049    import com.unboundid.ldif.LDIFException;
050    import com.unboundid.ldif.LDIFReader;
051    import com.unboundid.ldif.LDIFReaderChangeRecordTranslator;
052    import com.unboundid.ldif.LDIFReaderEntryTranslator;
053    import com.unboundid.ldif.LDIFRecord;
054    import com.unboundid.util.AggregateInputStream;
055    import com.unboundid.util.ByteStringBuffer;
056    import com.unboundid.util.CommandLineTool;
057    import com.unboundid.util.Debug;
058    import com.unboundid.util.StaticUtils;
059    import com.unboundid.util.ThreadSafety;
060    import com.unboundid.util.ThreadSafetyLevel;
061    import com.unboundid.util.args.ArgumentException;
062    import com.unboundid.util.args.ArgumentParser;
063    import com.unboundid.util.args.BooleanArgument;
064    import com.unboundid.util.args.DNArgument;
065    import com.unboundid.util.args.FileArgument;
066    import com.unboundid.util.args.FilterArgument;
067    import com.unboundid.util.args.IntegerArgument;
068    import com.unboundid.util.args.ScopeArgument;
069    import com.unboundid.util.args.StringArgument;
070    
071    import static com.unboundid.ldap.sdk.transformations.TransformationMessages.*;
072    
073    
074    
075    /**
076     * This class provides a command-line tool that can be used to apply a number of
077     * transformations to an LDIF file.  The transformations that can be applied
078     * include:
079     * <UL>
080     *   <LI>
081     *     It can scramble the values of a specified set of attributes in a manner
082     *     that attempts to preserve the syntax and consistently scrambles the same
083     *     value to the same representation.
084     *   </LI>
085     *   <LI>
086     *     It can strip a specified set of attributes out of entries.
087     *   </LI>
088     *   <LI>
089     *     It can redact the values of a specified set of attributes, to indicate
090     *     that the values are there but providing no information about what their
091     *     values are.
092     *   </LI>
093     *   <LI>
094     *     It can replace the values of a specified attribute with a given set of
095     *     values.
096     *   </LI>
097     *   <LI>
098     *     It can add an attribute with a given set of values to any entry that does
099     *     not contain that attribute.
100     *   </LI>
101     *   <LI>
102     *     It can replace the values of a specified attribute with a value that
103     *     contains a sequentially-incrementing counter.
104     *   </LI>
105     *   <LI>
106     *     It can strip entries matching a given base DN, scope, and filter out of
107     *     the LDIF file.
108     *   </LI>
109     *   <LI>
110     *     It can perform DN mapping, so that entries that exist below one base DN
111     *     are moved below a different base DN.
112     *   </LI>
113     *   <LI>
114     *     It can perform attribute mapping, to replace uses of one attribute name
115     *     with another.
116     *   </LI>
117     * </UL>
118     */
119    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
120    public final class TransformLDIF
121           extends CommandLineTool
122           implements LDIFReaderEntryTranslator
123    {
124      /**
125       * The maximum length of any message to write to standard output or standard
126       * error.
127       */
128      private static final int MAX_OUTPUT_LINE_LENGTH =
129           StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
130    
131    
132    
133      // The arguments for use by this program.
134      private BooleanArgument addToExistingValues = null;
135      private BooleanArgument appendToTargetLDIF = null;
136      private BooleanArgument compressTarget = null;
137      private BooleanArgument excludeNonMatchingEntries = null;
138      private BooleanArgument flattenAddOmittedRDNAttributesToEntry = null;
139      private BooleanArgument flattenAddOmittedRDNAttributesToRDN = null;
140      private BooleanArgument hideRedactedValueCount = null;
141      private BooleanArgument processDNs = null;
142      private BooleanArgument sourceCompressed = null;
143      private BooleanArgument sourceContainsChangeRecords = null;
144      private BooleanArgument sourceFromStandardInput = null;
145      private BooleanArgument targetToStandardOutput = null;
146      private DNArgument addAttributeBaseDN = null;
147      private DNArgument excludeEntryBaseDN = null;
148      private DNArgument flattenBaseDN = null;
149      private DNArgument moveSubtreeFrom = null;
150      private DNArgument moveSubtreeTo = null;
151      private FileArgument schemaPath = null;
152      private FileArgument sourceLDIF = null;
153      private FileArgument targetLDIF = null;
154      private FilterArgument addAttributeFilter = null;
155      private FilterArgument excludeEntryFilter = null;
156      private FilterArgument flattenExcludeFilter = null;
157      private IntegerArgument initialSequentialValue = null;
158      private IntegerArgument numThreads = null;
159      private IntegerArgument randomSeed = null;
160      private IntegerArgument sequentialValueIncrement = null;
161      private IntegerArgument wrapColumn = null;
162      private ScopeArgument addAttributeScope = null;
163      private ScopeArgument excludeEntryScope = null;
164      private StringArgument addAttributeName = null;
165      private StringArgument addAttributeValue = null;
166      private StringArgument excludeAttribute = null;
167      private StringArgument redactAttribute = null;
168      private StringArgument renameAttributeFrom = null;
169      private StringArgument renameAttributeTo = null;
170      private StringArgument replaceValuesAttribute = null;
171      private StringArgument replacementValue = null;
172      private StringArgument scrambleAttribute = null;
173      private StringArgument scrambleJSONField = null;
174      private StringArgument sequentialAttribute = null;
175      private StringArgument textAfterSequentialValue = null;
176      private StringArgument textBeforeSequentialValue = null;
177    
178      // A set of thread-local byte stream buffers that will be used to construct
179      // the LDIF representations of records.
180      private final ThreadLocal<ByteStringBuffer> byteStringBuffers =
181           new ThreadLocal<ByteStringBuffer>();
182    
183    
184    
185      /**
186       * Invokes this tool with the provided set of arguments.
187       *
188       * @param  args  The command-line arguments provided to this program.
189       */
190      public static void main(final String... args)
191      {
192        final ResultCode resultCode = main(System.out, System.err, args);
193        if (resultCode != ResultCode.SUCCESS)
194        {
195          System.exit(resultCode.intValue());
196        }
197      }
198    
199    
200    
201      /**
202       * Invokes this tool with the provided set of arguments.
203       *
204       * @param  out   The output stream to use for standard output.  It may be
205       *               {@code null} if standard output should be suppressed.
206       * @param  err   The output stream to use for standard error.  It may be
207       *               {@code null} if standard error should be suppressed.
208       * @param  args  The command-line arguments provided to this program.
209       *
210       * @return  A result code indicating whether processing completed
211       *          successfully.
212       */
213      public static ResultCode main(final OutputStream out, final OutputStream err,
214                                    final String... args)
215      {
216        final TransformLDIF tool = new TransformLDIF(out, err);
217        return tool.runTool(args);
218      }
219    
220    
221    
222      /**
223       * Creates a new instance of this tool with the provided information.
224       *
225       * @param  out  The output stream to use for standard output.  It may be
226       *              {@code null} if standard output should be suppressed.
227       * @param  err  The output stream to use for standard error.  It may be
228       *              {@code null} if standard error should be suppressed.
229       */
230      public TransformLDIF(final OutputStream out, final OutputStream err)
231      {
232        super(out, err);
233      }
234    
235    
236    
237      /**
238       * {@inheritDoc}
239       */
240      @Override()
241      public String getToolName()
242      {
243        return "transform-ldif";
244      }
245    
246    
247    
248      /**
249       * {@inheritDoc}
250       */
251      @Override()
252      public String getToolDescription()
253      {
254        return INFO_TRANSFORM_LDIF_TOOL_DESCRIPTION.get();
255      }
256    
257    
258    
259      /**
260       * {@inheritDoc}
261       */
262      @Override()
263      public String getToolVersion()
264      {
265        return Version.NUMERIC_VERSION_STRING;
266      }
267    
268    
269    
270      /**
271       * {@inheritDoc}
272       */
273      @Override()
274      public boolean supportsInteractiveMode()
275      {
276        return true;
277      }
278    
279    
280    
281      /**
282       * {@inheritDoc}
283       */
284      @Override()
285      public boolean defaultsToInteractiveMode()
286      {
287        return true;
288      }
289    
290    
291    
292      /**
293       * {@inheritDoc}
294       */
295      @Override()
296      public boolean supportsPropertiesFile()
297      {
298        return true;
299      }
300    
301    
302    
303      /**
304       * {@inheritDoc}
305       */
306      @Override()
307      public void addToolArguments(final ArgumentParser parser)
308             throws ArgumentException
309      {
310        // Add arguments pertaining to the source and target LDIF files.
311        sourceLDIF = new FileArgument('l', "sourceLDIF", false, 0, null,
312             INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_LDIF.get(), true, true, true,
313             false);
314        sourceLDIF.addLongIdentifier("inputLDIF");
315        sourceLDIF.addLongIdentifier("source-ldif");
316        sourceLDIF.addLongIdentifier("input-ldif");
317        sourceLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
318        parser.addArgument(sourceLDIF);
319    
320        sourceFromStandardInput = new BooleanArgument(null,
321             "sourceFromStandardInput", 1,
322             INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_STD_IN.get());
323        sourceFromStandardInput.addLongIdentifier("source-from-standard-input");
324        sourceFromStandardInput.setArgumentGroupName(
325             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
326        parser.addArgument(sourceFromStandardInput);
327        parser.addRequiredArgumentSet(sourceLDIF, sourceFromStandardInput);
328        parser.addExclusiveArgumentSet(sourceLDIF, sourceFromStandardInput);
329    
330        targetLDIF = new FileArgument('o', "targetLDIF", false, 1, null,
331             INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_LDIF.get(), false, true, true,
332             false);
333        targetLDIF.addLongIdentifier("outputLDIF");
334        targetLDIF.addLongIdentifier("target-ldif");
335        targetLDIF.addLongIdentifier("output-ldif");
336        targetLDIF.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
337        parser.addArgument(targetLDIF);
338    
339        targetToStandardOutput = new BooleanArgument(null, "targetToStandardOutput",
340             1, INFO_TRANSFORM_LDIF_ARG_DESC_TARGET_STD_OUT.get());
341        targetToStandardOutput.addLongIdentifier("target-to-standard-output");
342        targetToStandardOutput.setArgumentGroupName(
343             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
344        parser.addArgument(targetToStandardOutput);
345        parser.addExclusiveArgumentSet(targetLDIF, targetToStandardOutput);
346    
347        sourceContainsChangeRecords = new BooleanArgument(null,
348             "sourceContainsChangeRecords",
349             INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_CONTAINS_CHANGE_RECORDS.get());
350        sourceContainsChangeRecords.addLongIdentifier(
351             "source-contains-change-records");
352        sourceContainsChangeRecords.setArgumentGroupName(
353             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
354        parser.addArgument(sourceContainsChangeRecords);
355    
356        appendToTargetLDIF = new BooleanArgument(null, "appendToTargetLDIF",
357             INFO_TRANSFORM_LDIF_ARG_DESC_APPEND_TO_TARGET.get());
358        appendToTargetLDIF.addLongIdentifier("append-to-target-ldif");
359        appendToTargetLDIF.setArgumentGroupName(
360             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
361        parser.addArgument(appendToTargetLDIF);
362        parser.addExclusiveArgumentSet(targetToStandardOutput, appendToTargetLDIF);
363    
364        wrapColumn = new IntegerArgument(null, "wrapColumn", false, 1, null,
365             INFO_TRANSFORM_LDIF_ARG_DESC_WRAP_COLUMN.get(), 5, Integer.MAX_VALUE);
366        wrapColumn.addLongIdentifier("wrap-column");
367        wrapColumn.setArgumentGroupName(INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
368        parser.addArgument(wrapColumn);
369    
370        sourceCompressed = new BooleanArgument('C', "sourceCompressed",
371             INFO_TRANSFORM_LDIF_ARG_DESC_SOURCE_COMPRESSED.get());
372        sourceCompressed.addLongIdentifier("inputCompressed");
373        sourceCompressed.addLongIdentifier("source-compressed");
374        sourceCompressed.addLongIdentifier("input-compressed");
375        sourceCompressed.setArgumentGroupName(
376             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
377        parser.addArgument(sourceCompressed);
378    
379        compressTarget = new BooleanArgument('c', "compressTarget",
380             INFO_TRANSFORM_LDIF_ARG_DESC_COMPRESS_TARGET.get());
381        compressTarget.addLongIdentifier("compressOutput");
382        compressTarget.addLongIdentifier("compress");
383        compressTarget.addLongIdentifier("compress-target");
384        compressTarget.addLongIdentifier("compress-output");
385        compressTarget.setArgumentGroupName(
386             INFO_TRANSFORM_LDIF_ARG_GROUP_LDIF.get());
387        parser.addArgument(compressTarget);
388    
389    
390        // Add arguments pertaining to attribute scrambling.
391        scrambleAttribute = new StringArgument('a', "scrambleAttribute", false, 0,
392             INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
393             INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_ATTR.get());
394        scrambleAttribute.addLongIdentifier("attributeName");
395        scrambleAttribute.addLongIdentifier("scramble-attribute");
396        scrambleAttribute.addLongIdentifier("attribute-name");
397        scrambleAttribute.setArgumentGroupName(
398             INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
399        parser.addArgument(scrambleAttribute);
400    
401        scrambleJSONField = new StringArgument(null, "scrambleJSONField", false, 0,
402             INFO_TRANSFORM_LDIF_PLACEHOLDER_FIELD_NAME.get(),
403             INFO_TRANSFORM_LDIF_ARG_DESC_SCRAMBLE_JSON_FIELD.get(
404                  scrambleAttribute.getIdentifierString()));
405        scrambleJSONField.addLongIdentifier("scramble-json-field");
406        scrambleJSONField.setArgumentGroupName(
407             INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
408        parser.addArgument(scrambleJSONField);
409        parser.addDependentArgumentSet(scrambleJSONField, scrambleAttribute);
410    
411        randomSeed = new IntegerArgument('s', "randomSeed", false, 1, null,
412             INFO_TRANSFORM_LDIF_ARG_DESC_RANDOM_SEED.get());
413        randomSeed.addLongIdentifier("random-seed");
414        randomSeed.setArgumentGroupName(
415             INFO_TRANSFORM_LDIF_ARG_GROUP_SCRAMBLE.get());
416        parser.addArgument(randomSeed);
417    
418    
419        // Add arguments pertaining to replacing attribute values with a generated
420        // value using a sequential counter.
421        sequentialAttribute = new StringArgument('S', "sequentialAttribute",
422             false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
423             INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_ATTR.get(
424                  sourceContainsChangeRecords.getIdentifierString()));
425        sequentialAttribute.addLongIdentifier("sequentialAttributeName");
426        sequentialAttribute.addLongIdentifier("sequential-attribute");
427        sequentialAttribute.addLongIdentifier("sequential-attribute-name");
428        sequentialAttribute.setArgumentGroupName(
429             INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
430        parser.addArgument(sequentialAttribute);
431        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
432             sequentialAttribute);
433    
434        initialSequentialValue = new IntegerArgument('i', "initialSequentialValue",
435             false, 1, null,
436             INFO_TRANSFORM_LDIF_ARG_DESC_INITIAL_SEQUENTIAL_VALUE.get(
437                  sequentialAttribute.getIdentifierString()));
438        initialSequentialValue.addLongIdentifier("initial-sequential-value");
439        initialSequentialValue.setArgumentGroupName(
440             INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
441        parser.addArgument(initialSequentialValue);
442        parser.addDependentArgumentSet(initialSequentialValue, sequentialAttribute);
443    
444        sequentialValueIncrement = new IntegerArgument(null,
445             "sequentialValueIncrement", false, 1, null,
446             INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_INCREMENT.get(
447                  sequentialAttribute.getIdentifierString()));
448        sequentialValueIncrement.addLongIdentifier("sequential-value-increment");
449        sequentialValueIncrement.setArgumentGroupName(
450             INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
451        parser.addArgument(sequentialValueIncrement);
452        parser.addDependentArgumentSet(sequentialValueIncrement,
453             sequentialAttribute);
454    
455        textBeforeSequentialValue = new StringArgument(null,
456             "textBeforeSequentialValue", false, 1, null,
457             INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_BEFORE.get(
458                  sequentialAttribute.getIdentifierString()));
459        textBeforeSequentialValue.addLongIdentifier("text-before-sequential-value");
460        textBeforeSequentialValue.setArgumentGroupName(
461             INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
462        parser.addArgument(textBeforeSequentialValue);
463        parser.addDependentArgumentSet(textBeforeSequentialValue,
464             sequentialAttribute);
465    
466        textAfterSequentialValue = new StringArgument(null,
467             "textAfterSequentialValue", false, 1, null,
468             INFO_TRANSFORM_LDIF_ARG_DESC_SEQUENTIAL_TEXT_AFTER.get(
469                  sequentialAttribute.getIdentifierString()));
470        textAfterSequentialValue.addLongIdentifier("text-after-sequential-value");
471        textAfterSequentialValue.setArgumentGroupName(
472             INFO_TRANSFORM_LDIF_ARG_GROUP_SEQUENTIAL.get());
473        parser.addArgument(textAfterSequentialValue);
474        parser.addDependentArgumentSet(textAfterSequentialValue,
475             sequentialAttribute);
476    
477    
478        // Add arguments pertaining to attribute value replacement.
479        replaceValuesAttribute = new StringArgument(null, "replaceValuesAttribute",
480             false, 1, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
481             INFO_TRANSFORM_LDIF_ARG_DESC_REPLACE_VALUES_ATTR.get(
482                  sourceContainsChangeRecords.getIdentifierString()));
483        replaceValuesAttribute.addLongIdentifier("replace-values-attribute");
484        replaceValuesAttribute.setArgumentGroupName(
485             INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
486        parser.addArgument(replaceValuesAttribute);
487        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
488             replaceValuesAttribute);
489    
490        replacementValue = new StringArgument(null, "replacementValue", false, 0,
491             null,
492             INFO_TRANSFORM_LDIF_ARG_DESC_REPLACEMENT_VALUE.get(
493                  replaceValuesAttribute.getIdentifierString()));
494        replacementValue.addLongIdentifier("replacement-value");
495        replacementValue.setArgumentGroupName(
496             INFO_TRANSFORM_LDIF_ARG_GROUP_REPLACE_VALUES.get());
497        parser.addArgument(replacementValue);
498        parser.addDependentArgumentSet(replaceValuesAttribute, replacementValue);
499        parser.addDependentArgumentSet(replacementValue, replaceValuesAttribute);
500    
501    
502        // Add arguments pertaining to adding missing attributes.
503        addAttributeName = new StringArgument(null, "addAttributeName", false, 1,
504             INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
505             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_ATTR.get(
506                  "--addAttributeValue",
507                  sourceContainsChangeRecords.getIdentifierString()));
508        addAttributeName.addLongIdentifier("add-attribute-name");
509        addAttributeName.setArgumentGroupName(
510             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
511        parser.addArgument(addAttributeName);
512        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
513             addAttributeName);
514    
515        addAttributeValue = new StringArgument(null, "addAttributeValue", false, 0,
516             null,
517             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_VALUE.get(
518                  addAttributeName.getIdentifierString()));
519        addAttributeValue.addLongIdentifier("add-attribute-value");
520        addAttributeValue.setArgumentGroupName(
521             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
522        parser.addArgument(addAttributeValue);
523        parser.addDependentArgumentSet(addAttributeName, addAttributeValue);
524        parser.addDependentArgumentSet(addAttributeValue, addAttributeName);
525    
526        addToExistingValues = new BooleanArgument(null, "addToExistingValues",
527             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_MERGE_VALUES.get(
528                  addAttributeName.getIdentifierString(),
529                  addAttributeValue.getIdentifierString()));
530        addToExistingValues.addLongIdentifier("add-to-existing-values");
531        addToExistingValues.setArgumentGroupName(
532             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
533        parser.addArgument(addToExistingValues);
534        parser.addDependentArgumentSet(addToExistingValues, addAttributeName);
535    
536        addAttributeBaseDN = new DNArgument(null, "addAttributeBaseDN", false, 1,
537             null,
538             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_BASE_DN.get(
539                  addAttributeName.getIdentifierString()));
540        addAttributeBaseDN.addLongIdentifier("add-attribute-base-dn");
541        addAttributeBaseDN.setArgumentGroupName(
542             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
543        parser.addArgument(addAttributeBaseDN);
544        parser.addDependentArgumentSet(addAttributeBaseDN, addAttributeName);
545    
546        addAttributeScope = new ScopeArgument(null, "addAttributeScope", false,
547             null,
548             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_SCOPE.get(
549                  addAttributeBaseDN.getIdentifierString(),
550                  addAttributeName.getIdentifierString()));
551        addAttributeScope.addLongIdentifier("add-attribute-scope");
552        addAttributeScope.setArgumentGroupName(
553             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
554        parser.addArgument(addAttributeScope);
555        parser.addDependentArgumentSet(addAttributeScope, addAttributeName);
556    
557        addAttributeFilter = new FilterArgument(null, "addAttributeFilter", false,
558             1, null,
559             INFO_TRANSFORM_LDIF_ARG_DESC_ADD_FILTER.get(
560                  addAttributeName.getIdentifierString()));
561        addAttributeFilter.addLongIdentifier("add-attribute-filter");
562        addAttributeFilter.setArgumentGroupName(
563             INFO_TRANSFORM_LDIF_ARG_GROUP_ADD_ATTR.get());
564        parser.addArgument(addAttributeFilter);
565        parser.addDependentArgumentSet(addAttributeFilter, addAttributeName);
566    
567    
568        // Add arguments pertaining to renaming attributes.
569        renameAttributeFrom = new StringArgument(null, "renameAttributeFrom",
570             false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
571             INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_FROM.get());
572        renameAttributeFrom.addLongIdentifier("rename-attribute-from");
573        renameAttributeFrom.setArgumentGroupName(
574             INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
575        parser.addArgument(renameAttributeFrom);
576    
577        renameAttributeTo = new StringArgument(null, "renameAttributeTo",
578             false, 0, INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
579             INFO_TRANSFORM_LDIF_ARG_DESC_RENAME_TO.get(
580                  renameAttributeFrom.getIdentifierString()));
581        renameAttributeTo.addLongIdentifier("rename-attribute-to");
582        renameAttributeTo.setArgumentGroupName(
583             INFO_TRANSFORM_LDIF_ARG_GROUP_RENAME.get());
584        parser.addArgument(renameAttributeTo);
585        parser.addDependentArgumentSet(renameAttributeFrom, renameAttributeTo);
586        parser.addDependentArgumentSet(renameAttributeTo, renameAttributeFrom);
587    
588    
589        // Add arguments pertaining to flattening subtrees.
590        flattenBaseDN = new DNArgument(null, "flattenBaseDN", false, 1, null,
591             INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_BASE_DN.get());
592        flattenBaseDN.addLongIdentifier("flatten-base-dn");
593        flattenBaseDN.setArgumentGroupName(
594             INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
595        parser.addArgument(flattenBaseDN);
596        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
597             flattenBaseDN);
598    
599        flattenAddOmittedRDNAttributesToEntry = new BooleanArgument(null,
600             "flattenAddOmittedRDNAttributesToEntry", 1,
601             INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_ENTRY.get());
602        flattenAddOmittedRDNAttributesToEntry.addLongIdentifier(
603             "flatten-add-omitted-rdn-attributes-to-entry");
604        flattenAddOmittedRDNAttributesToEntry.setArgumentGroupName(
605             INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
606        parser.addArgument(flattenAddOmittedRDNAttributesToEntry);
607        parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToEntry,
608             flattenBaseDN);
609    
610        flattenAddOmittedRDNAttributesToRDN = new BooleanArgument(null,
611             "flattenAddOmittedRDNAttributesToRDN", 1,
612             INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_ADD_OMITTED_TO_RDN.get());
613        flattenAddOmittedRDNAttributesToRDN.addLongIdentifier(
614             "flatten-add-omitted-rdn-attributes-to-rdn");
615        flattenAddOmittedRDNAttributesToRDN.setArgumentGroupName(
616             INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
617        parser.addArgument(flattenAddOmittedRDNAttributesToRDN);
618        parser.addDependentArgumentSet(flattenAddOmittedRDNAttributesToRDN,
619             flattenBaseDN);
620    
621        flattenExcludeFilter = new FilterArgument(null, "flattenExcludeFilter",
622             false, 1, null,
623             INFO_TRANSFORM_LDIF_ARG_DESC_FLATTEN_EXCLUDE_FILTER.get());
624        flattenExcludeFilter.addLongIdentifier("flatten-exclude-filter");
625        flattenExcludeFilter.setArgumentGroupName(
626             INFO_TRANSFORM_LDIF_ARG_GROUP_FLATTEN.get());
627        parser.addArgument(flattenExcludeFilter);
628        parser.addDependentArgumentSet(flattenExcludeFilter, flattenBaseDN);
629    
630    
631        // Add arguments pertaining to moving subtrees.
632        moveSubtreeFrom = new DNArgument(null, "moveSubtreeFrom", false, 0, null,
633             INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_FROM.get());
634        moveSubtreeFrom.addLongIdentifier("move-subtree-from");
635        moveSubtreeFrom.setArgumentGroupName(
636             INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
637        parser.addArgument(moveSubtreeFrom);
638    
639        moveSubtreeTo = new DNArgument(null, "moveSubtreeTo", false, 0, null,
640             INFO_TRANSFORM_LDIF_ARG_DESC_MOVE_SUBTREE_TO.get(
641                  moveSubtreeFrom.getIdentifierString()));
642        moveSubtreeTo.addLongIdentifier("move-subtree-to");
643        moveSubtreeTo.setArgumentGroupName(
644             INFO_TRANSFORM_LDIF_ARG_GROUP_MOVE.get());
645        parser.addArgument(moveSubtreeTo);
646        parser.addDependentArgumentSet(moveSubtreeFrom, moveSubtreeTo);
647        parser.addDependentArgumentSet(moveSubtreeTo, moveSubtreeFrom);
648    
649    
650        // Add arguments pertaining to redacting attribute values.
651        redactAttribute = new StringArgument(null, "redactAttribute", false, 0,
652             INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
653             INFO_TRANSFORM_LDIF_ARG_DESC_REDACT_ATTR.get());
654        redactAttribute.addLongIdentifier("redact-attribute");
655        redactAttribute.setArgumentGroupName(
656             INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
657        parser.addArgument(redactAttribute);
658    
659        hideRedactedValueCount = new BooleanArgument(null, "hideRedactedValueCount",
660             INFO_TRANSFORM_LDIF_ARG_DESC_HIDE_REDACTED_COUNT.get());
661        hideRedactedValueCount.addLongIdentifier("hide-redacted-value-count");
662        hideRedactedValueCount.setArgumentGroupName(
663             INFO_TRANSFORM_LDIF_ARG_GROUP_REDACT.get());
664        parser.addArgument(hideRedactedValueCount);
665        parser.addDependentArgumentSet(hideRedactedValueCount, redactAttribute);
666    
667    
668        // Add arguments pertaining to excluding attributes and entries.
669        excludeAttribute = new StringArgument(null, "excludeAttribute", false, 0,
670             INFO_TRANSFORM_LDIF_PLACEHOLDER_ATTR_NAME.get(),
671             INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ATTR.get());
672        excludeAttribute.addLongIdentifier("suppressAttribute");
673        excludeAttribute.addLongIdentifier("exclude-attribute");
674        excludeAttribute.addLongIdentifier("suppress-attribute");
675        excludeAttribute.setArgumentGroupName(
676             INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
677        parser.addArgument(excludeAttribute);
678    
679        excludeEntryBaseDN = new DNArgument(null, "excludeEntryBaseDN", false, 1,
680             null,
681             INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_BASE_DN.get(
682                  sourceContainsChangeRecords.getIdentifierString()));
683        excludeEntryBaseDN.addLongIdentifier("suppressEntryBaseDN");
684        excludeEntryBaseDN.addLongIdentifier("exclude-entry-base-dn");
685        excludeEntryBaseDN.addLongIdentifier("suppress-entry-base-dn");
686        excludeEntryBaseDN.setArgumentGroupName(
687             INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
688        parser.addArgument(excludeEntryBaseDN);
689        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
690             excludeEntryBaseDN);
691    
692        excludeEntryScope = new ScopeArgument(null, "excludeEntryScope", false,
693             null,
694             INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_SCOPE.get(
695                  sourceContainsChangeRecords.getIdentifierString()));
696        excludeEntryScope.addLongIdentifier("suppressEntryScope");
697        excludeEntryScope.addLongIdentifier("exclude-entry-scope");
698        excludeEntryScope.addLongIdentifier("suppress-entry-scope");
699        excludeEntryScope.setArgumentGroupName(
700             INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
701        parser.addArgument(excludeEntryScope);
702        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
703             excludeEntryScope);
704    
705        excludeEntryFilter = new FilterArgument(null, "excludeEntryFilter", false,
706             1, null,
707             INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_ENTRY_FILTER.get(
708                  sourceContainsChangeRecords.getIdentifierString()));
709        excludeEntryFilter.addLongIdentifier("suppressEntryFilter");
710        excludeEntryFilter.addLongIdentifier("exclude-entry-filter");
711        excludeEntryFilter.addLongIdentifier("suppress-entry-filter");
712        excludeEntryFilter.setArgumentGroupName(
713             INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
714        parser.addArgument(excludeEntryFilter);
715        parser.addExclusiveArgumentSet(sourceContainsChangeRecords,
716             excludeEntryFilter);
717    
718        excludeNonMatchingEntries = new BooleanArgument(null,
719             "excludeNonMatchingEntries",
720             INFO_TRANSFORM_LDIF_ARG_DESC_EXCLUDE_NON_MATCHING.get());
721        excludeNonMatchingEntries.addLongIdentifier("exclude-non-matching-entries");
722        excludeNonMatchingEntries.setArgumentGroupName(
723             INFO_TRANSFORM_LDIF_ARG_GROUP_EXCLUDE.get());
724        parser.addArgument(excludeNonMatchingEntries);
725        parser.addDependentArgumentSet(excludeNonMatchingEntries,
726             excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
727    
728    
729        // Add the remaining arguments.
730        schemaPath = new FileArgument(null, "schemaPath", false, 0, null,
731             INFO_TRANSFORM_LDIF_ARG_DESC_SCHEMA_PATH.get(),
732             true, true, false, false);
733        schemaPath.addLongIdentifier("schemaFile");
734        schemaPath.addLongIdentifier("schemaDirectory");
735        schemaPath.addLongIdentifier("schema-path");
736        schemaPath.addLongIdentifier("schema-file");
737        schemaPath.addLongIdentifier("schema-directory");
738        parser.addArgument(schemaPath);
739    
740        numThreads = new IntegerArgument('t', "numThreads", false, 1, null,
741             INFO_TRANSFORM_LDIF_ARG_DESC_NUM_THREADS.get(), 1, Integer.MAX_VALUE,
742             1);
743        numThreads.addLongIdentifier("num-threads");
744        parser.addArgument(numThreads);
745    
746        processDNs = new BooleanArgument('d', "processDNs",
747             INFO_TRANSFORM_LDIF_ARG_DESC_PROCESS_DNS.get());
748        processDNs.addLongIdentifier("process-dns");
749        parser.addArgument(processDNs);
750    
751    
752        // Ensure that at least one kind of transformation was requested.
753        parser.addRequiredArgumentSet(scrambleAttribute, sequentialAttribute,
754             replaceValuesAttribute, addAttributeName, renameAttributeFrom,
755             flattenBaseDN, moveSubtreeFrom, redactAttribute, excludeAttribute,
756             excludeEntryBaseDN, excludeEntryScope, excludeEntryFilter);
757      }
758    
759    
760    
761      /**
762       * {@inheritDoc}
763       */
764      @Override()
765      public void doExtendedArgumentValidation()
766             throws ArgumentException
767      {
768        // Ideally, exactly one of the targetLDIF and targetToStandardOutput
769        // arguments should always be provided.  But in order to preserve backward
770        // compatibility with a legacy scramble-ldif tool, we will allow both to be
771        // omitted if either --scrambleAttribute or --sequentialArgument is
772        // provided.  In that case, the path of the output file will be the path of
773        // the first input file with ".scrambled" appended to it.
774        if (! (targetLDIF.isPresent() || targetToStandardOutput.isPresent()))
775        {
776          if (! (scrambleAttribute.isPresent() || sequentialAttribute.isPresent()))
777          {
778            throw new ArgumentException(ERR_TRANSFORM_LDIF_MISSING_TARGET_ARG.get(
779                 targetLDIF.getIdentifierString(),
780                 targetToStandardOutput.getIdentifierString()));
781          }
782        }
783    
784    
785        // Make sure that the --renameAttributeFrom and --renameAttributeTo
786        // arguments were provided an equal number of times.
787        final int renameFromOccurrences = renameAttributeFrom.getNumOccurrences();
788        final int renameToOccurrences = renameAttributeTo.getNumOccurrences();
789        if (renameFromOccurrences != renameToOccurrences)
790        {
791          throw new ArgumentException(
792               ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
793                    renameAttributeFrom.getIdentifierString(),
794                    renameAttributeTo.getIdentifierString()));
795        }
796    
797    
798        // Make sure that the --moveSubtreeFrom and --moveSubtreeTo arguments were
799        // provided an equal number of times.
800        final int moveFromOccurrences = moveSubtreeFrom.getNumOccurrences();
801        final int moveToOccurrences = moveSubtreeTo.getNumOccurrences();
802        if (moveFromOccurrences != moveToOccurrences)
803        {
804          throw new ArgumentException(
805               ERR_TRANSFORM_LDIF_ARG_COUNT_MISMATCH.get(
806                    moveSubtreeFrom.getIdentifierString(),
807                    moveSubtreeTo.getIdentifierString()));
808        }
809      }
810    
811    
812    
813      /**
814       * {@inheritDoc}
815       */
816      @Override()
817      public ResultCode doToolProcessing()
818      {
819        final Schema schema;
820        try
821        {
822          schema = getSchema();
823        }
824        catch (final LDAPException le)
825        {
826          wrapErr(0, MAX_OUTPUT_LINE_LENGTH, le.getMessage());
827          return le.getResultCode();
828        }
829    
830    
831        // Create the translators to use to apply the transformations.
832        final ArrayList<LDIFReaderEntryTranslator> entryTranslators =
833             new ArrayList<LDIFReaderEntryTranslator>(10);
834        final ArrayList<LDIFReaderChangeRecordTranslator> changeRecordTranslators =
835             new ArrayList<LDIFReaderChangeRecordTranslator>(10);
836    
837        final AtomicLong excludedEntryCount = new AtomicLong(0L);
838        createTranslators(entryTranslators, changeRecordTranslators,
839             schema, excludedEntryCount);
840    
841        final AggregateLDIFReaderEntryTranslator entryTranslator =
842             new AggregateLDIFReaderEntryTranslator(entryTranslators);
843        final AggregateLDIFReaderChangeRecordTranslator changeRecordTranslator =
844             new AggregateLDIFReaderChangeRecordTranslator(changeRecordTranslators);
845    
846    
847        // Determine the path to the target file to be written.
848        final File targetFile;
849        if (targetLDIF.isPresent())
850        {
851          targetFile = targetLDIF.getValue();
852        }
853        else if (targetToStandardOutput.isPresent())
854        {
855          targetFile = null;
856        }
857        else
858        {
859          targetFile =
860               new File(sourceLDIF.getValue().getAbsolutePath() + ".scrambled");
861        }
862    
863    
864        // Create the LDIF reader.
865        final LDIFReader ldifReader;
866        try
867        {
868          InputStream inputStream;
869          if (sourceLDIF.isPresent())
870          {
871            final List<File> sourceFiles = sourceLDIF.getValues();
872            final ArrayList<InputStream> fileInputStreams =
873                 new ArrayList<InputStream>(2*sourceFiles.size());
874            for (final File f : sourceFiles)
875            {
876              if (! fileInputStreams.isEmpty())
877              {
878                // Go ahead and ensure that there are at least new end-of-line
879                // markers between each file.  Otherwise, it's possible for entries
880                // to run together.
881                final byte[] doubleEOL = new byte[StaticUtils.EOL_BYTES.length * 2];
882                System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL, 0,
883                     StaticUtils.EOL_BYTES.length);
884                System.arraycopy(StaticUtils.EOL_BYTES, 0, doubleEOL,
885                     StaticUtils.EOL_BYTES.length, StaticUtils.EOL_BYTES.length);
886                fileInputStreams.add(new ByteArrayInputStream(doubleEOL));
887              }
888              fileInputStreams.add(new FileInputStream(f));
889            }
890    
891            if (fileInputStreams.size() == 1)
892            {
893              inputStream = fileInputStreams.get(0);
894            }
895            else
896            {
897              inputStream = new AggregateInputStream(fileInputStreams);
898            }
899          }
900          else
901          {
902            inputStream = System.in;
903          }
904    
905          if (sourceCompressed.isPresent())
906          {
907            inputStream = new GZIPInputStream(inputStream);
908          }
909    
910          ldifReader = new LDIFReader(inputStream, numThreads.getValue(),
911               entryTranslator, changeRecordTranslator);
912          if (schema != null)
913          {
914            ldifReader.setSchema(schema);
915          }
916        }
917        catch (final Exception e)
918        {
919          Debug.debugException(e);
920          wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
921               ERR_TRANSFORM_LDIF_ERROR_CREATING_LDIF_READER.get(
922                    StaticUtils.getExceptionMessage(e)));
923          return ResultCode.LOCAL_ERROR;
924        }
925    
926    
927        ResultCode resultCode = ResultCode.SUCCESS;
928        OutputStream outputStream = null;
929    processingBlock:
930        try
931        {
932          // Create the output stream to use to write the transformed data.
933          try
934          {
935            if (targetFile == null)
936            {
937              outputStream = getOut();
938            }
939            else
940            {
941              outputStream =
942                   new FileOutputStream(targetFile, appendToTargetLDIF.isPresent());
943            }
944    
945            if (compressTarget.isPresent())
946            {
947              outputStream = new GZIPOutputStream(outputStream);
948            }
949          }
950          catch (final Exception e)
951          {
952            Debug.debugException(e);
953            wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
954                 ERR_TRANSFORM_LDIF_ERROR_CREATING_OUTPUT_STREAM.get(
955                      targetFile.getAbsolutePath(),
956                      StaticUtils.getExceptionMessage(e)));
957            resultCode = ResultCode.LOCAL_ERROR;
958            break processingBlock;
959          }
960    
961    
962          // Read the source data one record at a time.  The transformations will
963          // automatically be applied by the LDIF reader's translators, and even if
964          // there are multiple reader threads, we're guaranteed to get the results
965          // in the right order.
966          long entriesWritten = 0L;
967          while (true)
968          {
969            final LDIFRecord ldifRecord;
970            try
971            {
972              ldifRecord = ldifReader.readLDIFRecord();
973            }
974            catch (final LDIFException le)
975            {
976              Debug.debugException(le);
977              if (le.mayContinueReading())
978              {
979                wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
980                     ERR_TRANSFORM_LDIF_RECOVERABLE_MALFORMED_RECORD.get(
981                          StaticUtils.getExceptionMessage(le)));
982                if (resultCode == ResultCode.SUCCESS)
983                {
984                  resultCode = ResultCode.PARAM_ERROR;
985                }
986                continue;
987              }
988              else
989              {
990                wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
991                     ERR_TRANSFORM_LDIF_UNRECOVERABLE_MALFORMED_RECORD.get(
992                          StaticUtils.getExceptionMessage(le)));
993                if (resultCode == ResultCode.SUCCESS)
994                {
995                  resultCode = ResultCode.PARAM_ERROR;
996                }
997                break processingBlock;
998              }
999            }
1000            catch (final Exception e)
1001            {
1002              Debug.debugException(e);
1003              wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1004                   ERR_TRANSFORM_LDIF_UNEXPECTED_READ_ERROR.get(
1005                        StaticUtils.getExceptionMessage(e)));
1006              resultCode = ResultCode.LOCAL_ERROR;
1007              break processingBlock;
1008            }
1009    
1010    
1011            // If the LDIF record is null, then we've run out of records so we're
1012            // done.
1013            if (ldifRecord == null)
1014            {
1015              break;
1016            }
1017    
1018    
1019            // Write the record to the output stream.
1020            try
1021            {
1022              if (ldifRecord instanceof PreEncodedLDIFEntry)
1023              {
1024                outputStream.write(
1025                     ((PreEncodedLDIFEntry) ldifRecord).getLDIFBytes());
1026              }
1027              else
1028              {
1029                final ByteStringBuffer buffer = getBuffer();
1030                if (wrapColumn.isPresent())
1031                {
1032                  ldifRecord.toLDIF(buffer, wrapColumn.getValue());
1033                }
1034                else
1035                {
1036                  ldifRecord.toLDIF(buffer, 0);
1037                }
1038                buffer.append(StaticUtils.EOL_BYTES);
1039                buffer.write(outputStream);
1040              }
1041            }
1042            catch (final Exception e)
1043            {
1044              Debug.debugException(e);
1045              wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1046                   ERR_TRANSFORM_LDIF_WRITE_ERROR.get(targetFile.getAbsolutePath(),
1047                        StaticUtils.getExceptionMessage(e)));
1048              resultCode = ResultCode.LOCAL_ERROR;
1049              break processingBlock;
1050            }
1051    
1052    
1053            // If we've written a multiple of 1000 entries, print a progress
1054            // message.
1055            entriesWritten++;
1056            if ((! targetToStandardOutput.isPresent()) &&
1057                ((entriesWritten % 1000L) == 0))
1058            {
1059              final long numExcluded = excludedEntryCount.get();
1060              if (numExcluded > 0L)
1061              {
1062                wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1063                     INFO_TRANSFORM_LDIF_WROTE_ENTRIES_WITH_EXCLUDED.get(
1064                          entriesWritten, numExcluded));
1065              }
1066              else
1067              {
1068                wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1069                     INFO_TRANSFORM_LDIF_WROTE_ENTRIES_NONE_EXCLUDED.get(
1070                          entriesWritten));
1071              }
1072            }
1073          }
1074    
1075    
1076          if (! targetToStandardOutput.isPresent())
1077          {
1078            final long numExcluded = excludedEntryCount.get();
1079            if (numExcluded > 0L)
1080            {
1081              wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1082                   INFO_TRANSFORM_LDIF_COMPLETE_WITH_EXCLUDED.get(entriesWritten,
1083                        numExcluded));
1084            }
1085            else
1086            {
1087              wrapOut(0, MAX_OUTPUT_LINE_LENGTH,
1088                   INFO_TRANSFORM_LDIF_COMPLETE_NONE_EXCLUDED.get(entriesWritten));
1089            }
1090          }
1091        }
1092        finally
1093        {
1094          if (outputStream != null)
1095          {
1096            try
1097            {
1098              outputStream.close();
1099            }
1100            catch (final Exception e)
1101            {
1102              Debug.debugException(e);
1103              wrapErr(0, MAX_OUTPUT_LINE_LENGTH,
1104                   ERR_TRANSFORM_LDIF_ERROR_CLOSING_OUTPUT_STREAM.get(
1105                        targetFile.getAbsolutePath(),
1106                        StaticUtils.getExceptionMessage(e)));
1107              if (resultCode == ResultCode.SUCCESS)
1108              {
1109                resultCode = ResultCode.LOCAL_ERROR;
1110              }
1111            }
1112          }
1113    
1114          try
1115          {
1116            ldifReader.close();
1117          }
1118          catch (final Exception e)
1119          {
1120            Debug.debugException(e);
1121            // We can ignore this.
1122          }
1123        }
1124    
1125    
1126        return resultCode;
1127      }
1128    
1129    
1130    
1131      /**
1132       * Retrieves the schema that should be used for processing.
1133       *
1134       * @return  The schema that was created.
1135       *
1136       * @throws  LDAPException  If a problem is encountered while retrieving the
1137       *                         schema.
1138       */
1139      private Schema getSchema()
1140              throws LDAPException
1141      {
1142        // If any schema paths were specified, then load the schema only from those
1143        // paths.
1144        if (schemaPath.isPresent())
1145        {
1146          final ArrayList<File> schemaFiles = new ArrayList<File>(10);
1147          for (final File path : schemaPath.getValues())
1148          {
1149            if (path.isFile())
1150            {
1151              schemaFiles.add(path);
1152            }
1153            else
1154            {
1155              final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1156              for (final File schemaDirFile : path.listFiles())
1157              {
1158                final String name = schemaDirFile.getName();
1159                if (schemaDirFile.isFile() && name.toLowerCase().endsWith(".ldif"))
1160                {
1161                  fileMap.put(name, schemaDirFile);
1162                }
1163              }
1164              schemaFiles.addAll(fileMap.values());
1165            }
1166          }
1167    
1168          if (schemaFiles.isEmpty())
1169          {
1170            throw new LDAPException(ResultCode.PARAM_ERROR,
1171                 ERR_TRANSFORM_LDIF_NO_SCHEMA_FILES.get(
1172                      schemaPath.getIdentifierString()));
1173          }
1174          else
1175          {
1176            try
1177            {
1178              return Schema.getSchema(schemaFiles);
1179            }
1180            catch (final Exception e)
1181            {
1182              Debug.debugException(e);
1183              throw new LDAPException(ResultCode.LOCAL_ERROR,
1184                   ERR_TRANSFORM_LDIF_ERROR_LOADING_SCHEMA.get(
1185                        StaticUtils.getExceptionMessage(e)));
1186            }
1187          }
1188        }
1189        else
1190        {
1191          // If the INSTANCE_ROOT environment variable is set and it refers to a
1192          // directory that has a config/schema subdirectory that has one or more
1193          // schema files in it, then read the schema from that directory.
1194          try
1195          {
1196            final String instanceRootStr = System.getenv("INSTANCE_ROOT");
1197            if (instanceRootStr != null)
1198            {
1199              final File instanceRoot = new File(instanceRootStr);
1200              final File configDir = new File(instanceRoot, "config");
1201              final File schemaDir = new File(configDir, "schema");
1202              if (schemaDir.exists())
1203              {
1204                final TreeMap<String,File> fileMap = new TreeMap<String,File>();
1205                for (final File schemaDirFile : schemaDir.listFiles())
1206                {
1207                  final String name = schemaDirFile.getName();
1208                  if (schemaDirFile.isFile() &&
1209                      name.toLowerCase().endsWith(".ldif"))
1210                  {
1211                    fileMap.put(name, schemaDirFile);
1212                  }
1213                }
1214    
1215                if (! fileMap.isEmpty())
1216                {
1217                  return Schema.getSchema(new ArrayList<File>(fileMap.values()));
1218                }
1219              }
1220            }
1221          }
1222          catch (final Exception e)
1223          {
1224            Debug.debugException(e);
1225          }
1226        }
1227    
1228    
1229        // If we've gotten here, then just return null and the tool will try to use
1230        // the default standard schema.
1231        return null;
1232      }
1233    
1234    
1235    
1236      /**
1237       * Creates the entry and change record translators that will be used to
1238       * perform the transformations.
1239       *
1240       * @param  entryTranslators         A list to which all created entry
1241       *                                  translators should be written.
1242       * @param  changeRecordTranslators  A list to which all created change record
1243       *                                  translators should be written.
1244       * @param  schema                   The schema to use when processing.
1245       * @param  excludedEntryCount       A counter used to keep track of the number
1246       *                                  of entries that have been excluded from
1247       *                                  the result set.
1248       */
1249      private void createTranslators(
1250           final List<LDIFReaderEntryTranslator> entryTranslators,
1251           final List<LDIFReaderChangeRecordTranslator> changeRecordTranslators,
1252           final Schema schema, final AtomicLong excludedEntryCount)
1253      {
1254        if (scrambleAttribute.isPresent())
1255        {
1256          final Long seed;
1257          if (randomSeed.isPresent())
1258          {
1259            seed = randomSeed.getValue().longValue();
1260          }
1261          else
1262          {
1263            seed = null;
1264          }
1265    
1266          final ScrambleAttributeTransformation t =
1267               new ScrambleAttributeTransformation(schema, seed,
1268                    processDNs.isPresent(), scrambleAttribute.getValues(),
1269                    scrambleJSONField.getValues());
1270          entryTranslators.add(t);
1271          changeRecordTranslators.add(t);
1272        }
1273    
1274        if (sequentialAttribute.isPresent())
1275        {
1276          final long initialValue;
1277          if (initialSequentialValue.isPresent())
1278          {
1279            initialValue = initialSequentialValue.getValue().longValue();
1280          }
1281          else
1282          {
1283            initialValue = 0L;
1284          }
1285    
1286          final long incrementAmount;
1287          if (sequentialValueIncrement.isPresent())
1288          {
1289            incrementAmount = sequentialValueIncrement.getValue().longValue();
1290          }
1291          else
1292          {
1293            incrementAmount = 1L;
1294          }
1295    
1296          for (final String attrName : sequentialAttribute.getValues())
1297          {
1298    
1299    
1300            final ReplaceWithCounterTransformation t =
1301                 new ReplaceWithCounterTransformation(schema, attrName,
1302                      initialValue, incrementAmount,
1303                      textBeforeSequentialValue.getValue(),
1304                      textAfterSequentialValue.getValue(), processDNs.isPresent());
1305            entryTranslators.add(t);
1306          }
1307        }
1308    
1309        if (replaceValuesAttribute.isPresent())
1310        {
1311          final ReplaceAttributeTransformation t =
1312               new ReplaceAttributeTransformation(schema,
1313                    replaceValuesAttribute.getValue(),
1314                    replacementValue.getValues());
1315          entryTranslators.add(t);
1316        }
1317    
1318        if (addAttributeName.isPresent())
1319        {
1320          final AddAttributeTransformation t = new AddAttributeTransformation(
1321               schema, addAttributeBaseDN.getValue(), addAttributeScope.getValue(),
1322               addAttributeFilter.getValue(),
1323               new Attribute(addAttributeName.getValue(), schema,
1324                    addAttributeValue.getValues()),
1325               (! addToExistingValues.isPresent()));
1326          entryTranslators.add(t);
1327        }
1328    
1329        if (renameAttributeFrom.isPresent())
1330        {
1331          final Iterator<String> renameFromIterator =
1332               renameAttributeFrom.getValues().iterator();
1333          final Iterator<String> renameToIterator =
1334               renameAttributeTo.getValues().iterator();
1335          while (renameFromIterator.hasNext())
1336          {
1337            final RenameAttributeTransformation t =
1338                 new RenameAttributeTransformation(schema,
1339                      renameFromIterator.next(), renameToIterator.next(),
1340                      processDNs.isPresent());
1341            entryTranslators.add(t);
1342            changeRecordTranslators.add(t);
1343          }
1344        }
1345    
1346        if (flattenBaseDN.isPresent())
1347        {
1348          final FlattenSubtreeTransformation t = new FlattenSubtreeTransformation(
1349               schema, flattenBaseDN.getValue(),
1350               flattenAddOmittedRDNAttributesToEntry.isPresent(),
1351               flattenAddOmittedRDNAttributesToRDN.isPresent(),
1352               flattenExcludeFilter.getValue());
1353          entryTranslators.add(t);
1354        }
1355    
1356        if (moveSubtreeFrom.isPresent())
1357        {
1358          final Iterator<DN> moveFromIterator =
1359               moveSubtreeFrom.getValues().iterator();
1360          final Iterator<DN> moveToIterator = moveSubtreeTo.getValues().iterator();
1361          while (moveFromIterator.hasNext())
1362          {
1363            final MoveSubtreeTransformation t =
1364                 new MoveSubtreeTransformation(moveFromIterator.next(),
1365                      moveToIterator.next());
1366            entryTranslators.add(t);
1367            changeRecordTranslators.add(t);
1368          }
1369        }
1370    
1371        if (redactAttribute.isPresent())
1372        {
1373          final RedactAttributeTransformation t = new RedactAttributeTransformation(
1374               schema, processDNs.isPresent(),
1375               (! hideRedactedValueCount.isPresent()), redactAttribute.getValues());
1376          entryTranslators.add(t);
1377          changeRecordTranslators.add(t);
1378        }
1379    
1380        if (excludeAttribute.isPresent())
1381        {
1382          final ExcludeAttributeTransformation t =
1383               new ExcludeAttributeTransformation(schema,
1384                    excludeAttribute.getValues());
1385          entryTranslators.add(t);
1386          changeRecordTranslators.add(t);
1387        }
1388    
1389        if (excludeEntryBaseDN.isPresent() || excludeEntryScope.isPresent() ||
1390            excludeEntryFilter.isPresent())
1391        {
1392          final ExcludeEntryTransformation t = new ExcludeEntryTransformation(
1393               schema, excludeEntryBaseDN.getValue(), excludeEntryScope.getValue(),
1394               excludeEntryFilter.getValue(),
1395               (! excludeNonMatchingEntries.isPresent()), excludedEntryCount);
1396          entryTranslators.add(t);
1397        }
1398    
1399        entryTranslators.add(this);
1400      }
1401    
1402    
1403    
1404      /**
1405       * {@inheritDoc}
1406       */
1407      @Override()
1408      public LinkedHashMap<String[],String> getExampleUsages()
1409      {
1410        final LinkedHashMap<String[],String> examples =
1411             new LinkedHashMap<String[],String>(4);
1412    
1413        examples.put(
1414             new String[]
1415             {
1416               "--sourceLDIF", "input.ldif",
1417               "--targetLDIF", "scrambled.ldif",
1418               "--scrambleAttribute", "givenName",
1419               "--scrambleAttribute", "sn",
1420               "--scrambleAttribute", "cn",
1421               "--numThreads", "10",
1422               "--schemaPath", "/ds/UnboundID-DS/config/schema",
1423               "--processDNs"
1424             },
1425             INFO_TRANSFORM_LDIF_EXAMPLE_SCRAMBLE.get());
1426    
1427        examples.put(
1428             new String[]
1429             {
1430               "--sourceLDIF", "input.ldif",
1431               "--targetLDIF", "sequential.ldif",
1432               "--sequentialAttribute", "uid",
1433               "--initialSequentialValue", "1",
1434               "--sequentialValueIncrement", "1",
1435               "--textBeforeSequentialValue", "user.",
1436               "--numThreads", "10",
1437               "--schemaPath", "/ds/UnboundID-DS/config/schema",
1438               "--processDNs"
1439             },
1440             INFO_TRANSFORM_LDIF_EXAMPLE_SEQUENTIAL.get());
1441    
1442        examples.put(
1443             new String[]
1444             {
1445               "--sourceLDIF", "input.ldif",
1446               "--targetLDIF", "added-organization.ldif",
1447               "--addAttributeName", "o",
1448               "--addAttributeValue", "Example Corp.",
1449               "--addAttributeFilter", "(objectClass=person)",
1450               "--numThreads", "10",
1451               "--schemaPath", "/ds/UnboundID-DS/config/schema"
1452             },
1453             INFO_TRANSFORM_LDIF_EXAMPLE_ADD.get());
1454    
1455        examples.put(
1456             new String[]
1457             {
1458               "--sourceLDIF", "input.ldif",
1459               "--targetLDIF", "rebased.ldif",
1460               "--moveSubtreeFrom", "o=example.com",
1461               "--moveSubtreeTo", "dc=example,dc=com",
1462               "--numThreads", "10",
1463               "--schemaPath", "/ds/UnboundID-DS/config/schema"
1464             },
1465             INFO_TRANSFORM_LDIF_EXAMPLE_REBASE.get());
1466    
1467        return examples;
1468      }
1469    
1470    
1471    
1472      /**
1473       * {@inheritDoc}
1474       */
1475      public Entry translate(final Entry original, final long firstLineNumber)
1476             throws LDIFException
1477      {
1478        final ByteStringBuffer buffer = getBuffer();
1479        if (wrapColumn.isPresent())
1480        {
1481          original.toLDIF(buffer, wrapColumn.getValue());
1482        }
1483        else
1484        {
1485          original.toLDIF(buffer, 0);
1486        }
1487        buffer.append(StaticUtils.EOL_BYTES);
1488    
1489        return new PreEncodedLDIFEntry(original, buffer.toByteArray());
1490      }
1491    
1492    
1493    
1494      /**
1495       * Retrieves a byte string buffer that can be used to perform LDIF encoding.
1496       *
1497       * @return  A byte string buffer that can be used to perform LDIF encoding.
1498       */
1499      private ByteStringBuffer getBuffer()
1500      {
1501        ByteStringBuffer buffer = byteStringBuffers.get();
1502        if (buffer == null)
1503        {
1504          buffer = new ByteStringBuffer();
1505          byteStringBuffers.set(buffer);
1506        }
1507        else
1508        {
1509          buffer.clear();
1510        }
1511    
1512        return buffer;
1513      }
1514    }