001    /*
002     * Copyright 2010-2015 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2010-2015 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.IOException;
026    import java.io.OutputStream;
027    import java.io.Serializable;
028    import java.text.ParseException;
029    import java.util.LinkedHashMap;
030    import java.util.LinkedHashSet;
031    import java.util.List;
032    import java.util.Random;
033    import java.util.concurrent.CyclicBarrier;
034    import java.util.concurrent.atomic.AtomicBoolean;
035    import java.util.concurrent.atomic.AtomicLong;
036    
037    import com.unboundid.ldap.sdk.LDAPConnection;
038    import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039    import com.unboundid.ldap.sdk.LDAPException;
040    import com.unboundid.ldap.sdk.ResultCode;
041    import com.unboundid.ldap.sdk.SearchScope;
042    import com.unboundid.ldap.sdk.Version;
043    import com.unboundid.util.ColumnFormatter;
044    import com.unboundid.util.FixedRateBarrier;
045    import com.unboundid.util.FormattableColumn;
046    import com.unboundid.util.HorizontalAlignment;
047    import com.unboundid.util.LDAPCommandLineTool;
048    import com.unboundid.util.ObjectPair;
049    import com.unboundid.util.OutputFormat;
050    import com.unboundid.util.RateAdjustor;
051    import com.unboundid.util.ResultCodeCounter;
052    import com.unboundid.util.ThreadSafety;
053    import com.unboundid.util.ThreadSafetyLevel;
054    import com.unboundid.util.ValuePattern;
055    import com.unboundid.util.WakeableSleeper;
056    import com.unboundid.util.args.ArgumentException;
057    import com.unboundid.util.args.ArgumentParser;
058    import com.unboundid.util.args.BooleanArgument;
059    import com.unboundid.util.args.FileArgument;
060    import com.unboundid.util.args.IntegerArgument;
061    import com.unboundid.util.args.ScopeArgument;
062    import com.unboundid.util.args.StringArgument;
063    
064    import static com.unboundid.util.Debug.*;
065    import static com.unboundid.util.StaticUtils.*;
066    
067    
068    
069    /**
070     * This class provides a tool that can be used to search an LDAP directory
071     * server repeatedly using multiple threads, and then modify each entry
072     * returned by that server.  It can help provide an estimate of the combined
073     * search and modify performance that a directory server is able to achieve.
074     * Either or both of the base DN and the search filter may be a value pattern as
075     * described in the {@link ValuePattern} class.  This makes it possible to
076     * search over a range of entries rather than repeatedly performing searches
077     * with the same base DN and filter.
078     * <BR><BR>
079     * Some of the APIs demonstrated by this example include:
080     * <UL>
081     *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
082     *       package)</LI>
083     *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
084     *       package)</LI>
085     *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
086     *       package)</LI>
087     *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
088     * </UL>
089     * <BR><BR>
090     * All of the necessary information is provided using command line arguments.
091     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
092     * class, as well as the following additional arguments:
093     * <UL>
094     *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
095     *       for the searches.  This must be provided.  It may be a simple DN, or it
096     *       may be a value pattern to express a range of base DNs.</LI>
097     *   <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
098     *       search.  The scope value should be one of "base", "one", "sub", or
099     *       "subord".  If this isn't specified, then a scope of "sub" will be
100     *       used.</LI>
101     *   <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
102     *       the searches.  This must be provided.  It may be a simple filter, or it
103     *       may be a value pattern to express a range of filters.</LI>
104     *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
105     *       attribute that should be included in entries returned from the server.
106     *       If this is not provided, then all user attributes will be requested.
107     *       This may include special tokens that the server may interpret, like
108     *       "1.1" to indicate that no attributes should be returned, "*", for all
109     *       user attributes, or "+" for all operational attributes.  Multiple
110     *       attributes may be requested with multiple instances of this
111     *       argument.</LI>
112     *   <LI>"-m {name}" or "--modifyAttribute {name}" -- specifies the name of the
113     *       attribute to modify.  Multiple attributes may be modified by providing
114     *       multiple instances of this argument.  At least one attribute must be
115     *       provided.</LI>
116     *   <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
117     *       use for the values of the target attributes to modify.  If this is not
118     *       provided, then a default length of 10 bytes will be used.</LI>
119     *   <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
120     *       characters that will be used to generate the values to use for the
121     *       target attributes to modify.  It should only include ASCII characters.
122     *       Values will be generated from randomly-selected characters from this
123     *       set.  If this is not provided, then a default set of lowercase
124     *       alphabetic characters will be used.</LI>
125     *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
126     *       concurrent threads to use when performing the searches.  If this is not
127     *       provided, then a default of one thread will be used.</LI>
128     *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
129     *       time in seconds between lines out output.  If this is not provided,
130     *       then a default interval duration of five seconds will be used.</LI>
131     *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
132     *       intervals for which to run.  If this is not provided, then it will
133     *       run forever.</LI>
134     *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
135     *       iterations that should be performed on a connection before that
136     *       connection is closed and replaced with a newly-established (and
137     *       authenticated, if appropriate) connection.</LI>
138     *   <LI>"-r {ops-per-second}" or "--ratePerSecond {ops-per-second}" --
139     *       specifies the target number of operations to perform per second.  Each
140     *       search and modify operation will be counted separately for this
141     *       purpose, so if a value of 1 is specified and a search returns two
142     *       entries, then a total of three seconds will be required (one for the
143     *       search and one for the modify for each entry).  It is still necessary
144     *       to specify a sufficient number of threads for achieving this rate.  If
145     *       this option is not provided, then the tool will run at the maximum rate
146     *       for the specified number of threads.</LI>
147     *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
148     *       information needed to allow the tool to vary the target rate over time.
149     *       If this option is not provided, then the tool will either use a fixed
150     *       target rate as specified by the "--ratePerSecond" argument, or it will
151     *       run at the maximum rate.</LI>
152     *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
153     *       which sample data will be written illustrating and describing the
154     *       format of the file expected to be used in conjunction with the
155     *       "--variableRateData" argument.</LI>
156     *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
157     *       complete before beginning overall statistics collection.</LI>
158     *   <LI>"--timestampFormat {format}" -- specifies the format to use for
159     *       timestamps included before each output line.  The format may be one of
160     *       "none" (for no timestamps), "with-date" (to include both the date and
161     *       the time), or "without-date" (to include only time time).</LI>
162     *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
163     *       authorization v2 control to request that the operations be processed
164     *       using an alternate authorization identity.  In this case, the bind DN
165     *       should be that of a user that has permission to use this control.  The
166     *       authorization identity may be a value pattern.</LI>
167     *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
168     *       result codes for failed operations should not be displayed.</LI>
169     *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
170     *       display-friendly format.</LI>
171     * </UL>
172     */
173    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
174    public final class SearchAndModRate
175           extends LDAPCommandLineTool
176           implements Serializable
177    {
178      /**
179       * The serial version UID for this serializable class.
180       */
181      private static final long serialVersionUID = 3242469381380526294L;
182    
183    
184    
185      // Indicates whether a request has been made to stop running.
186      private final AtomicBoolean stopRequested;
187    
188      // The argument used to indicate whether to generate output in CSV format.
189      private BooleanArgument csvFormat;
190    
191      // The argument used to indicate whether to suppress information about error
192      // result codes.
193      private BooleanArgument suppressErrors;
194    
195      // The argument used to specify the collection interval.
196      private IntegerArgument collectionInterval;
197    
198      // The argument used to specify the number of search and modify iterations on
199      // a connection before it is closed and re-established.
200      private IntegerArgument iterationsBeforeReconnect;
201    
202      // The argument used to specify the number of intervals.
203      private IntegerArgument numIntervals;
204    
205      // The argument used to specify the number of threads.
206      private IntegerArgument numThreads;
207    
208      // The argument used to specify the seed to use for the random number
209      // generator.
210      private IntegerArgument randomSeed;
211    
212      // The target rate of operations per second.
213      private IntegerArgument ratePerSecond;
214    
215      // The argument used to specify a variable rate file.
216      private FileArgument sampleRateFile;
217    
218      // The argument used to specify a variable rate file.
219      private FileArgument variableRateData;
220    
221      // The argument used to specify the length of the values to generate.
222      private IntegerArgument valueLength;
223    
224      // The number of warm-up intervals to perform.
225      private IntegerArgument warmUpIntervals;
226    
227      // The argument used to specify the scope for the searches.
228      private ScopeArgument scopeArg;
229    
230      // The argument used to specify the base DNs for the searches.
231      private StringArgument baseDN;
232    
233      // The argument used to specify the set of characters to use when generating
234      // values.
235      private StringArgument characterSet;
236    
237      // The argument used to specify the filters for the searches.
238      private StringArgument filter;
239    
240      // The argument used to specify the attributes to modify.
241      private StringArgument modifyAttributes;
242    
243      // The argument used to specify the proxied authorization identity.
244      private StringArgument proxyAs;
245    
246      // The argument used to specify the attributes to return.
247      private StringArgument returnAttributes;
248    
249      // The argument used to specify the timestamp format.
250      private StringArgument timestampFormat;
251    
252      // The thread currently being used to run the searchrate tool.
253      private volatile Thread runningThread;
254    
255      // A wakeable sleeper that will be used to sleep between reporting intervals.
256      private final WakeableSleeper sleeper;
257    
258    
259    
260      /**
261       * Parse the provided command line arguments and make the appropriate set of
262       * changes.
263       *
264       * @param  args  The command line arguments provided to this program.
265       */
266      public static void main(final String[] args)
267      {
268        final ResultCode resultCode = main(args, System.out, System.err);
269        if (resultCode != ResultCode.SUCCESS)
270        {
271          System.exit(resultCode.intValue());
272        }
273      }
274    
275    
276    
277      /**
278       * Parse the provided command line arguments and make the appropriate set of
279       * changes.
280       *
281       * @param  args       The command line arguments provided to this program.
282       * @param  outStream  The output stream to which standard out should be
283       *                    written.  It may be {@code null} if output should be
284       *                    suppressed.
285       * @param  errStream  The output stream to which standard error should be
286       *                    written.  It may be {@code null} if error messages
287       *                    should be suppressed.
288       *
289       * @return  A result code indicating whether the processing was successful.
290       */
291      public static ResultCode main(final String[] args,
292                                    final OutputStream outStream,
293                                    final OutputStream errStream)
294      {
295        final SearchAndModRate searchAndModRate =
296             new SearchAndModRate(outStream, errStream);
297        return searchAndModRate.runTool(args);
298      }
299    
300    
301    
302      /**
303       * Creates a new instance of this tool.
304       *
305       * @param  outStream  The output stream to which standard out should be
306       *                    written.  It may be {@code null} if output should be
307       *                    suppressed.
308       * @param  errStream  The output stream to which standard error should be
309       *                    written.  It may be {@code null} if error messages
310       *                    should be suppressed.
311       */
312      public SearchAndModRate(final OutputStream outStream,
313                              final OutputStream errStream)
314      {
315        super(outStream, errStream);
316    
317        stopRequested = new AtomicBoolean(false);
318        sleeper = new WakeableSleeper();
319      }
320    
321    
322    
323      /**
324       * Retrieves the name for this tool.
325       *
326       * @return  The name for this tool.
327       */
328      @Override()
329      public String getToolName()
330      {
331        return "search-and-mod-rate";
332      }
333    
334    
335    
336      /**
337       * Retrieves the description for this tool.
338       *
339       * @return  The description for this tool.
340       */
341      @Override()
342      public String getToolDescription()
343      {
344        return "Perform repeated searches against an " +
345               "LDAP directory server and modify each entry returned.";
346      }
347    
348    
349    
350      /**
351       * Retrieves the version string for this tool.
352       *
353       * @return  The version string for this tool.
354       */
355      @Override()
356      public String getToolVersion()
357      {
358        return Version.NUMERIC_VERSION_STRING;
359      }
360    
361    
362    
363      /**
364       * Adds the arguments used by this program that aren't already provided by the
365       * generic {@code LDAPCommandLineTool} framework.
366       *
367       * @param  parser  The argument parser to which the arguments should be added.
368       *
369       * @throws  ArgumentException  If a problem occurs while adding the arguments.
370       */
371      @Override()
372      public void addNonLDAPArguments(final ArgumentParser parser)
373             throws ArgumentException
374      {
375        String description = "The base DN to use for the searches.  It may be a " +
376             "simple DN or a value pattern to specify a range of DNs (e.g., " +
377             "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  This must be " +
378             "provided.";
379        baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
380        parser.addArgument(baseDN);
381    
382    
383        description = "The scope to use for the searches.  It should be 'base', " +
384                      "'one', 'sub', or 'subord'.  If this is not provided, then " +
385                      "a default scope of 'sub' will be used.";
386        scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
387                                     SearchScope.SUB);
388        parser.addArgument(scopeArg);
389    
390    
391        description = "The filter to use for the searches.  It may be a simple " +
392                      "filter or a value pattern to specify a range of filters " +
393                      "(e.g., \"(uid=user.[1-1000])\").  This must be provided.";
394        filter = new StringArgument('f', "filter", true, 1, "{filter}",
395                                    description);
396        parser.addArgument(filter);
397    
398    
399        description = "The name of an attribute to include in entries returned " +
400                      "from the searches.  Multiple attributes may be requested " +
401                      "by providing this argument multiple times.  If no request " +
402                      "attributes are provided, then the entries returned will " +
403                      "include all user attributes.";
404        returnAttributes = new StringArgument('A', "attribute", false, 0, "{name}",
405                                              description);
406        parser.addArgument(returnAttributes);
407    
408    
409        description = "The name of the attribute to modify.  Multiple attributes " +
410                      "may be specified by providing this argument multiple " +
411                      "times.  At least one attribute must be specified.";
412        modifyAttributes = new StringArgument('m', "modifyAttribute", true, 0,
413                                              "{name}", description);
414        parser.addArgument(modifyAttributes);
415    
416    
417        description = "The length in bytes to use when generating values for the " +
418                      "modifications.  If this is not provided, then a default " +
419                      "length of ten bytes will be used.";
420        valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
421                                          description, 1, Integer.MAX_VALUE, 10);
422        parser.addArgument(valueLength);
423    
424    
425        description = "The set of characters to use to generate the values for " +
426                      "the modifications.  It should only include ASCII " +
427                      "characters.  If this is not provided, then a default set " +
428                      "of lowercase alphabetic characters will be used.";
429        characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
430                                          description,
431                                          "abcdefghijklmnopqrstuvwxyz");
432        parser.addArgument(characterSet);
433    
434    
435        description = "The number of threads to use to perform the searches.  If " +
436                      "this is not provided, then a default of one thread will " +
437                      "be used.";
438        numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
439                                         description, 1, Integer.MAX_VALUE, 1);
440        parser.addArgument(numThreads);
441    
442    
443        description = "The length of time in seconds between output lines.  If " +
444                      "this is not provided, then a default interval of five " +
445                      "seconds will be used.";
446        collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
447                                                 "{num}", description, 1,
448                                                 Integer.MAX_VALUE, 5);
449        parser.addArgument(collectionInterval);
450    
451    
452        description = "The maximum number of intervals for which to run.  If " +
453                      "this is not provided, then the tool will run until it is " +
454                      "interrupted.";
455        numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
456                                           description, 1, Integer.MAX_VALUE,
457                                           Integer.MAX_VALUE);
458        parser.addArgument(numIntervals);
459    
460        description = "The number of search and modify iterations that should be " +
461                      "processed on a connection before that connection is " +
462                      "closed and replaced with a newly-established (and " +
463                      "authenticated, if appropriate) connection.  If this is " +
464                      "not provided, then connections will not be periodically " +
465                      "closed and re-established.";
466        iterationsBeforeReconnect = new IntegerArgument(null,
467             "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
468        parser.addArgument(iterationsBeforeReconnect);
469    
470        description = "The target number of searches to perform per second.  It " +
471                      "is still necessary to specify a sufficient number of " +
472                      "threads for achieving this rate.  If neither this option " +
473                      "nor --variableRateData is provided, then the tool will " +
474                      "run at the maximum rate for the specified number of " +
475                      "threads.";
476        ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
477                                            "{searches-per-second}", description,
478                                            1, Integer.MAX_VALUE);
479        parser.addArgument(ratePerSecond);
480    
481        final String variableRateDataArgName = "variableRateData";
482        final String generateSampleRateFileArgName = "generateSampleRateFile";
483        description = RateAdjustor.getVariableRateDataArgumentDescription(
484             generateSampleRateFileArgName);
485        variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
486                                            "{path}", description, true, true, true,
487                                            false);
488        parser.addArgument(variableRateData);
489    
490        description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
491             variableRateDataArgName);
492        sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
493                                          false, 1, "{path}", description, false,
494                                          true, true, false);
495        sampleRateFile.setUsageArgument(true);
496        parser.addArgument(sampleRateFile);
497        parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
498    
499        description = "The number of intervals to complete before beginning " +
500                      "overall statistics collection.  Specifying a nonzero " +
501                      "number of warm-up intervals gives the client and server " +
502                      "a chance to warm up without skewing performance results.";
503        warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
504             "{num}", description, 0, Integer.MAX_VALUE, 0);
505        parser.addArgument(warmUpIntervals);
506    
507        description = "Indicates the format to use for timestamps included in " +
508                      "the output.  A value of 'none' indicates that no " +
509                      "timestamps should be included.  A value of 'with-date' " +
510                      "indicates that both the date and the time should be " +
511                      "included.  A value of 'without-date' indicates that only " +
512                      "the time should be included.";
513        final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
514        allowedFormats.add("none");
515        allowedFormats.add("with-date");
516        allowedFormats.add("without-date");
517        timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
518             "{format}", description, allowedFormats, "none");
519        parser.addArgument(timestampFormat);
520    
521        description = "Indicates that the proxied authorization control (as " +
522                      "defined in RFC 4370) should be used to request that " +
523                      "operations be processed using an alternate authorization " +
524                      "identity.";
525        proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
526                                     description);
527        parser.addArgument(proxyAs);
528    
529        description = "Indicates that information about the result codes for " +
530                      "failed operations should not be displayed.";
531        suppressErrors = new BooleanArgument(null,
532             "suppressErrorResultCodes", 1, description);
533        parser.addArgument(suppressErrors);
534    
535        description = "Generate output in CSV format rather than a " +
536                      "display-friendly format";
537        csvFormat = new BooleanArgument('c', "csv", 1, description);
538        parser.addArgument(csvFormat);
539    
540        description = "Specifies the seed to use for the random number generator.";
541        randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
542             description);
543        parser.addArgument(randomSeed);
544      }
545    
546    
547    
548      /**
549       * Indicates whether this tool supports creating connections to multiple
550       * servers.  If it is to support multiple servers, then the "--hostname" and
551       * "--port" arguments will be allowed to be provided multiple times, and
552       * will be required to be provided the same number of times.  The same type of
553       * communication security and bind credentials will be used for all servers.
554       *
555       * @return  {@code true} if this tool supports creating connections to
556       *          multiple servers, or {@code false} if not.
557       */
558      @Override()
559      protected boolean supportsMultipleServers()
560      {
561        return true;
562      }
563    
564    
565    
566      /**
567       * Retrieves the connection options that should be used for connections
568       * created for use with this tool.
569       *
570       * @return  The connection options that should be used for connections created
571       *          for use with this tool.
572       */
573      @Override()
574      public LDAPConnectionOptions getConnectionOptions()
575      {
576        final LDAPConnectionOptions options = new LDAPConnectionOptions();
577        options.setAutoReconnect(true);
578        options.setUseSynchronousMode(true);
579        return options;
580      }
581    
582    
583    
584      /**
585       * Performs the actual processing for this tool.  In this case, it gets a
586       * connection to the directory server and uses it to perform the requested
587       * searches.
588       *
589       * @return  The result code for the processing that was performed.
590       */
591      @Override()
592      public ResultCode doToolProcessing()
593      {
594        runningThread = Thread.currentThread();
595    
596        try
597        {
598          return doToolProcessingInternal();
599        }
600        finally
601        {
602          runningThread = null;
603        }
604      }
605    
606    
607    
608      /**
609       * Performs the actual processing for this tool.  In this case, it gets a
610       * connection to the directory server and uses it to perform the requested
611       * searches.
612       *
613       * @return  The result code for the processing that was performed.
614       */
615      private ResultCode doToolProcessingInternal()
616      {
617        // If the sample rate file argument was specified, then generate the sample
618        // variable rate data file and return.
619        if (sampleRateFile.isPresent())
620        {
621          try
622          {
623            RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
624            return ResultCode.SUCCESS;
625          }
626          catch (final Exception e)
627          {
628            debugException(e);
629            err("An error occurred while trying to write sample variable data " +
630                 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
631                 "':  ", getExceptionMessage(e));
632            return ResultCode.LOCAL_ERROR;
633          }
634        }
635    
636    
637        // Determine the random seed to use.
638        final Long seed;
639        if (randomSeed.isPresent())
640        {
641          seed = Long.valueOf(randomSeed.getValue());
642        }
643        else
644        {
645          seed = null;
646        }
647    
648        // Create value patterns for the base DN, filter, and proxied authorization
649        // DN.
650        final ValuePattern dnPattern;
651        try
652        {
653          dnPattern = new ValuePattern(baseDN.getValue(), seed);
654        }
655        catch (final ParseException pe)
656        {
657          debugException(pe);
658          err("Unable to parse the base DN value pattern:  ", pe.getMessage());
659          return ResultCode.PARAM_ERROR;
660        }
661    
662        final ValuePattern filterPattern;
663        try
664        {
665          filterPattern = new ValuePattern(filter.getValue(), seed);
666        }
667        catch (final ParseException pe)
668        {
669          debugException(pe);
670          err("Unable to parse the filter pattern:  ", pe.getMessage());
671          return ResultCode.PARAM_ERROR;
672        }
673    
674        final ValuePattern authzIDPattern;
675        if (proxyAs.isPresent())
676        {
677          try
678          {
679            authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
680          }
681          catch (final ParseException pe)
682          {
683            debugException(pe);
684            err("Unable to parse the proxied authorization pattern:  ",
685                pe.getMessage());
686            return ResultCode.PARAM_ERROR;
687          }
688        }
689        else
690        {
691          authzIDPattern = null;
692        }
693    
694    
695        // Get the attributes to return.
696        final String[] returnAttrs;
697        if (returnAttributes.isPresent())
698        {
699          final List<String> attrList = returnAttributes.getValues();
700          returnAttrs = new String[attrList.size()];
701          attrList.toArray(returnAttrs);
702        }
703        else
704        {
705          returnAttrs = NO_STRINGS;
706        }
707    
708    
709        // Get the names of the attributes to modify.
710        final String[] modAttrs = new String[modifyAttributes.getValues().size()];
711        modifyAttributes.getValues().toArray(modAttrs);
712    
713    
714        // Get the character set as a byte array.
715        final byte[] charSet = getBytes(characterSet.getValue());
716    
717    
718        // If the --ratePerSecond option was specified, then limit the rate
719        // accordingly.
720        FixedRateBarrier fixedRateBarrier = null;
721        if (ratePerSecond.isPresent() || variableRateData.isPresent())
722        {
723          // We might not have a rate per second if --variableRateData is specified.
724          // The rate typically doesn't matter except when we have warm-up
725          // intervals.  In this case, we'll run at the max rate.
726          final int intervalSeconds = collectionInterval.getValue();
727          final int ratePerInterval =
728               (ratePerSecond.getValue() == null)
729               ? Integer.MAX_VALUE
730               : ratePerSecond.getValue() * intervalSeconds;
731          fixedRateBarrier =
732               new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
733        }
734    
735    
736        // If --variableRateData was specified, then initialize a RateAdjustor.
737        RateAdjustor rateAdjustor = null;
738        if (variableRateData.isPresent())
739        {
740          try
741          {
742            rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
743                 ratePerSecond.getValue(), variableRateData.getValue());
744          }
745          catch (final IOException e)
746          {
747            debugException(e);
748            err("Initializing the variable rates failed: " + e.getMessage());
749            return ResultCode.PARAM_ERROR;
750          }
751          catch (final IllegalArgumentException e)
752          {
753            debugException(e);
754            err("Initializing the variable rates failed: " + e.getMessage());
755            return ResultCode.PARAM_ERROR;
756          }
757        }
758    
759    
760        // Determine whether to include timestamps in the output and if so what
761        // format should be used for them.
762        final boolean includeTimestamp;
763        final String timeFormat;
764        if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
765        {
766          includeTimestamp = true;
767          timeFormat       = "dd/MM/yyyy HH:mm:ss";
768        }
769        else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
770        {
771          includeTimestamp = true;
772          timeFormat       = "HH:mm:ss";
773        }
774        else
775        {
776          includeTimestamp = false;
777          timeFormat       = null;
778        }
779    
780    
781        // Determine whether any warm-up intervals should be run.
782        final long totalIntervals;
783        final boolean warmUp;
784        int remainingWarmUpIntervals = warmUpIntervals.getValue();
785        if (remainingWarmUpIntervals > 0)
786        {
787          warmUp = true;
788          totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
789        }
790        else
791        {
792          warmUp = true;
793          totalIntervals = 0L + numIntervals.getValue();
794        }
795    
796    
797        // Create the table that will be used to format the output.
798        final OutputFormat outputFormat;
799        if (csvFormat.isPresent())
800        {
801          outputFormat = OutputFormat.CSV;
802        }
803        else
804        {
805          outputFormat = OutputFormat.COLUMNS;
806        }
807    
808        final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
809             timeFormat, outputFormat, " ",
810             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
811                      "Searches/Sec"),
812             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
813                      "Srch Dur ms"),
814             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
815                      "Mods/Sec"),
816             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
817                      "Mod Dur ms"),
818             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
819                      "Errors/Sec"),
820             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
821                      "Searches/Sec"),
822             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
823                      "Srch Dur ms"),
824             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
825                      "Mods/Sec"),
826             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
827                      "Mod Dur ms"));
828    
829    
830        // Create values to use for statistics collection.
831        final AtomicLong        searchCounter   = new AtomicLong(0L);
832        final AtomicLong        errorCounter    = new AtomicLong(0L);
833        final AtomicLong        modCounter      = new AtomicLong(0L);
834        final AtomicLong        modDurations    = new AtomicLong(0L);
835        final AtomicLong        searchDurations = new AtomicLong(0L);
836        final ResultCodeCounter rcCounter       = new ResultCodeCounter();
837    
838    
839        // Determine the length of each interval in milliseconds.
840        final long intervalMillis = 1000L * collectionInterval.getValue();
841    
842    
843        // Create the threads to use for the searches.
844        final Random random = new Random();
845        final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
846        final SearchAndModRateThread[] threads =
847             new SearchAndModRateThread[numThreads.getValue()];
848        for (int i=0; i < threads.length; i++)
849        {
850          final LDAPConnection connection;
851          try
852          {
853            connection = getConnection();
854          }
855          catch (final LDAPException le)
856          {
857            debugException(le);
858            err("Unable to connect to the directory server:  ",
859                getExceptionMessage(le));
860            return le.getResultCode();
861          }
862    
863          threads[i] = new SearchAndModRateThread(this, i, connection, dnPattern,
864               scopeArg.getValue(), filterPattern, returnAttrs, modAttrs,
865               valueLength.getValue(), charSet, authzIDPattern,
866               iterationsBeforeReconnect.getValue(), random.nextLong(), barrier,
867               searchCounter, modCounter, searchDurations, modDurations,
868               errorCounter, rcCounter, fixedRateBarrier);
869          threads[i].start();
870        }
871    
872    
873        // Display the table header.
874        for (final String headerLine : formatter.getHeaderLines(true))
875        {
876          out(headerLine);
877        }
878    
879    
880        // Start the RateAdjustor before the threads so that the initial value is
881        // in place before any load is generated unless we're doing a warm-up in
882        // which case, we'll start it after the warm-up is complete.
883        if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
884        {
885          rateAdjustor.start();
886        }
887    
888    
889        // Indicate that the threads can start running.
890        try
891        {
892          barrier.await();
893        }
894        catch (final Exception e)
895        {
896          debugException(e);
897        }
898    
899        long overallStartTime = System.nanoTime();
900        long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
901    
902    
903        boolean setOverallStartTime = false;
904        long    lastSearchDuration  = 0L;
905        long    lastModDuration     = 0L;
906        long    lastNumErrors       = 0L;
907        long    lastNumSearches     = 0L;
908        long    lastNumMods          = 0L;
909        long    lastEndTime         = System.nanoTime();
910        for (long i=0; i < totalIntervals; i++)
911        {
912          if (rateAdjustor != null)
913          {
914            if (! rateAdjustor.isAlive())
915            {
916              out("All of the rates in " + variableRateData.getValue().getName() +
917                  " have been completed.");
918              break;
919            }
920          }
921    
922          final long startTimeMillis = System.currentTimeMillis();
923          final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
924          nextIntervalStartTime += intervalMillis;
925          if (sleepTimeMillis > 0)
926          {
927            sleeper.sleep(sleepTimeMillis);
928          }
929    
930          if (stopRequested.get())
931          {
932            break;
933          }
934    
935          final long endTime          = System.nanoTime();
936          final long intervalDuration = endTime - lastEndTime;
937    
938          final long numSearches;
939          final long numMods;
940          final long numErrors;
941          final long totalSearchDuration;
942          final long totalModDuration;
943          if (warmUp && (remainingWarmUpIntervals > 0))
944          {
945            numSearches         = searchCounter.getAndSet(0L);
946            numMods             = modCounter.getAndSet(0L);
947            numErrors           = errorCounter.getAndSet(0L);
948            totalSearchDuration = searchDurations.getAndSet(0L);
949            totalModDuration    = modDurations.getAndSet(0L);
950          }
951          else
952          {
953            numSearches         = searchCounter.get();
954            numMods             = modCounter.get();
955            numErrors           = errorCounter.get();
956            totalSearchDuration = searchDurations.get();
957            totalModDuration    = modDurations.get();
958          }
959    
960          final long recentNumSearches = numSearches - lastNumSearches;
961          final long recentNumMods = numMods - lastNumMods;
962          final long recentNumErrors = numErrors - lastNumErrors;
963          final long recentSearchDuration =
964               totalSearchDuration - lastSearchDuration;
965          final long recentModDuration = totalModDuration - lastModDuration;
966    
967          final double numSeconds = intervalDuration / 1000000000.0d;
968          final double recentSearchRate = recentNumSearches / numSeconds;
969          final double recentModRate = recentNumMods / numSeconds;
970          final double recentErrorRate  = recentNumErrors / numSeconds;
971    
972          final double recentAvgSearchDuration;
973          if (recentNumSearches > 0L)
974          {
975            recentAvgSearchDuration =
976                 1.0d * recentSearchDuration / recentNumSearches / 1000000;
977          }
978          else
979          {
980            recentAvgSearchDuration = 0.0d;
981          }
982    
983          final double recentAvgModDuration;
984          if (recentNumMods > 0L)
985          {
986            recentAvgModDuration =
987                 1.0d * recentModDuration / recentNumMods / 1000000;
988          }
989          else
990          {
991            recentAvgModDuration = 0.0d;
992          }
993    
994          if (warmUp && (remainingWarmUpIntervals > 0))
995          {
996            out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
997                 recentModRate, recentAvgModDuration, recentErrorRate, "warming up",
998                 "warming up", "warming up", "warming up"));
999    
1000            remainingWarmUpIntervals--;
1001            if (remainingWarmUpIntervals == 0)
1002            {
1003              out("Warm-up completed.  Beginning overall statistics collection.");
1004              setOverallStartTime = true;
1005              if (rateAdjustor != null)
1006              {
1007                rateAdjustor.start();
1008              }
1009            }
1010          }
1011          else
1012          {
1013            if (setOverallStartTime)
1014            {
1015              overallStartTime    = lastEndTime;
1016              setOverallStartTime = false;
1017            }
1018    
1019            final double numOverallSeconds =
1020                 (endTime - overallStartTime) / 1000000000.0d;
1021            final double overallSearchRate = numSearches / numOverallSeconds;
1022            final double overallModRate = numMods / numOverallSeconds;
1023    
1024            final double overallAvgSearchDuration;
1025            if (numSearches > 0L)
1026            {
1027              overallAvgSearchDuration =
1028                   1.0d * totalSearchDuration / numSearches / 1000000;
1029            }
1030            else
1031            {
1032              overallAvgSearchDuration = 0.0d;
1033            }
1034    
1035            final double overallAvgModDuration;
1036            if (numMods > 0L)
1037            {
1038              overallAvgModDuration =
1039                   1.0d * totalModDuration / numMods / 1000000;
1040            }
1041            else
1042            {
1043              overallAvgModDuration = 0.0d;
1044            }
1045    
1046            out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1047                 recentModRate, recentAvgModDuration, recentErrorRate,
1048                 overallSearchRate, overallAvgSearchDuration, overallModRate,
1049                 overallAvgModDuration));
1050    
1051            lastNumSearches    = numSearches;
1052            lastNumMods        = numMods;
1053            lastNumErrors      = numErrors;
1054            lastSearchDuration = totalSearchDuration;
1055            lastModDuration    = totalModDuration;
1056          }
1057    
1058          final List<ObjectPair<ResultCode,Long>> rcCounts =
1059               rcCounter.getCounts(true);
1060          if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1061          {
1062            err("\tError Results:");
1063            for (final ObjectPair<ResultCode,Long> p : rcCounts)
1064            {
1065              err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1066            }
1067          }
1068    
1069          lastEndTime = endTime;
1070        }
1071    
1072    
1073        // Shut down the RateAdjustor if we have one.
1074        if (rateAdjustor != null)
1075        {
1076          rateAdjustor.shutDown();
1077        }
1078    
1079        // Stop all of the threads.
1080        ResultCode resultCode = ResultCode.SUCCESS;
1081        for (final SearchAndModRateThread t : threads)
1082        {
1083          final ResultCode r = t.stopRunning();
1084          if (resultCode == ResultCode.SUCCESS)
1085          {
1086            resultCode = r;
1087          }
1088        }
1089    
1090        return resultCode;
1091      }
1092    
1093    
1094    
1095      /**
1096       * Requests that this tool stop running.  This method will attempt to wait
1097       * for all threads to complete before returning control to the caller.
1098       */
1099      public void stopRunning()
1100      {
1101        stopRequested.set(true);
1102        sleeper.wakeup();
1103    
1104        final Thread t = runningThread;
1105        if (t != null)
1106        {
1107          try
1108          {
1109            t.join();
1110          }
1111          catch (final Exception e)
1112          {
1113            debugException(e);
1114          }
1115        }
1116      }
1117    
1118    
1119    
1120      /**
1121       * {@inheritDoc}
1122       */
1123      @Override()
1124      public LinkedHashMap<String[],String> getExampleUsages()
1125      {
1126        final LinkedHashMap<String[],String> examples =
1127             new LinkedHashMap<String[],String>(2);
1128    
1129        String[] args =
1130        {
1131          "--hostname", "server.example.com",
1132          "--port", "389",
1133          "--bindDN", "uid=admin,dc=example,dc=com",
1134          "--bindPassword", "password",
1135          "--baseDN", "dc=example,dc=com",
1136          "--scope", "sub",
1137          "--filter", "(uid=user.[1-1000000])",
1138          "--attribute", "givenName",
1139          "--attribute", "sn",
1140          "--attribute", "mail",
1141          "--modifyAttribute", "description",
1142          "--valueLength", "10",
1143          "--characterSet", "abcdefghijklmnopqrstuvwxyz0123456789",
1144          "--numThreads", "10"
1145        };
1146        String description =
1147             "Test search and modify performance by searching randomly across a " +
1148             "set of one million users located below 'dc=example,dc=com' with " +
1149             "ten concurrent threads.  The entries returned to the client will " +
1150             "include the givenName, sn, and mail attributes, and the " +
1151             "description attribute of each entry returned will be replaced " +
1152             "with a string of ten randomly-selected alphanumeric characters.";
1153        examples.put(args, description);
1154    
1155        args = new String[]
1156        {
1157          "--generateSampleRateFile", "variable-rate-data.txt"
1158        };
1159        description =
1160             "Generate a sample variable rate definition file that may be used " +
1161             "in conjunction with the --variableRateData argument.  The sample " +
1162             "file will include comments that describe the format for data to be " +
1163             "included in this file.";
1164        examples.put(args, description);
1165    
1166        return examples;
1167      }
1168    }