001    /*
002     * Copyright 2010-2016 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2010-2016 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.IOException;
026    import java.io.OutputStream;
027    import java.io.Serializable;
028    import java.text.ParseException;
029    import java.util.LinkedHashMap;
030    import java.util.LinkedHashSet;
031    import java.util.List;
032    import java.util.Random;
033    import java.util.concurrent.CyclicBarrier;
034    import java.util.concurrent.atomic.AtomicBoolean;
035    import java.util.concurrent.atomic.AtomicLong;
036    
037    import com.unboundid.ldap.sdk.LDAPConnection;
038    import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039    import com.unboundid.ldap.sdk.LDAPException;
040    import com.unboundid.ldap.sdk.ResultCode;
041    import com.unboundid.ldap.sdk.SearchScope;
042    import com.unboundid.ldap.sdk.Version;
043    import com.unboundid.util.ColumnFormatter;
044    import com.unboundid.util.FixedRateBarrier;
045    import com.unboundid.util.FormattableColumn;
046    import com.unboundid.util.HorizontalAlignment;
047    import com.unboundid.util.LDAPCommandLineTool;
048    import com.unboundid.util.ObjectPair;
049    import com.unboundid.util.OutputFormat;
050    import com.unboundid.util.RateAdjustor;
051    import com.unboundid.util.ResultCodeCounter;
052    import com.unboundid.util.ThreadSafety;
053    import com.unboundid.util.ThreadSafetyLevel;
054    import com.unboundid.util.ValuePattern;
055    import com.unboundid.util.WakeableSleeper;
056    import com.unboundid.util.args.ArgumentException;
057    import com.unboundid.util.args.ArgumentParser;
058    import com.unboundid.util.args.BooleanArgument;
059    import com.unboundid.util.args.FileArgument;
060    import com.unboundid.util.args.IntegerArgument;
061    import com.unboundid.util.args.ScopeArgument;
062    import com.unboundid.util.args.StringArgument;
063    
064    import static com.unboundid.util.Debug.*;
065    import static com.unboundid.util.StaticUtils.*;
066    
067    
068    
069    /**
070     * This class provides a tool that can be used to search an LDAP directory
071     * server repeatedly using multiple threads, and then modify each entry
072     * returned by that server.  It can help provide an estimate of the combined
073     * search and modify performance that a directory server is able to achieve.
074     * Either or both of the base DN and the search filter may be a value pattern as
075     * described in the {@link ValuePattern} class.  This makes it possible to
076     * search over a range of entries rather than repeatedly performing searches
077     * with the same base DN and filter.
078     * <BR><BR>
079     * Some of the APIs demonstrated by this example include:
080     * <UL>
081     *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
082     *       package)</LI>
083     *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
084     *       package)</LI>
085     *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
086     *       package)</LI>
087     *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
088     * </UL>
089     * <BR><BR>
090     * All of the necessary information is provided using command line arguments.
091     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
092     * class, as well as the following additional arguments:
093     * <UL>
094     *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
095     *       for the searches.  This must be provided.  It may be a simple DN, or it
096     *       may be a value pattern to express a range of base DNs.</LI>
097     *   <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
098     *       search.  The scope value should be one of "base", "one", "sub", or
099     *       "subord".  If this isn't specified, then a scope of "sub" will be
100     *       used.</LI>
101     *   <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
102     *       the searches.  This must be provided.  It may be a simple filter, or it
103     *       may be a value pattern to express a range of filters.</LI>
104     *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
105     *       attribute that should be included in entries returned from the server.
106     *       If this is not provided, then all user attributes will be requested.
107     *       This may include special tokens that the server may interpret, like
108     *       "1.1" to indicate that no attributes should be returned, "*", for all
109     *       user attributes, or "+" for all operational attributes.  Multiple
110     *       attributes may be requested with multiple instances of this
111     *       argument.</LI>
112     *   <LI>"-m {name}" or "--modifyAttribute {name}" -- specifies the name of the
113     *       attribute to modify.  Multiple attributes may be modified by providing
114     *       multiple instances of this argument.  At least one attribute must be
115     *       provided.</LI>
116     *   <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
117     *       use for the values of the target attributes to modify.  If this is not
118     *       provided, then a default length of 10 bytes will be used.</LI>
119     *   <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
120     *       characters that will be used to generate the values to use for the
121     *       target attributes to modify.  It should only include ASCII characters.
122     *       Values will be generated from randomly-selected characters from this
123     *       set.  If this is not provided, then a default set of lowercase
124     *       alphabetic characters will be used.</LI>
125     *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
126     *       concurrent threads to use when performing the searches.  If this is not
127     *       provided, then a default of one thread will be used.</LI>
128     *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
129     *       time in seconds between lines out output.  If this is not provided,
130     *       then a default interval duration of five seconds will be used.</LI>
131     *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
132     *       intervals for which to run.  If this is not provided, then it will
133     *       run forever.</LI>
134     *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
135     *       iterations that should be performed on a connection before that
136     *       connection is closed and replaced with a newly-established (and
137     *       authenticated, if appropriate) connection.</LI>
138     *   <LI>"-r {ops-per-second}" or "--ratePerSecond {ops-per-second}" --
139     *       specifies the target number of operations to perform per second.  Each
140     *       search and modify operation will be counted separately for this
141     *       purpose, so if a value of 1 is specified and a search returns two
142     *       entries, then a total of three seconds will be required (one for the
143     *       search and one for the modify for each entry).  It is still necessary
144     *       to specify a sufficient number of threads for achieving this rate.  If
145     *       this option is not provided, then the tool will run at the maximum rate
146     *       for the specified number of threads.</LI>
147     *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
148     *       information needed to allow the tool to vary the target rate over time.
149     *       If this option is not provided, then the tool will either use a fixed
150     *       target rate as specified by the "--ratePerSecond" argument, or it will
151     *       run at the maximum rate.</LI>
152     *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
153     *       which sample data will be written illustrating and describing the
154     *       format of the file expected to be used in conjunction with the
155     *       "--variableRateData" argument.</LI>
156     *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
157     *       complete before beginning overall statistics collection.</LI>
158     *   <LI>"--timestampFormat {format}" -- specifies the format to use for
159     *       timestamps included before each output line.  The format may be one of
160     *       "none" (for no timestamps), "with-date" (to include both the date and
161     *       the time), or "without-date" (to include only time time).</LI>
162     *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
163     *       authorization v2 control to request that the operations be processed
164     *       using an alternate authorization identity.  In this case, the bind DN
165     *       should be that of a user that has permission to use this control.  The
166     *       authorization identity may be a value pattern.</LI>
167     *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
168     *       result codes for failed operations should not be displayed.</LI>
169     *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
170     *       display-friendly format.</LI>
171     * </UL>
172     */
173    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
174    public final class SearchAndModRate
175           extends LDAPCommandLineTool
176           implements Serializable
177    {
178      /**
179       * The serial version UID for this serializable class.
180       */
181      private static final long serialVersionUID = 3242469381380526294L;
182    
183    
184    
185      // Indicates whether a request has been made to stop running.
186      private final AtomicBoolean stopRequested;
187    
188      // The argument used to indicate whether to generate output in CSV format.
189      private BooleanArgument csvFormat;
190    
191      // The argument used to indicate whether to suppress information about error
192      // result codes.
193      private BooleanArgument suppressErrors;
194    
195      // The argument used to specify the collection interval.
196      private IntegerArgument collectionInterval;
197    
198      // The argument used to specify the number of search and modify iterations on
199      // a connection before it is closed and re-established.
200      private IntegerArgument iterationsBeforeReconnect;
201    
202      // The argument used to specify the number of intervals.
203      private IntegerArgument numIntervals;
204    
205      // The argument used to specify the number of threads.
206      private IntegerArgument numThreads;
207    
208      // The argument used to specify the seed to use for the random number
209      // generator.
210      private IntegerArgument randomSeed;
211    
212      // The target rate of operations per second.
213      private IntegerArgument ratePerSecond;
214    
215      // The argument used to specify a variable rate file.
216      private FileArgument sampleRateFile;
217    
218      // The argument used to specify a variable rate file.
219      private FileArgument variableRateData;
220    
221      // The argument used to specify the length of the values to generate.
222      private IntegerArgument valueLength;
223    
224      // The number of warm-up intervals to perform.
225      private IntegerArgument warmUpIntervals;
226    
227      // The argument used to specify the scope for the searches.
228      private ScopeArgument scopeArg;
229    
230      // The argument used to specify the base DNs for the searches.
231      private StringArgument baseDN;
232    
233      // The argument used to specify the set of characters to use when generating
234      // values.
235      private StringArgument characterSet;
236    
237      // The argument used to specify the filters for the searches.
238      private StringArgument filter;
239    
240      // The argument used to specify the attributes to modify.
241      private StringArgument modifyAttributes;
242    
243      // The argument used to specify the proxied authorization identity.
244      private StringArgument proxyAs;
245    
246      // The argument used to specify the attributes to return.
247      private StringArgument returnAttributes;
248    
249      // The argument used to specify the timestamp format.
250      private StringArgument timestampFormat;
251    
252      // The thread currently being used to run the searchrate tool.
253      private volatile Thread runningThread;
254    
255      // A wakeable sleeper that will be used to sleep between reporting intervals.
256      private final WakeableSleeper sleeper;
257    
258    
259    
260      /**
261       * Parse the provided command line arguments and make the appropriate set of
262       * changes.
263       *
264       * @param  args  The command line arguments provided to this program.
265       */
266      public static void main(final String[] args)
267      {
268        final ResultCode resultCode = main(args, System.out, System.err);
269        if (resultCode != ResultCode.SUCCESS)
270        {
271          System.exit(resultCode.intValue());
272        }
273      }
274    
275    
276    
277      /**
278       * Parse the provided command line arguments and make the appropriate set of
279       * changes.
280       *
281       * @param  args       The command line arguments provided to this program.
282       * @param  outStream  The output stream to which standard out should be
283       *                    written.  It may be {@code null} if output should be
284       *                    suppressed.
285       * @param  errStream  The output stream to which standard error should be
286       *                    written.  It may be {@code null} if error messages
287       *                    should be suppressed.
288       *
289       * @return  A result code indicating whether the processing was successful.
290       */
291      public static ResultCode main(final String[] args,
292                                    final OutputStream outStream,
293                                    final OutputStream errStream)
294      {
295        final SearchAndModRate searchAndModRate =
296             new SearchAndModRate(outStream, errStream);
297        return searchAndModRate.runTool(args);
298      }
299    
300    
301    
302      /**
303       * Creates a new instance of this tool.
304       *
305       * @param  outStream  The output stream to which standard out should be
306       *                    written.  It may be {@code null} if output should be
307       *                    suppressed.
308       * @param  errStream  The output stream to which standard error should be
309       *                    written.  It may be {@code null} if error messages
310       *                    should be suppressed.
311       */
312      public SearchAndModRate(final OutputStream outStream,
313                              final OutputStream errStream)
314      {
315        super(outStream, errStream);
316    
317        stopRequested = new AtomicBoolean(false);
318        sleeper = new WakeableSleeper();
319      }
320    
321    
322    
323      /**
324       * Retrieves the name for this tool.
325       *
326       * @return  The name for this tool.
327       */
328      @Override()
329      public String getToolName()
330      {
331        return "search-and-mod-rate";
332      }
333    
334    
335    
336      /**
337       * Retrieves the description for this tool.
338       *
339       * @return  The description for this tool.
340       */
341      @Override()
342      public String getToolDescription()
343      {
344        return "Perform repeated searches against an " +
345               "LDAP directory server and modify each entry returned.";
346      }
347    
348    
349    
350      /**
351       * Retrieves the version string for this tool.
352       *
353       * @return  The version string for this tool.
354       */
355      @Override()
356      public String getToolVersion()
357      {
358        return Version.NUMERIC_VERSION_STRING;
359      }
360    
361    
362    
363      /**
364       * Indicates whether this tool should provide support for an interactive mode,
365       * in which the tool offers a mode in which the arguments can be provided in
366       * a text-driven menu rather than requiring them to be given on the command
367       * line.  If interactive mode is supported, it may be invoked using the
368       * "--interactive" argument.  Alternately, if interactive mode is supported
369       * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
370       * interactive mode may be invoked by simply launching the tool without any
371       * arguments.
372       *
373       * @return  {@code true} if this tool supports interactive mode, or
374       *          {@code false} if not.
375       */
376      @Override()
377      public boolean supportsInteractiveMode()
378      {
379        return true;
380      }
381    
382    
383    
384      /**
385       * Indicates whether this tool defaults to launching in interactive mode if
386       * the tool is invoked without any command-line arguments.  This will only be
387       * used if {@link #supportsInteractiveMode()} returns {@code true}.
388       *
389       * @return  {@code true} if this tool defaults to using interactive mode if
390       *          launched without any command-line arguments, or {@code false} if
391       *          not.
392       */
393      @Override()
394      public boolean defaultsToInteractiveMode()
395      {
396        return true;
397      }
398    
399    
400    
401      /**
402       * Indicates whether this tool should provide arguments for redirecting output
403       * to a file.  If this method returns {@code true}, then the tool will offer
404       * an "--outputFile" argument that will specify the path to a file to which
405       * all standard output and standard error content will be written, and it will
406       * also offer a "--teeToStandardOut" argument that can only be used if the
407       * "--outputFile" argument is present and will cause all output to be written
408       * to both the specified output file and to standard output.
409       *
410       * @return  {@code true} if this tool should provide arguments for redirecting
411       *          output to a file, or {@code false} if not.
412       */
413      @Override()
414      protected boolean supportsOutputFile()
415      {
416        return true;
417      }
418    
419    
420    
421      /**
422       * Indicates whether this tool should default to interactively prompting for
423       * the bind password if a password is required but no argument was provided
424       * to indicate how to get the password.
425       *
426       * @return  {@code true} if this tool should default to interactively
427       *          prompting for the bind password, or {@code false} if not.
428       */
429      @Override()
430      protected boolean defaultToPromptForBindPassword()
431      {
432        return true;
433      }
434    
435    
436    
437      /**
438       * Indicates whether this tool supports the use of a properties file for
439       * specifying default values for arguments that aren't specified on the
440       * command line.
441       *
442       * @return  {@code true} if this tool supports the use of a properties file
443       *          for specifying default values for arguments that aren't specified
444       *          on the command line, or {@code false} if not.
445       */
446      @Override()
447      public boolean supportsPropertiesFile()
448      {
449        return true;
450      }
451    
452    
453    
454      /**
455       * Indicates whether the LDAP-specific arguments should include alternate
456       * versions of all long identifiers that consist of multiple words so that
457       * they are available in both camelCase and dash-separated versions.
458       *
459       * @return  {@code true} if this tool should provide multiple versions of
460       *          long identifiers for LDAP-specific arguments, or {@code false} if
461       *          not.
462       */
463      @Override()
464      protected boolean includeAlternateLongIdentifiers()
465      {
466        return true;
467      }
468    
469    
470    
471      /**
472       * Adds the arguments used by this program that aren't already provided by the
473       * generic {@code LDAPCommandLineTool} framework.
474       *
475       * @param  parser  The argument parser to which the arguments should be added.
476       *
477       * @throws  ArgumentException  If a problem occurs while adding the arguments.
478       */
479      @Override()
480      public void addNonLDAPArguments(final ArgumentParser parser)
481             throws ArgumentException
482      {
483        String description = "The base DN to use for the searches.  It may be a " +
484             "simple DN or a value pattern to specify a range of DNs (e.g., " +
485             "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  See " +
486             ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
487             "value pattern syntax.  This must be provided.";
488        baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
489        baseDN.setArgumentGroupName("Search And Modification Arguments");
490        baseDN.addLongIdentifier("base-dn");
491        parser.addArgument(baseDN);
492    
493    
494        description = "The scope to use for the searches.  It should be 'base', " +
495                      "'one', 'sub', or 'subord'.  If this is not provided, then " +
496                      "a default scope of 'sub' will be used.";
497        scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
498                                     SearchScope.SUB);
499        scopeArg.setArgumentGroupName("Search And Modification Arguments");
500        parser.addArgument(scopeArg);
501    
502    
503        description = "The filter to use for the searches.  It may be a simple " +
504                      "filter or a value pattern to specify a range of filters " +
505                      "(e.g., \"(uid=user.[1-1000])\").  See " +
506                      ValuePattern.PUBLIC_JAVADOC_URL + " for complete details " +
507                      "about the value pattern syntax.  This must be provided.";
508        filter = new StringArgument('f', "filter", true, 1, "{filter}",
509                                    description);
510        filter.setArgumentGroupName("Search And Modification Arguments");
511        parser.addArgument(filter);
512    
513    
514        description = "The name of an attribute to include in entries returned " +
515                      "from the searches.  Multiple attributes may be requested " +
516                      "by providing this argument multiple times.  If no request " +
517                      "attributes are provided, then the entries returned will " +
518                      "include all user attributes.";
519        returnAttributes = new StringArgument('A', "attribute", false, 0, "{name}",
520                                              description);
521        returnAttributes.setArgumentGroupName("Search And Modification Arguments");
522        parser.addArgument(returnAttributes);
523    
524    
525        description = "The name of the attribute to modify.  Multiple attributes " +
526                      "may be specified by providing this argument multiple " +
527                      "times.  At least one attribute must be specified.";
528        modifyAttributes = new StringArgument('m', "modifyAttribute", true, 0,
529                                              "{name}", description);
530        modifyAttributes.setArgumentGroupName("Search And Modification Arguments");
531        modifyAttributes.addLongIdentifier("modify-attribute");
532        parser.addArgument(modifyAttributes);
533    
534    
535        description = "The length in bytes to use when generating values for the " +
536                      "modifications.  If this is not provided, then a default " +
537                      "length of ten bytes will be used.";
538        valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
539                                          description, 1, Integer.MAX_VALUE, 10);
540        valueLength.setArgumentGroupName("Search And Modification Arguments");
541        valueLength.addLongIdentifier("value-length");
542        parser.addArgument(valueLength);
543    
544    
545        description = "The set of characters to use to generate the values for " +
546                      "the modifications.  It should only include ASCII " +
547                      "characters.  If this is not provided, then a default set " +
548                      "of lowercase alphabetic characters will be used.";
549        characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
550                                          description,
551                                          "abcdefghijklmnopqrstuvwxyz");
552        characterSet.setArgumentGroupName("Search And Modification Arguments");
553        characterSet.addLongIdentifier("character-set");
554        parser.addArgument(characterSet);
555    
556    
557        description = "The number of threads to use to perform the searches.  If " +
558                      "this is not provided, then a default of one thread will " +
559                      "be used.";
560        numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
561                                         description, 1, Integer.MAX_VALUE, 1);
562        numThreads.setArgumentGroupName("Rate Management Arguments");
563        numThreads.addLongIdentifier("num-threads");
564        parser.addArgument(numThreads);
565    
566    
567        description = "The length of time in seconds between output lines.  If " +
568                      "this is not provided, then a default interval of five " +
569                      "seconds will be used.";
570        collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
571                                                 "{num}", description, 1,
572                                                 Integer.MAX_VALUE, 5);
573        collectionInterval.setArgumentGroupName("Rate Management Arguments");
574        collectionInterval.addLongIdentifier("interval-duration");
575        parser.addArgument(collectionInterval);
576    
577    
578        description = "The maximum number of intervals for which to run.  If " +
579                      "this is not provided, then the tool will run until it is " +
580                      "interrupted.";
581        numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
582                                           description, 1, Integer.MAX_VALUE,
583                                           Integer.MAX_VALUE);
584        numIntervals.setArgumentGroupName("Rate Management Arguments");
585        numIntervals.addLongIdentifier("num-intervals");
586        parser.addArgument(numIntervals);
587    
588        description = "The number of search and modify iterations that should be " +
589                      "processed on a connection before that connection is " +
590                      "closed and replaced with a newly-established (and " +
591                      "authenticated, if appropriate) connection.  If this is " +
592                      "not provided, then connections will not be periodically " +
593                      "closed and re-established.";
594        iterationsBeforeReconnect = new IntegerArgument(null,
595             "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
596        iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
597        iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect");
598        parser.addArgument(iterationsBeforeReconnect);
599    
600        description = "The target number of searches to perform per second.  It " +
601                      "is still necessary to specify a sufficient number of " +
602                      "threads for achieving this rate.  If neither this option " +
603                      "nor --variableRateData is provided, then the tool will " +
604                      "run at the maximum rate for the specified number of " +
605                      "threads.";
606        ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
607                                            "{searches-per-second}", description,
608                                            1, Integer.MAX_VALUE);
609        ratePerSecond.setArgumentGroupName("Rate Management Arguments");
610        ratePerSecond.addLongIdentifier("rate-per-second");
611        parser.addArgument(ratePerSecond);
612    
613        final String variableRateDataArgName = "variableRateData";
614        final String generateSampleRateFileArgName = "generateSampleRateFile";
615        description = RateAdjustor.getVariableRateDataArgumentDescription(
616             generateSampleRateFileArgName);
617        variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
618                                            "{path}", description, true, true, true,
619                                            false);
620        variableRateData.setArgumentGroupName("Rate Management Arguments");
621        variableRateData.addLongIdentifier("variable-rate-data");
622        parser.addArgument(variableRateData);
623    
624        description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
625             variableRateDataArgName);
626        sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
627                                          false, 1, "{path}", description, false,
628                                          true, true, false);
629        sampleRateFile.setArgumentGroupName("Rate Management Arguments");
630        sampleRateFile.addLongIdentifier("generate-sample-rate-file");
631        sampleRateFile.setUsageArgument(true);
632        parser.addArgument(sampleRateFile);
633        parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
634    
635        description = "The number of intervals to complete before beginning " +
636                      "overall statistics collection.  Specifying a nonzero " +
637                      "number of warm-up intervals gives the client and server " +
638                      "a chance to warm up without skewing performance results.";
639        warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
640             "{num}", description, 0, Integer.MAX_VALUE, 0);
641        warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
642        warmUpIntervals.addLongIdentifier("warm-up-intervals");
643        parser.addArgument(warmUpIntervals);
644    
645        description = "Indicates the format to use for timestamps included in " +
646                      "the output.  A value of 'none' indicates that no " +
647                      "timestamps should be included.  A value of 'with-date' " +
648                      "indicates that both the date and the time should be " +
649                      "included.  A value of 'without-date' indicates that only " +
650                      "the time should be included.";
651        final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
652        allowedFormats.add("none");
653        allowedFormats.add("with-date");
654        allowedFormats.add("without-date");
655        timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
656             "{format}", description, allowedFormats, "none");
657        timestampFormat.addLongIdentifier("timestamp-format");
658        parser.addArgument(timestampFormat);
659    
660        description = "Indicates that the proxied authorization control (as " +
661                      "defined in RFC 4370) should be used to request that " +
662                      "operations be processed using an alternate authorization " +
663                      "identity.  This may be a simple authorization ID or it " +
664                      "may be a value pattern to specify a range of " +
665                      "identities.  See " + ValuePattern.PUBLIC_JAVADOC_URL +
666                      " for complete details about the value pattern syntax.";
667        proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
668                                     description);
669        proxyAs.addLongIdentifier("proxy-as");
670        parser.addArgument(proxyAs);
671    
672        description = "Indicates that information about the result codes for " +
673                      "failed operations should not be displayed.";
674        suppressErrors = new BooleanArgument(null,
675             "suppressErrorResultCodes", 1, description);
676        suppressErrors.addLongIdentifier("suppress-error-result-codes");
677        parser.addArgument(suppressErrors);
678    
679        description = "Generate output in CSV format rather than a " +
680                      "display-friendly format";
681        csvFormat = new BooleanArgument('c', "csv", 1, description);
682        parser.addArgument(csvFormat);
683    
684        description = "Specifies the seed to use for the random number generator.";
685        randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
686             description);
687        randomSeed.addLongIdentifier("random-seed");
688        parser.addArgument(randomSeed);
689      }
690    
691    
692    
693      /**
694       * Indicates whether this tool supports creating connections to multiple
695       * servers.  If it is to support multiple servers, then the "--hostname" and
696       * "--port" arguments will be allowed to be provided multiple times, and
697       * will be required to be provided the same number of times.  The same type of
698       * communication security and bind credentials will be used for all servers.
699       *
700       * @return  {@code true} if this tool supports creating connections to
701       *          multiple servers, or {@code false} if not.
702       */
703      @Override()
704      protected boolean supportsMultipleServers()
705      {
706        return true;
707      }
708    
709    
710    
711      /**
712       * Retrieves the connection options that should be used for connections
713       * created for use with this tool.
714       *
715       * @return  The connection options that should be used for connections created
716       *          for use with this tool.
717       */
718      @Override()
719      public LDAPConnectionOptions getConnectionOptions()
720      {
721        final LDAPConnectionOptions options = new LDAPConnectionOptions();
722        options.setUseSynchronousMode(true);
723        return options;
724      }
725    
726    
727    
728      /**
729       * Performs the actual processing for this tool.  In this case, it gets a
730       * connection to the directory server and uses it to perform the requested
731       * searches.
732       *
733       * @return  The result code for the processing that was performed.
734       */
735      @Override()
736      public ResultCode doToolProcessing()
737      {
738        runningThread = Thread.currentThread();
739    
740        try
741        {
742          return doToolProcessingInternal();
743        }
744        finally
745        {
746          runningThread = null;
747        }
748      }
749    
750    
751    
752      /**
753       * Performs the actual processing for this tool.  In this case, it gets a
754       * connection to the directory server and uses it to perform the requested
755       * searches.
756       *
757       * @return  The result code for the processing that was performed.
758       */
759      private ResultCode doToolProcessingInternal()
760      {
761        // If the sample rate file argument was specified, then generate the sample
762        // variable rate data file and return.
763        if (sampleRateFile.isPresent())
764        {
765          try
766          {
767            RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
768            return ResultCode.SUCCESS;
769          }
770          catch (final Exception e)
771          {
772            debugException(e);
773            err("An error occurred while trying to write sample variable data " +
774                 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
775                 "':  ", getExceptionMessage(e));
776            return ResultCode.LOCAL_ERROR;
777          }
778        }
779    
780    
781        // Determine the random seed to use.
782        final Long seed;
783        if (randomSeed.isPresent())
784        {
785          seed = Long.valueOf(randomSeed.getValue());
786        }
787        else
788        {
789          seed = null;
790        }
791    
792        // Create value patterns for the base DN, filter, and proxied authorization
793        // DN.
794        final ValuePattern dnPattern;
795        try
796        {
797          dnPattern = new ValuePattern(baseDN.getValue(), seed);
798        }
799        catch (final ParseException pe)
800        {
801          debugException(pe);
802          err("Unable to parse the base DN value pattern:  ", pe.getMessage());
803          return ResultCode.PARAM_ERROR;
804        }
805    
806        final ValuePattern filterPattern;
807        try
808        {
809          filterPattern = new ValuePattern(filter.getValue(), seed);
810        }
811        catch (final ParseException pe)
812        {
813          debugException(pe);
814          err("Unable to parse the filter pattern:  ", pe.getMessage());
815          return ResultCode.PARAM_ERROR;
816        }
817    
818        final ValuePattern authzIDPattern;
819        if (proxyAs.isPresent())
820        {
821          try
822          {
823            authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
824          }
825          catch (final ParseException pe)
826          {
827            debugException(pe);
828            err("Unable to parse the proxied authorization pattern:  ",
829                pe.getMessage());
830            return ResultCode.PARAM_ERROR;
831          }
832        }
833        else
834        {
835          authzIDPattern = null;
836        }
837    
838    
839        // Get the attributes to return.
840        final String[] returnAttrs;
841        if (returnAttributes.isPresent())
842        {
843          final List<String> attrList = returnAttributes.getValues();
844          returnAttrs = new String[attrList.size()];
845          attrList.toArray(returnAttrs);
846        }
847        else
848        {
849          returnAttrs = NO_STRINGS;
850        }
851    
852    
853        // Get the names of the attributes to modify.
854        final String[] modAttrs = new String[modifyAttributes.getValues().size()];
855        modifyAttributes.getValues().toArray(modAttrs);
856    
857    
858        // Get the character set as a byte array.
859        final byte[] charSet = getBytes(characterSet.getValue());
860    
861    
862        // If the --ratePerSecond option was specified, then limit the rate
863        // accordingly.
864        FixedRateBarrier fixedRateBarrier = null;
865        if (ratePerSecond.isPresent() || variableRateData.isPresent())
866        {
867          // We might not have a rate per second if --variableRateData is specified.
868          // The rate typically doesn't matter except when we have warm-up
869          // intervals.  In this case, we'll run at the max rate.
870          final int intervalSeconds = collectionInterval.getValue();
871          final int ratePerInterval =
872               (ratePerSecond.getValue() == null)
873               ? Integer.MAX_VALUE
874               : ratePerSecond.getValue() * intervalSeconds;
875          fixedRateBarrier =
876               new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
877        }
878    
879    
880        // If --variableRateData was specified, then initialize a RateAdjustor.
881        RateAdjustor rateAdjustor = null;
882        if (variableRateData.isPresent())
883        {
884          try
885          {
886            rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
887                 ratePerSecond.getValue(), variableRateData.getValue());
888          }
889          catch (final IOException e)
890          {
891            debugException(e);
892            err("Initializing the variable rates failed: " + e.getMessage());
893            return ResultCode.PARAM_ERROR;
894          }
895          catch (final IllegalArgumentException e)
896          {
897            debugException(e);
898            err("Initializing the variable rates failed: " + e.getMessage());
899            return ResultCode.PARAM_ERROR;
900          }
901        }
902    
903    
904        // Determine whether to include timestamps in the output and if so what
905        // format should be used for them.
906        final boolean includeTimestamp;
907        final String timeFormat;
908        if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
909        {
910          includeTimestamp = true;
911          timeFormat       = "dd/MM/yyyy HH:mm:ss";
912        }
913        else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
914        {
915          includeTimestamp = true;
916          timeFormat       = "HH:mm:ss";
917        }
918        else
919        {
920          includeTimestamp = false;
921          timeFormat       = null;
922        }
923    
924    
925        // Determine whether any warm-up intervals should be run.
926        final long totalIntervals;
927        final boolean warmUp;
928        int remainingWarmUpIntervals = warmUpIntervals.getValue();
929        if (remainingWarmUpIntervals > 0)
930        {
931          warmUp = true;
932          totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
933        }
934        else
935        {
936          warmUp = true;
937          totalIntervals = 0L + numIntervals.getValue();
938        }
939    
940    
941        // Create the table that will be used to format the output.
942        final OutputFormat outputFormat;
943        if (csvFormat.isPresent())
944        {
945          outputFormat = OutputFormat.CSV;
946        }
947        else
948        {
949          outputFormat = OutputFormat.COLUMNS;
950        }
951    
952        final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
953             timeFormat, outputFormat, " ",
954             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
955                      "Searches/Sec"),
956             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
957                      "Srch Dur ms"),
958             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
959                      "Mods/Sec"),
960             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
961                      "Mod Dur ms"),
962             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
963                      "Errors/Sec"),
964             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
965                      "Searches/Sec"),
966             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
967                      "Srch Dur ms"),
968             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
969                      "Mods/Sec"),
970             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
971                      "Mod Dur ms"));
972    
973    
974        // Create values to use for statistics collection.
975        final AtomicLong        searchCounter   = new AtomicLong(0L);
976        final AtomicLong        errorCounter    = new AtomicLong(0L);
977        final AtomicLong        modCounter      = new AtomicLong(0L);
978        final AtomicLong        modDurations    = new AtomicLong(0L);
979        final AtomicLong        searchDurations = new AtomicLong(0L);
980        final ResultCodeCounter rcCounter       = new ResultCodeCounter();
981    
982    
983        // Determine the length of each interval in milliseconds.
984        final long intervalMillis = 1000L * collectionInterval.getValue();
985    
986    
987        // Create the threads to use for the searches.
988        final Random random = new Random();
989        final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
990        final SearchAndModRateThread[] threads =
991             new SearchAndModRateThread[numThreads.getValue()];
992        for (int i=0; i < threads.length; i++)
993        {
994          final LDAPConnection connection;
995          try
996          {
997            connection = getConnection();
998          }
999          catch (final LDAPException le)
1000          {
1001            debugException(le);
1002            err("Unable to connect to the directory server:  ",
1003                getExceptionMessage(le));
1004            return le.getResultCode();
1005          }
1006    
1007          threads[i] = new SearchAndModRateThread(this, i, connection, dnPattern,
1008               scopeArg.getValue(), filterPattern, returnAttrs, modAttrs,
1009               valueLength.getValue(), charSet, authzIDPattern,
1010               iterationsBeforeReconnect.getValue(), random.nextLong(), barrier,
1011               searchCounter, modCounter, searchDurations, modDurations,
1012               errorCounter, rcCounter, fixedRateBarrier);
1013          threads[i].start();
1014        }
1015    
1016    
1017        // Display the table header.
1018        for (final String headerLine : formatter.getHeaderLines(true))
1019        {
1020          out(headerLine);
1021        }
1022    
1023    
1024        // Start the RateAdjustor before the threads so that the initial value is
1025        // in place before any load is generated unless we're doing a warm-up in
1026        // which case, we'll start it after the warm-up is complete.
1027        if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1028        {
1029          rateAdjustor.start();
1030        }
1031    
1032    
1033        // Indicate that the threads can start running.
1034        try
1035        {
1036          barrier.await();
1037        }
1038        catch (final Exception e)
1039        {
1040          debugException(e);
1041        }
1042    
1043        long overallStartTime = System.nanoTime();
1044        long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1045    
1046    
1047        boolean setOverallStartTime = false;
1048        long    lastSearchDuration  = 0L;
1049        long    lastModDuration     = 0L;
1050        long    lastNumErrors       = 0L;
1051        long    lastNumSearches     = 0L;
1052        long    lastNumMods          = 0L;
1053        long    lastEndTime         = System.nanoTime();
1054        for (long i=0; i < totalIntervals; i++)
1055        {
1056          if (rateAdjustor != null)
1057          {
1058            if (! rateAdjustor.isAlive())
1059            {
1060              out("All of the rates in " + variableRateData.getValue().getName() +
1061                  " have been completed.");
1062              break;
1063            }
1064          }
1065    
1066          final long startTimeMillis = System.currentTimeMillis();
1067          final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1068          nextIntervalStartTime += intervalMillis;
1069          if (sleepTimeMillis > 0)
1070          {
1071            sleeper.sleep(sleepTimeMillis);
1072          }
1073    
1074          if (stopRequested.get())
1075          {
1076            break;
1077          }
1078    
1079          final long endTime          = System.nanoTime();
1080          final long intervalDuration = endTime - lastEndTime;
1081    
1082          final long numSearches;
1083          final long numMods;
1084          final long numErrors;
1085          final long totalSearchDuration;
1086          final long totalModDuration;
1087          if (warmUp && (remainingWarmUpIntervals > 0))
1088          {
1089            numSearches         = searchCounter.getAndSet(0L);
1090            numMods             = modCounter.getAndSet(0L);
1091            numErrors           = errorCounter.getAndSet(0L);
1092            totalSearchDuration = searchDurations.getAndSet(0L);
1093            totalModDuration    = modDurations.getAndSet(0L);
1094          }
1095          else
1096          {
1097            numSearches         = searchCounter.get();
1098            numMods             = modCounter.get();
1099            numErrors           = errorCounter.get();
1100            totalSearchDuration = searchDurations.get();
1101            totalModDuration    = modDurations.get();
1102          }
1103    
1104          final long recentNumSearches = numSearches - lastNumSearches;
1105          final long recentNumMods = numMods - lastNumMods;
1106          final long recentNumErrors = numErrors - lastNumErrors;
1107          final long recentSearchDuration =
1108               totalSearchDuration - lastSearchDuration;
1109          final long recentModDuration = totalModDuration - lastModDuration;
1110    
1111          final double numSeconds = intervalDuration / 1000000000.0d;
1112          final double recentSearchRate = recentNumSearches / numSeconds;
1113          final double recentModRate = recentNumMods / numSeconds;
1114          final double recentErrorRate  = recentNumErrors / numSeconds;
1115    
1116          final double recentAvgSearchDuration;
1117          if (recentNumSearches > 0L)
1118          {
1119            recentAvgSearchDuration =
1120                 1.0d * recentSearchDuration / recentNumSearches / 1000000;
1121          }
1122          else
1123          {
1124            recentAvgSearchDuration = 0.0d;
1125          }
1126    
1127          final double recentAvgModDuration;
1128          if (recentNumMods > 0L)
1129          {
1130            recentAvgModDuration =
1131                 1.0d * recentModDuration / recentNumMods / 1000000;
1132          }
1133          else
1134          {
1135            recentAvgModDuration = 0.0d;
1136          }
1137    
1138          if (warmUp && (remainingWarmUpIntervals > 0))
1139          {
1140            out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1141                 recentModRate, recentAvgModDuration, recentErrorRate, "warming up",
1142                 "warming up", "warming up", "warming up"));
1143    
1144            remainingWarmUpIntervals--;
1145            if (remainingWarmUpIntervals == 0)
1146            {
1147              out("Warm-up completed.  Beginning overall statistics collection.");
1148              setOverallStartTime = true;
1149              if (rateAdjustor != null)
1150              {
1151                rateAdjustor.start();
1152              }
1153            }
1154          }
1155          else
1156          {
1157            if (setOverallStartTime)
1158            {
1159              overallStartTime    = lastEndTime;
1160              setOverallStartTime = false;
1161            }
1162    
1163            final double numOverallSeconds =
1164                 (endTime - overallStartTime) / 1000000000.0d;
1165            final double overallSearchRate = numSearches / numOverallSeconds;
1166            final double overallModRate = numMods / numOverallSeconds;
1167    
1168            final double overallAvgSearchDuration;
1169            if (numSearches > 0L)
1170            {
1171              overallAvgSearchDuration =
1172                   1.0d * totalSearchDuration / numSearches / 1000000;
1173            }
1174            else
1175            {
1176              overallAvgSearchDuration = 0.0d;
1177            }
1178    
1179            final double overallAvgModDuration;
1180            if (numMods > 0L)
1181            {
1182              overallAvgModDuration =
1183                   1.0d * totalModDuration / numMods / 1000000;
1184            }
1185            else
1186            {
1187              overallAvgModDuration = 0.0d;
1188            }
1189    
1190            out(formatter.formatRow(recentSearchRate, recentAvgSearchDuration,
1191                 recentModRate, recentAvgModDuration, recentErrorRate,
1192                 overallSearchRate, overallAvgSearchDuration, overallModRate,
1193                 overallAvgModDuration));
1194    
1195            lastNumSearches    = numSearches;
1196            lastNumMods        = numMods;
1197            lastNumErrors      = numErrors;
1198            lastSearchDuration = totalSearchDuration;
1199            lastModDuration    = totalModDuration;
1200          }
1201    
1202          final List<ObjectPair<ResultCode,Long>> rcCounts =
1203               rcCounter.getCounts(true);
1204          if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1205          {
1206            err("\tError Results:");
1207            for (final ObjectPair<ResultCode,Long> p : rcCounts)
1208            {
1209              err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1210            }
1211          }
1212    
1213          lastEndTime = endTime;
1214        }
1215    
1216    
1217        // Shut down the RateAdjustor if we have one.
1218        if (rateAdjustor != null)
1219        {
1220          rateAdjustor.shutDown();
1221        }
1222    
1223        // Stop all of the threads.
1224        ResultCode resultCode = ResultCode.SUCCESS;
1225        for (final SearchAndModRateThread t : threads)
1226        {
1227          final ResultCode r = t.stopRunning();
1228          if (resultCode == ResultCode.SUCCESS)
1229          {
1230            resultCode = r;
1231          }
1232        }
1233    
1234        return resultCode;
1235      }
1236    
1237    
1238    
1239      /**
1240       * Requests that this tool stop running.  This method will attempt to wait
1241       * for all threads to complete before returning control to the caller.
1242       */
1243      public void stopRunning()
1244      {
1245        stopRequested.set(true);
1246        sleeper.wakeup();
1247    
1248        final Thread t = runningThread;
1249        if (t != null)
1250        {
1251          try
1252          {
1253            t.join();
1254          }
1255          catch (final Exception e)
1256          {
1257            debugException(e);
1258          }
1259        }
1260      }
1261    
1262    
1263    
1264      /**
1265       * {@inheritDoc}
1266       */
1267      @Override()
1268      public LinkedHashMap<String[],String> getExampleUsages()
1269      {
1270        final LinkedHashMap<String[],String> examples =
1271             new LinkedHashMap<String[],String>(2);
1272    
1273        String[] args =
1274        {
1275          "--hostname", "server.example.com",
1276          "--port", "389",
1277          "--bindDN", "uid=admin,dc=example,dc=com",
1278          "--bindPassword", "password",
1279          "--baseDN", "dc=example,dc=com",
1280          "--scope", "sub",
1281          "--filter", "(uid=user.[1-1000000])",
1282          "--attribute", "givenName",
1283          "--attribute", "sn",
1284          "--attribute", "mail",
1285          "--modifyAttribute", "description",
1286          "--valueLength", "10",
1287          "--characterSet", "abcdefghijklmnopqrstuvwxyz0123456789",
1288          "--numThreads", "10"
1289        };
1290        String description =
1291             "Test search and modify performance by searching randomly across a " +
1292             "set of one million users located below 'dc=example,dc=com' with " +
1293             "ten concurrent threads.  The entries returned to the client will " +
1294             "include the givenName, sn, and mail attributes, and the " +
1295             "description attribute of each entry returned will be replaced " +
1296             "with a string of ten randomly-selected alphanumeric characters.";
1297        examples.put(args, description);
1298    
1299        args = new String[]
1300        {
1301          "--generateSampleRateFile", "variable-rate-data.txt"
1302        };
1303        description =
1304             "Generate a sample variable rate definition file that may be used " +
1305             "in conjunction with the --variableRateData argument.  The sample " +
1306             "file will include comments that describe the format for data to be " +
1307             "included in this file.";
1308        examples.put(args, description);
1309    
1310        return examples;
1311      }
1312    }