001    /*
002     * Copyright 2008-2015 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2008-2015 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.IOException;
026    import java.io.OutputStream;
027    import java.io.Serializable;
028    import java.text.ParseException;
029    import java.util.LinkedHashMap;
030    import java.util.LinkedHashSet;
031    import java.util.List;
032    import java.util.concurrent.CyclicBarrier;
033    import java.util.concurrent.Semaphore;
034    import java.util.concurrent.atomic.AtomicBoolean;
035    import java.util.concurrent.atomic.AtomicLong;
036    
037    import com.unboundid.ldap.sdk.LDAPConnection;
038    import com.unboundid.ldap.sdk.LDAPConnectionOptions;
039    import com.unboundid.ldap.sdk.LDAPException;
040    import com.unboundid.ldap.sdk.ResultCode;
041    import com.unboundid.ldap.sdk.SearchScope;
042    import com.unboundid.ldap.sdk.Version;
043    import com.unboundid.util.ColumnFormatter;
044    import com.unboundid.util.FixedRateBarrier;
045    import com.unboundid.util.FormattableColumn;
046    import com.unboundid.util.HorizontalAlignment;
047    import com.unboundid.util.LDAPCommandLineTool;
048    import com.unboundid.util.ObjectPair;
049    import com.unboundid.util.OutputFormat;
050    import com.unboundid.util.RateAdjustor;
051    import com.unboundid.util.ResultCodeCounter;
052    import com.unboundid.util.ThreadSafety;
053    import com.unboundid.util.ThreadSafetyLevel;
054    import com.unboundid.util.WakeableSleeper;
055    import com.unboundid.util.ValuePattern;
056    import com.unboundid.util.args.ArgumentException;
057    import com.unboundid.util.args.ArgumentParser;
058    import com.unboundid.util.args.BooleanArgument;
059    import com.unboundid.util.args.FileArgument;
060    import com.unboundid.util.args.IntegerArgument;
061    import com.unboundid.util.args.ScopeArgument;
062    import com.unboundid.util.args.StringArgument;
063    
064    import static com.unboundid.util.Debug.*;
065    import static com.unboundid.util.StaticUtils.*;
066    
067    
068    
069    /**
070     * This class provides a tool that can be used to search an LDAP directory
071     * server repeatedly using multiple threads.  It can help provide an estimate of
072     * the search performance that a directory server is able to achieve.  Either or
073     * both of the base DN and the search filter may be a value pattern as
074     * described in the {@link ValuePattern} class.  This makes it possible to
075     * search over a range of entries rather than repeatedly performing searches
076     * with the same base DN and filter.
077     * <BR><BR>
078     * Some of the APIs demonstrated by this example include:
079     * <UL>
080     *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
081     *       package)</LI>
082     *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
083     *       package)</LI>
084     *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
085     *       package)</LI>
086     *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
087     * </UL>
088     * <BR><BR>
089     * All of the necessary information is provided using command line arguments.
090     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
091     * class, as well as the following additional arguments:
092     * <UL>
093     *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
094     *       for the searches.  This must be provided.  It may be a simple DN, or it
095     *       may be a value pattern to express a range of base DNs.</LI>
096     *   <LI>"-s {scope}" or "--scope {scope}" -- specifies the scope to use for the
097     *       search.  The scope value should be one of "base", "one", "sub", or
098     *       "subord".  If this isn't specified, then a scope of "sub" will be
099     *       used.</LI>
100     *   <LI>"-f {filter}" or "--filter {filter}" -- specifies the filter to use for
101     *       the searches.  This must be provided.  It may be a simple filter, or it
102     *       may be a value pattern to express a range of filters.</LI>
103     *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of an
104     *       attribute that should be included in entries returned from the server.
105     *       If this is not provided, then all user attributes will be requested.
106     *       This may include special tokens that the server may interpret, like
107     *       "1.1" to indicate that no attributes should be returned, "*", for all
108     *       user attributes, or "+" for all operational attributes.  Multiple
109     *       attributes may be requested with multiple instances of this
110     *       argument.</LI>
111     *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
112     *       concurrent threads to use when performing the searches.  If this is not
113     *       provided, then a default of one thread will be used.</LI>
114     *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
115     *       time in seconds between lines out output.  If this is not provided,
116     *       then a default interval duration of five seconds will be used.</LI>
117     *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
118     *       intervals for which to run.  If this is not provided, then it will
119     *       run forever.</LI>
120     *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of search
121     *       iterations that should be performed on a connection before that
122     *       connection is closed and replaced with a newly-established (and
123     *       authenticated, if appropriate) connection.</LI>
124     *   <LI>"-r {searches-per-second}" or "--ratePerSecond {searches-per-second}"
125     *       -- specifies the target number of searches to perform per second.  It
126     *       is still necessary to specify a sufficient number of threads for
127     *       achieving this rate.  If this option is not provided, then the tool
128     *       will run at the maximum rate for the specified number of threads.</LI>
129     *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
130     *       information needed to allow the tool to vary the target rate over time.
131     *       If this option is not provided, then the tool will either use a fixed
132     *       target rate as specified by the "--ratePerSecond" argument, or it will
133     *       run at the maximum rate.</LI>
134     *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
135     *       which sample data will be written illustrating and describing the
136     *       format of the file expected to be used in conjunction with the
137     *       "--variableRateData" argument.</LI>
138     *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
139     *       complete before beginning overall statistics collection.</LI>
140     *   <LI>"--timestampFormat {format}" -- specifies the format to use for
141     *       timestamps included before each output line.  The format may be one of
142     *       "none" (for no timestamps), "with-date" (to include both the date and
143     *       the time), or "without-date" (to include only time time).</LI>
144     *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
145     *       authorization v2 control to request that the operation be processed
146     *       using an alternate authorization identity.  In this case, the bind DN
147     *       should be that of a user that has permission to use this control.  The
148     *       authorization identity may be a value pattern.</LI>
149     *   <LI>"-a" or "--asynchronous" -- Indicates that searches should be performed
150     *       in asynchronous mode, in which the client will not wait for a response
151     *       to a previous request before sending the next request.  Either the
152     *       "--ratePerSecond" or "--maxOutstandingRequests" arguments must be
153     *       provided to limit the number of outstanding requests.</LI>
154     *   <LI>"-O {num}" or "--maxOutstandingRequests {num}" -- Specifies the maximum
155     *       number of outstanding requests that will be allowed in asynchronous
156     *       mode.</LI>
157     *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
158     *       result codes for failed operations should not be displayed.</LI>
159     *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
160     *       display-friendly format.</LI>
161     * </UL>
162     */
163    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
164    public final class SearchRate
165           extends LDAPCommandLineTool
166           implements Serializable
167    {
168      /**
169       * The serial version UID for this serializable class.
170       */
171      private static final long serialVersionUID = 3345838530404592182L;
172    
173    
174    
175      // Indicates whether a request has been made to stop running.
176      private final AtomicBoolean stopRequested;
177    
178      // The argument used to indicate whether to operate in asynchronous mode.
179      private BooleanArgument asynchronousMode;
180    
181      // The argument used to indicate whether to generate output in CSV format.
182      private BooleanArgument csvFormat;
183    
184      // The argument used to indicate whether to suppress information about error
185      // result codes.
186      private BooleanArgument suppressErrors;
187    
188      // The argument used to specify the collection interval.
189      private IntegerArgument collectionInterval;
190    
191      // The argument used to specify the number of search iterations on a
192      // connection before it is closed and re-established.
193      private IntegerArgument iterationsBeforeReconnect;
194    
195      // The argument used to specify the maximum number of outstanding asynchronous
196      // requests.
197      private IntegerArgument maxOutstandingRequests;
198    
199      // The argument used to specify the number of intervals.
200      private IntegerArgument numIntervals;
201    
202      // The argument used to specify the number of threads.
203      private IntegerArgument numThreads;
204    
205      // The argument used to specify the seed to use for the random number
206      // generator.
207      private IntegerArgument randomSeed;
208    
209      // The target rate of searches per second.
210      private IntegerArgument ratePerSecond;
211    
212      // The argument used to specify a variable rate file.
213      private FileArgument sampleRateFile;
214    
215      // The argument used to specify a variable rate file.
216      private FileArgument variableRateData;
217    
218      // The number of warm-up intervals to perform.
219      private IntegerArgument warmUpIntervals;
220    
221      // The argument used to specify the scope for the searches.
222      private ScopeArgument scopeArg;
223    
224      // The argument used to specify the attributes to return.
225      private StringArgument attributes;
226    
227      // The argument used to specify the base DNs for the searches.
228      private StringArgument baseDN;
229    
230      // The argument used to specify the filters for the searches.
231      private StringArgument filter;
232    
233      // The argument used to specify the proxied authorization identity.
234      private StringArgument proxyAs;
235    
236      // The argument used to specify the timestamp format.
237      private StringArgument timestampFormat;
238    
239      // The thread currently being used to run the searchrate tool.
240      private volatile Thread runningThread;
241    
242      // A wakeable sleeper that will be used to sleep between reporting intervals.
243      private final WakeableSleeper sleeper;
244    
245    
246    
247      /**
248       * Parse the provided command line arguments and make the appropriate set of
249       * changes.
250       *
251       * @param  args  The command line arguments provided to this program.
252       */
253      public static void main(final String[] args)
254      {
255        final ResultCode resultCode = main(args, System.out, System.err);
256        if (resultCode != ResultCode.SUCCESS)
257        {
258          System.exit(resultCode.intValue());
259        }
260      }
261    
262    
263    
264      /**
265       * Parse the provided command line arguments and make the appropriate set of
266       * changes.
267       *
268       * @param  args       The command line arguments provided to this program.
269       * @param  outStream  The output stream to which standard out should be
270       *                    written.  It may be {@code null} if output should be
271       *                    suppressed.
272       * @param  errStream  The output stream to which standard error should be
273       *                    written.  It may be {@code null} if error messages
274       *                    should be suppressed.
275       *
276       * @return  A result code indicating whether the processing was successful.
277       */
278      public static ResultCode main(final String[] args,
279                                    final OutputStream outStream,
280                                    final OutputStream errStream)
281      {
282        final SearchRate searchRate = new SearchRate(outStream, errStream);
283        return searchRate.runTool(args);
284      }
285    
286    
287    
288      /**
289       * Creates a new instance of this tool.
290       *
291       * @param  outStream  The output stream to which standard out should be
292       *                    written.  It may be {@code null} if output should be
293       *                    suppressed.
294       * @param  errStream  The output stream to which standard error should be
295       *                    written.  It may be {@code null} if error messages
296       *                    should be suppressed.
297       */
298      public SearchRate(final OutputStream outStream, final OutputStream errStream)
299      {
300        super(outStream, errStream);
301    
302        stopRequested = new AtomicBoolean(false);
303        sleeper = new WakeableSleeper();
304      }
305    
306    
307    
308      /**
309       * Retrieves the name for this tool.
310       *
311       * @return  The name for this tool.
312       */
313      @Override()
314      public String getToolName()
315      {
316        return "searchrate";
317      }
318    
319    
320    
321      /**
322       * Retrieves the description for this tool.
323       *
324       * @return  The description for this tool.
325       */
326      @Override()
327      public String getToolDescription()
328      {
329        return "Perform repeated searches against an " +
330               "LDAP directory server.";
331      }
332    
333    
334    
335      /**
336       * Retrieves the version string for this tool.
337       *
338       * @return  The version string for this tool.
339       */
340      @Override()
341      public String getToolVersion()
342      {
343        return Version.NUMERIC_VERSION_STRING;
344      }
345    
346    
347    
348      /**
349       * Adds the arguments used by this program that aren't already provided by the
350       * generic {@code LDAPCommandLineTool} framework.
351       *
352       * @param  parser  The argument parser to which the arguments should be added.
353       *
354       * @throws  ArgumentException  If a problem occurs while adding the arguments.
355       */
356      @Override()
357      public void addNonLDAPArguments(final ArgumentParser parser)
358             throws ArgumentException
359      {
360        String description = "The base DN to use for the searches.  It may be a " +
361             "simple DN or a value pattern to specify a range of DNs (e.g., " +
362             "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  This must be " +
363             "provided.";
364        baseDN = new StringArgument('b', "baseDN", true, 1, "{dn}", description);
365        parser.addArgument(baseDN);
366    
367    
368        description = "The scope to use for the searches.  It should be 'base', " +
369                      "'one', 'sub', or 'subord'.  If this is not provided, then " +
370                      "a default scope of 'sub' will be used.";
371        scopeArg = new ScopeArgument('s', "scope", false, "{scope}", description,
372                                     SearchScope.SUB);
373        parser.addArgument(scopeArg);
374    
375    
376        description = "The filter to use for the searches.  It may be a simple " +
377                      "filter or a value pattern to specify a range of filters " +
378                      "(e.g., \"(uid=user.[1-1000])\").  This must be provided.";
379        filter = new StringArgument('f', "filter", true, 1, "{filter}",
380                                    description);
381        parser.addArgument(filter);
382    
383    
384        description = "The name of an attribute to include in entries returned " +
385                      "from the searches.  Multiple attributes may be requested " +
386                      "by providing this argument multiple times.  If no request " +
387                      "attributes are provided, then the entries returned will " +
388                      "include all user attributes.";
389        attributes = new StringArgument('A', "attribute", false, 0, "{name}",
390                                        description);
391        parser.addArgument(attributes);
392    
393    
394        description = "The number of threads to use to perform the searches.  If " +
395                      "this is not provided, then a default of one thread will " +
396                      "be used.";
397        numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
398                                         description, 1, Integer.MAX_VALUE, 1);
399        parser.addArgument(numThreads);
400    
401    
402        description = "The length of time in seconds between output lines.  If " +
403                      "this is not provided, then a default interval of five " +
404                      "seconds will be used.";
405        collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
406                                                 "{num}", description, 1,
407                                                 Integer.MAX_VALUE, 5);
408        parser.addArgument(collectionInterval);
409    
410    
411        description = "The maximum number of intervals for which to run.  If " +
412                      "this is not provided, then the tool will run until it is " +
413                      "interrupted.";
414        numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
415                                           description, 1, Integer.MAX_VALUE,
416                                           Integer.MAX_VALUE);
417        parser.addArgument(numIntervals);
418    
419        description = "The number of search iterations that should be processed " +
420                      "on a connection before that connection is closed and " +
421                      "replaced with a newly-established (and authenticated, if " +
422                      "appropriate) connection.  If this is not provided, then " +
423                      "connections will not be periodically closed and " +
424                      "re-established.";
425        iterationsBeforeReconnect = new IntegerArgument(null,
426             "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
427        parser.addArgument(iterationsBeforeReconnect);
428    
429        description = "The target number of searches to perform per second.  It " +
430                      "is still necessary to specify a sufficient number of " +
431                      "threads for achieving this rate.  If neither this option " +
432                      "nor --variableRateData is provided, then the tool will " +
433                      "run at the maximum rate for the specified number of " +
434                      "threads.";
435        ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
436                                            "{searches-per-second}", description,
437                                            1, Integer.MAX_VALUE);
438        parser.addArgument(ratePerSecond);
439    
440        final String variableRateDataArgName = "variableRateData";
441        final String generateSampleRateFileArgName = "generateSampleRateFile";
442        description = RateAdjustor.getVariableRateDataArgumentDescription(
443             generateSampleRateFileArgName);
444        variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
445                                            "{path}", description, true, true, true,
446                                            false);
447        parser.addArgument(variableRateData);
448    
449        description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
450             variableRateDataArgName);
451        sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
452                                          false, 1, "{path}", description, false,
453                                          true, true, false);
454        sampleRateFile.setUsageArgument(true);
455        parser.addArgument(sampleRateFile);
456        parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
457    
458        description = "The number of intervals to complete before beginning " +
459                      "overall statistics collection.  Specifying a nonzero " +
460                      "number of warm-up intervals gives the client and server " +
461                      "a chance to warm up without skewing performance results.";
462        warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
463             "{num}", description, 0, Integer.MAX_VALUE, 0);
464        parser.addArgument(warmUpIntervals);
465    
466        description = "Indicates the format to use for timestamps included in " +
467                      "the output.  A value of 'none' indicates that no " +
468                      "timestamps should be included.  A value of 'with-date' " +
469                      "indicates that both the date and the time should be " +
470                      "included.  A value of 'without-date' indicates that only " +
471                      "the time should be included.";
472        final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
473        allowedFormats.add("none");
474        allowedFormats.add("with-date");
475        allowedFormats.add("without-date");
476        timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
477             "{format}", description, allowedFormats, "none");
478        parser.addArgument(timestampFormat);
479    
480        description = "Indicates that the proxied authorization control (as " +
481                      "defined in RFC 4370) should be used to request that " +
482                      "operations be processed using an alternate authorization " +
483                      "identity.";
484        proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
485                                     description);
486        parser.addArgument(proxyAs);
487    
488        description = "Indicates that the client should operate in asynchronous " +
489                      "mode, in which it will not be necessary to wait for a " +
490                      "response to a previous request before sending the next " +
491                      "request.  Either the '--ratePerSecond' or the " +
492                      "'--maxOutstandingRequests' argument must be provided to " +
493                      "limit the number of outstanding requests.";
494        asynchronousMode = new BooleanArgument('a', "asynchronous", description);
495        parser.addArgument(asynchronousMode);
496    
497        description = "Specifies the maximum number of outstanding requests " +
498                      "that should be allowed when operating in asynchronous mode.";
499        maxOutstandingRequests = new IntegerArgument('O', "maxOutstandingRequests",
500             false, 1, "{num}", description, 1, Integer.MAX_VALUE, (Integer) null);
501        parser.addArgument(maxOutstandingRequests);
502    
503        description = "Indicates that information about the result codes for " +
504                      "failed operations should not be displayed.";
505        suppressErrors = new BooleanArgument(null,
506             "suppressErrorResultCodes", 1, description);
507        parser.addArgument(suppressErrors);
508    
509        description = "Generate output in CSV format rather than a " +
510                      "display-friendly format";
511        csvFormat = new BooleanArgument('c', "csv", 1, description);
512        parser.addArgument(csvFormat);
513    
514        description = "Specifies the seed to use for the random number generator.";
515        randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
516             description);
517        parser.addArgument(randomSeed);
518    
519    
520        parser.addDependentArgumentSet(asynchronousMode, ratePerSecond,
521             maxOutstandingRequests);
522        parser.addDependentArgumentSet(maxOutstandingRequests, asynchronousMode);
523      }
524    
525    
526    
527      /**
528       * Indicates whether this tool supports creating connections to multiple
529       * servers.  If it is to support multiple servers, then the "--hostname" and
530       * "--port" arguments will be allowed to be provided multiple times, and
531       * will be required to be provided the same number of times.  The same type of
532       * communication security and bind credentials will be used for all servers.
533       *
534       * @return  {@code true} if this tool supports creating connections to
535       *          multiple servers, or {@code false} if not.
536       */
537      @Override()
538      protected boolean supportsMultipleServers()
539      {
540        return true;
541      }
542    
543    
544    
545      /**
546       * Retrieves the connection options that should be used for connections
547       * created for use with this tool.
548       *
549       * @return  The connection options that should be used for connections created
550       *          for use with this tool.
551       */
552      @Override()
553      public LDAPConnectionOptions getConnectionOptions()
554      {
555        final LDAPConnectionOptions options = new LDAPConnectionOptions();
556        options.setAutoReconnect(true);
557        options.setUseSynchronousMode(! asynchronousMode.isPresent());
558        return options;
559      }
560    
561    
562    
563      /**
564       * Performs the actual processing for this tool.  In this case, it gets a
565       * connection to the directory server and uses it to perform the requested
566       * searches.
567       *
568       * @return  The result code for the processing that was performed.
569       */
570      @Override()
571      public ResultCode doToolProcessing()
572      {
573        runningThread = Thread.currentThread();
574    
575        try
576        {
577          return doToolProcessingInternal();
578        }
579        finally
580        {
581          runningThread = null;
582        }
583      }
584    
585    
586    
587      /**
588       * Performs the actual processing for this tool.  In this case, it gets a
589       * connection to the directory server and uses it to perform the requested
590       * searches.
591       *
592       * @return  The result code for the processing that was performed.
593       */
594      private ResultCode doToolProcessingInternal()
595      {
596        // If the sample rate file argument was specified, then generate the sample
597        // variable rate data file and return.
598        if (sampleRateFile.isPresent())
599        {
600          try
601          {
602            RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
603            return ResultCode.SUCCESS;
604          }
605          catch (final Exception e)
606          {
607            debugException(e);
608            err("An error occurred while trying to write sample variable data " +
609                 "rate file '", sampleRateFile.getValue().getAbsolutePath(),
610                 "':  ", getExceptionMessage(e));
611            return ResultCode.LOCAL_ERROR;
612          }
613        }
614    
615    
616        // Determine the random seed to use.
617        final Long seed;
618        if (randomSeed.isPresent())
619        {
620          seed = Long.valueOf(randomSeed.getValue());
621        }
622        else
623        {
624          seed = null;
625        }
626    
627        // Create value patterns for the base DN, filter, and proxied authorization
628        // DN.
629        final ValuePattern dnPattern;
630        try
631        {
632          dnPattern = new ValuePattern(baseDN.getValue(), seed);
633        }
634        catch (final ParseException pe)
635        {
636          debugException(pe);
637          err("Unable to parse the base DN value pattern:  ", pe.getMessage());
638          return ResultCode.PARAM_ERROR;
639        }
640    
641        final ValuePattern filterPattern;
642        try
643        {
644          filterPattern = new ValuePattern(filter.getValue(), seed);
645        }
646        catch (final ParseException pe)
647        {
648          debugException(pe);
649          err("Unable to parse the filter pattern:  ", pe.getMessage());
650          return ResultCode.PARAM_ERROR;
651        }
652    
653        final ValuePattern authzIDPattern;
654        if (proxyAs.isPresent())
655        {
656          try
657          {
658            authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
659          }
660          catch (final ParseException pe)
661          {
662            debugException(pe);
663            err("Unable to parse the proxied authorization pattern:  ",
664                pe.getMessage());
665            return ResultCode.PARAM_ERROR;
666          }
667        }
668        else
669        {
670          authzIDPattern = null;
671        }
672    
673    
674        // Get the attributes to return.
675        final String[] attrs;
676        if (attributes.isPresent())
677        {
678          final List<String> attrList = attributes.getValues();
679          attrs = new String[attrList.size()];
680          attrList.toArray(attrs);
681        }
682        else
683        {
684          attrs = NO_STRINGS;
685        }
686    
687    
688        // If the --ratePerSecond option was specified, then limit the rate
689        // accordingly.
690        FixedRateBarrier fixedRateBarrier = null;
691        if (ratePerSecond.isPresent() || variableRateData.isPresent())
692        {
693          // We might not have a rate per second if --variableRateData is specified.
694          // The rate typically doesn't matter except when we have warm-up
695          // intervals.  In this case, we'll run at the max rate.
696          final int intervalSeconds = collectionInterval.getValue();
697          final int ratePerInterval =
698               (ratePerSecond.getValue() == null)
699               ? Integer.MAX_VALUE
700               : ratePerSecond.getValue() * intervalSeconds;
701          fixedRateBarrier =
702               new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
703        }
704    
705    
706        // If --variableRateData was specified, then initialize a RateAdjustor.
707        RateAdjustor rateAdjustor = null;
708        if (variableRateData.isPresent())
709        {
710          try
711          {
712            rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
713                 ratePerSecond.getValue(), variableRateData.getValue());
714          }
715          catch (final IOException e)
716          {
717            debugException(e);
718            err("Initializing the variable rates failed: " + e.getMessage());
719            return ResultCode.PARAM_ERROR;
720          }
721          catch (final IllegalArgumentException e)
722          {
723            debugException(e);
724            err("Initializing the variable rates failed: " + e.getMessage());
725            return ResultCode.PARAM_ERROR;
726          }
727        }
728    
729    
730        // If the --maxOutstandingRequests option was specified, then create the
731        // semaphore used to enforce that limit.
732        final Semaphore asyncSemaphore;
733        if (maxOutstandingRequests.isPresent())
734        {
735          asyncSemaphore = new Semaphore(maxOutstandingRequests.getValue());
736        }
737        else
738        {
739          asyncSemaphore = null;
740        }
741    
742    
743        // Determine whether to include timestamps in the output and if so what
744        // format should be used for them.
745        final boolean includeTimestamp;
746        final String timeFormat;
747        if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
748        {
749          includeTimestamp = true;
750          timeFormat       = "dd/MM/yyyy HH:mm:ss";
751        }
752        else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
753        {
754          includeTimestamp = true;
755          timeFormat       = "HH:mm:ss";
756        }
757        else
758        {
759          includeTimestamp = false;
760          timeFormat       = null;
761        }
762    
763    
764        // Determine whether any warm-up intervals should be run.
765        final long totalIntervals;
766        final boolean warmUp;
767        int remainingWarmUpIntervals = warmUpIntervals.getValue();
768        if (remainingWarmUpIntervals > 0)
769        {
770          warmUp = true;
771          totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
772        }
773        else
774        {
775          warmUp = true;
776          totalIntervals = 0L + numIntervals.getValue();
777        }
778    
779    
780        // Create the table that will be used to format the output.
781        final OutputFormat outputFormat;
782        if (csvFormat.isPresent())
783        {
784          outputFormat = OutputFormat.CSV;
785        }
786        else
787        {
788          outputFormat = OutputFormat.COLUMNS;
789        }
790    
791        final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
792             timeFormat, outputFormat, " ",
793             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
794                      "Searches/Sec"),
795             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
796                      "Avg Dur ms"),
797             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
798                      "Entries/Srch"),
799             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
800                      "Errors/Sec"),
801             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
802                      "Searches/Sec"),
803             new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
804                      "Avg Dur ms"));
805    
806    
807        // Create values to use for statistics collection.
808        final AtomicLong        searchCounter   = new AtomicLong(0L);
809        final AtomicLong        entryCounter    = new AtomicLong(0L);
810        final AtomicLong        errorCounter    = new AtomicLong(0L);
811        final AtomicLong        searchDurations = new AtomicLong(0L);
812        final ResultCodeCounter rcCounter       = new ResultCodeCounter();
813    
814    
815        // Determine the length of each interval in milliseconds.
816        final long intervalMillis = 1000L * collectionInterval.getValue();
817    
818    
819        // Create the threads to use for the searches.
820        final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
821        final SearchRateThread[] threads =
822             new SearchRateThread[numThreads.getValue()];
823        for (int i=0; i < threads.length; i++)
824        {
825          final LDAPConnection connection;
826          try
827          {
828            connection = getConnection();
829          }
830          catch (final LDAPException le)
831          {
832            debugException(le);
833            err("Unable to connect to the directory server:  ",
834                getExceptionMessage(le));
835            return le.getResultCode();
836          }
837    
838          threads[i] = new SearchRateThread(this, i, connection,
839               asynchronousMode.isPresent(), dnPattern, scopeArg.getValue(),
840               filterPattern, attrs, authzIDPattern,
841               iterationsBeforeReconnect.getValue(), barrier, searchCounter,
842               entryCounter, searchDurations, errorCounter, rcCounter,
843               fixedRateBarrier, asyncSemaphore);
844          threads[i].start();
845        }
846    
847    
848        // Display the table header.
849        for (final String headerLine : formatter.getHeaderLines(true))
850        {
851          out(headerLine);
852        }
853    
854    
855        // Start the RateAdjustor before the threads so that the initial value is
856        // in place before any load is generated unless we're doing a warm-up in
857        // which case, we'll start it after the warm-up is complete.
858        if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
859        {
860          rateAdjustor.start();
861        }
862    
863    
864        // Indicate that the threads can start running.
865        try
866        {
867          barrier.await();
868        }
869        catch (final Exception e)
870        {
871          debugException(e);
872        }
873    
874        long overallStartTime = System.nanoTime();
875        long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
876    
877    
878        boolean setOverallStartTime = false;
879        long    lastDuration        = 0L;
880        long    lastNumEntries      = 0L;
881        long    lastNumErrors       = 0L;
882        long    lastNumSearches     = 0L;
883        long    lastEndTime         = System.nanoTime();
884        for (long i=0; i < totalIntervals; i++)
885        {
886          if (rateAdjustor != null)
887          {
888            if (! rateAdjustor.isAlive())
889            {
890              out("All of the rates in " + variableRateData.getValue().getName() +
891                  " have been completed.");
892              break;
893            }
894          }
895    
896          final long startTimeMillis = System.currentTimeMillis();
897          final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
898          nextIntervalStartTime += intervalMillis;
899          if (sleepTimeMillis > 0)
900          {
901            sleeper.sleep(sleepTimeMillis);
902          }
903    
904          if (stopRequested.get())
905          {
906            break;
907          }
908    
909          final long endTime          = System.nanoTime();
910          final long intervalDuration = endTime - lastEndTime;
911    
912          final long numSearches;
913          final long numEntries;
914          final long numErrors;
915          final long totalDuration;
916          if (warmUp && (remainingWarmUpIntervals > 0))
917          {
918            numSearches   = searchCounter.getAndSet(0L);
919            numEntries    = entryCounter.getAndSet(0L);
920            numErrors     = errorCounter.getAndSet(0L);
921            totalDuration = searchDurations.getAndSet(0L);
922          }
923          else
924          {
925            numSearches   = searchCounter.get();
926            numEntries    = entryCounter.get();
927            numErrors     = errorCounter.get();
928            totalDuration = searchDurations.get();
929          }
930    
931          final long recentNumSearches = numSearches - lastNumSearches;
932          final long recentNumEntries = numEntries - lastNumEntries;
933          final long recentNumErrors = numErrors - lastNumErrors;
934          final long recentDuration = totalDuration - lastDuration;
935    
936          final double numSeconds = intervalDuration / 1000000000.0d;
937          final double recentSearchRate = recentNumSearches / numSeconds;
938          final double recentErrorRate  = recentNumErrors / numSeconds;
939    
940          final double recentAvgDuration;
941          final double recentEntriesPerSearch;
942          if (recentNumSearches > 0L)
943          {
944            recentEntriesPerSearch = 1.0d * recentNumEntries / recentNumSearches;
945            recentAvgDuration = 1.0d * recentDuration / recentNumSearches / 1000000;
946          }
947          else
948          {
949            recentEntriesPerSearch = 0.0d;
950            recentAvgDuration = 0.0d;
951          }
952    
953    
954          if (warmUp && (remainingWarmUpIntervals > 0))
955          {
956            out(formatter.formatRow(recentSearchRate, recentAvgDuration,
957                 recentEntriesPerSearch, recentErrorRate, "warming up",
958                 "warming up"));
959    
960            remainingWarmUpIntervals--;
961            if (remainingWarmUpIntervals == 0)
962            {
963              out("Warm-up completed.  Beginning overall statistics collection.");
964              setOverallStartTime = true;
965              if (rateAdjustor != null)
966              {
967                rateAdjustor.start();
968              }
969            }
970          }
971          else
972          {
973            if (setOverallStartTime)
974            {
975              overallStartTime    = lastEndTime;
976              setOverallStartTime = false;
977            }
978    
979            final double numOverallSeconds =
980                 (endTime - overallStartTime) / 1000000000.0d;
981            final double overallSearchRate = numSearches / numOverallSeconds;
982    
983            final double overallAvgDuration;
984            if (numSearches > 0L)
985            {
986              overallAvgDuration = 1.0d * totalDuration / numSearches / 1000000;
987            }
988            else
989            {
990              overallAvgDuration = 0.0d;
991            }
992    
993            out(formatter.formatRow(recentSearchRate, recentAvgDuration,
994                 recentEntriesPerSearch, recentErrorRate, overallSearchRate,
995                 overallAvgDuration));
996    
997            lastNumSearches = numSearches;
998            lastNumEntries  = numEntries;
999            lastNumErrors   = numErrors;
1000            lastDuration    = totalDuration;
1001          }
1002    
1003          final List<ObjectPair<ResultCode,Long>> rcCounts =
1004               rcCounter.getCounts(true);
1005          if ((! suppressErrors.isPresent()) && (! rcCounts.isEmpty()))
1006          {
1007            err("\tError Results:");
1008            for (final ObjectPair<ResultCode,Long> p : rcCounts)
1009            {
1010              err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1011            }
1012          }
1013    
1014          lastEndTime = endTime;
1015        }
1016    
1017    
1018        // Shut down the RateAdjustor if we have one.
1019        if (rateAdjustor != null)
1020        {
1021          rateAdjustor.shutDown();
1022        }
1023    
1024    
1025        // Stop all of the threads.
1026        ResultCode resultCode = ResultCode.SUCCESS;
1027        for (final SearchRateThread t : threads)
1028        {
1029          t.signalShutdown();
1030        }
1031        for (final SearchRateThread t : threads)
1032        {
1033          final ResultCode r = t.waitForShutdown();
1034          if (resultCode == ResultCode.SUCCESS)
1035          {
1036            resultCode = r;
1037          }
1038        }
1039    
1040        return resultCode;
1041      }
1042    
1043    
1044    
1045      /**
1046       * Requests that this tool stop running.  This method will attempt to wait
1047       * for all threads to complete before returning control to the caller.
1048       */
1049      public void stopRunning()
1050      {
1051        stopRequested.set(true);
1052        sleeper.wakeup();
1053    
1054        final Thread t = runningThread;
1055        if (t != null)
1056        {
1057          try
1058          {
1059            t.join();
1060          }
1061          catch (final Exception e)
1062          {
1063            debugException(e);
1064          }
1065        }
1066      }
1067    
1068    
1069    
1070      /**
1071       * Retrieves the maximum number of outstanding requests that may be in
1072       * progress at any time, if appropriate.
1073       *
1074       * @return  The maximum number of outstanding requests that may be in progress
1075       *          at any time, or -1 if the tool was not configured to perform
1076       *          asynchronous searches with a maximum number of outstanding
1077       *          requests.
1078       */
1079      int getMaxOutstandingRequests()
1080      {
1081        if (maxOutstandingRequests.isPresent())
1082        {
1083          return maxOutstandingRequests.getValue();
1084        }
1085        else
1086        {
1087          return -1;
1088        }
1089      }
1090    
1091    
1092    
1093      /**
1094       * {@inheritDoc}
1095       */
1096      @Override()
1097      public LinkedHashMap<String[],String> getExampleUsages()
1098      {
1099        final LinkedHashMap<String[],String> examples =
1100             new LinkedHashMap<String[],String>(2);
1101    
1102        String[] args =
1103        {
1104          "--hostname", "server.example.com",
1105          "--port", "389",
1106          "--bindDN", "uid=admin,dc=example,dc=com",
1107          "--bindPassword", "password",
1108          "--baseDN", "dc=example,dc=com",
1109          "--scope", "sub",
1110          "--filter", "(uid=user.[1-1000000])",
1111          "--attribute", "givenName",
1112          "--attribute", "sn",
1113          "--attribute", "mail",
1114          "--numThreads", "10"
1115        };
1116        String description =
1117             "Test search performance by searching randomly across a set " +
1118             "of one million users located below 'dc=example,dc=com' with ten " +
1119             "concurrent threads.  The entries returned to the client will " +
1120             "include the givenName, sn, and mail attributes.";
1121        examples.put(args, description);
1122    
1123        args = new String[]
1124        {
1125          "--generateSampleRateFile", "variable-rate-data.txt"
1126        };
1127        description =
1128             "Generate a sample variable rate definition file that may be used " +
1129             "in conjunction with the --variableRateData argument.  The sample " +
1130             "file will include comments that describe the format for data to be " +
1131             "included in this file.";
1132        examples.put(args, description);
1133    
1134        return examples;
1135      }
1136    }