001/*
002 * Copyright 2009-2022 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright 2009-2022 Ping Identity Corporation
007 *
008 * Licensed under the Apache License, Version 2.0 (the "License");
009 * you may not use this file except in compliance with the License.
010 * You may obtain a copy of the License at
011 *
012 *    http://www.apache.org/licenses/LICENSE-2.0
013 *
014 * Unless required by applicable law or agreed to in writing, software
015 * distributed under the License is distributed on an "AS IS" BASIS,
016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
017 * See the License for the specific language governing permissions and
018 * limitations under the License.
019 */
020/*
021 * Copyright (C) 2009-2022 Ping Identity Corporation
022 *
023 * This program is free software; you can redistribute it and/or modify
024 * it under the terms of the GNU General Public License (GPLv2 only)
025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
026 * as published by the Free Software Foundation.
027 *
028 * This program is distributed in the hope that it will be useful,
029 * but WITHOUT ANY WARRANTY; without even the implied warranty of
030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
031 * GNU General Public License for more details.
032 *
033 * You should have received a copy of the GNU General Public License
034 * along with this program; if not, see <http://www.gnu.org/licenses>.
035 */
036package com.unboundid.ldap.sdk.unboundidds.examples;
037
038
039
040import java.io.File;
041import java.io.FileInputStream;
042import java.io.InputStream;
043import java.io.IOException;
044import java.io.OutputStream;
045import java.io.Serializable;
046import java.text.DecimalFormat;
047import java.util.ArrayList;
048import java.util.HashMap;
049import java.util.HashSet;
050import java.util.Iterator;
051import java.util.LinkedHashMap;
052import java.util.LinkedHashSet;
053import java.util.List;
054import java.util.Map;
055import java.util.TreeMap;
056import java.util.concurrent.atomic.AtomicLong;
057import java.util.zip.GZIPInputStream;
058import javax.crypto.BadPaddingException;
059
060import com.unboundid.ldap.sdk.DN;
061import com.unboundid.ldap.sdk.Filter;
062import com.unboundid.ldap.sdk.LDAPException;
063import com.unboundid.ldap.sdk.RDN;
064import com.unboundid.ldap.sdk.ResultCode;
065import com.unboundid.ldap.sdk.SearchScope;
066import com.unboundid.ldap.sdk.Version;
067import com.unboundid.ldap.sdk.unboundidds.logs.LogException;
068import com.unboundid.ldap.sdk.unboundidds.logs.v2.
069            AbandonRequestAccessLogMessage;
070import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogMessage;
071import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogReader;
072import com.unboundid.ldap.sdk.unboundidds.logs.v2.AddResultAccessLogMessage;
073import com.unboundid.ldap.sdk.unboundidds.logs.v2.BindResultAccessLogMessage;
074import com.unboundid.ldap.sdk.unboundidds.logs.v2.CompareResultAccessLogMessage;
075import com.unboundid.ldap.sdk.unboundidds.logs.v2.ConnectAccessLogMessage;
076import com.unboundid.ldap.sdk.unboundidds.logs.v2.DeleteResultAccessLogMessage;
077import com.unboundid.ldap.sdk.unboundidds.logs.v2.DisconnectAccessLogMessage;
078import com.unboundid.ldap.sdk.unboundidds.logs.v2.
079            ExtendedRequestAccessLogMessage;
080import com.unboundid.ldap.sdk.unboundidds.logs.v2.
081            ExtendedResultAccessLogMessage;
082import com.unboundid.ldap.sdk.unboundidds.logs.v2.
083            ModifyDNResultAccessLogMessage;
084import com.unboundid.ldap.sdk.unboundidds.logs.v2.ModifyResultAccessLogMessage;
085import com.unboundid.ldap.sdk.unboundidds.logs.v2.
086            OperationRequestAccessLogMessage;
087import com.unboundid.ldap.sdk.unboundidds.logs.v2.
088            OperationResultAccessLogMessage;
089import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchRequestAccessLogMessage;
090import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchResultAccessLogMessage;
091import com.unboundid.ldap.sdk.unboundidds.logs.v2.
092            SecurityNegotiationAccessLogMessage;
093import com.unboundid.ldap.sdk.unboundidds.logs.v2.UnbindRequestAccessLogMessage;
094import com.unboundid.ldap.sdk.unboundidds.logs.v2.json.JSONAccessLogReader;
095import com.unboundid.ldap.sdk.unboundidds.logs.v2.text.
096            TextFormattedAccessLogReader;
097import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils;
098import com.unboundid.util.CommandLineTool;
099import com.unboundid.util.Debug;
100import com.unboundid.util.NotMutable;
101import com.unboundid.util.NotNull;
102import com.unboundid.util.Nullable;
103import com.unboundid.util.OIDRegistry;
104import com.unboundid.util.OIDRegistryItem;
105import com.unboundid.util.ObjectPair;
106import com.unboundid.util.ReverseComparator;
107import com.unboundid.util.StaticUtils;
108import com.unboundid.util.ThreadSafety;
109import com.unboundid.util.ThreadSafetyLevel;
110import com.unboundid.util.args.ArgumentException;
111import com.unboundid.util.args.ArgumentParser;
112import com.unboundid.util.args.BooleanArgument;
113import com.unboundid.util.args.DurationArgument;
114import com.unboundid.util.args.FileArgument;
115import com.unboundid.util.args.IntegerArgument;
116
117
118
119/**
120 * This class provides a tool that may be used to read and summarize the
121 * contents of one or more access log files from Ping Identity, UnboundID and
122 * Nokia/Alcatel-Lucent 8661 server products.
123 * <BR>
124 * <BLOCKQUOTE>
125 *   <B>NOTE:</B>  This class, and other classes within the
126 *   {@code com.unboundid.ldap.sdk.unboundidds} package structure, are only
127 *   supported for use against Ping Identity, UnboundID, and
128 *   Nokia/Alcatel-Lucent 8661 server products.  These classes provide support
129 *   for proprietary functionality or for external specifications that are not
130 *   considered stable or mature enough to be guaranteed to work in an
131 *   interoperable way with other types of LDAP servers.
132 * </BLOCKQUOTE>
133 * Information that will be reported includes:
134 * <UL>
135 *   <LI>The total length of time covered by the log files.</LI>
136 *   <LI>The number of connections established and disconnected, the addresses
137 *       of the most commonly-connecting clients, and the average rate of
138 *       connects and disconnects per second.</LI>
139 *   <LI>The number of operations processed, overall and by operation type,
140 *       and the average rate of operations per second.</LI>
141 *   <LI>The average duration for operations processed, overall and by operation
142 *       type.</LI>
143 *   <LI>A breakdown of operation processing times into a number of predefined
144 *       categories, ranging from less than one millisecond to over one
145 *       minute.</LI>
146 *   <LI>A breakdown of the most common result codes for each type of operation
147 *       and their relative frequencies.</LI>
148 *   <LI>The most common types of extended requests processed and their
149 *       relative frequencies.</LI>
150 *   <LI>The number of unindexed search operations processed and the most common
151 *       types of filters used in unindexed searches.</LI>
152 *   <LI>A breakdown of the relative frequencies for each type of search
153 *       scope.</LI>
154 *   <LI>The most common types of search filters used for search
155 *       operations and their relative frequencies.</LI>
156 * </UL>
157 * It is designed to work with access log files using either the default log
158 * format with separate request and response messages, as well as log files
159 * in which the request and response details have been combined on the same
160 * line.  The log files to be processed should be provided as command-line
161 * arguments.
162 * <BR><BR>
163 * The APIs demonstrated by this example include:
164 * <UL>
165 *   <LI>Access log parsing (from the
166 *       {@code com.unboundid.ldap.sdk.unboundidds.logs} package)</LI>
167 *   <LI>Argument parsing (from the {@code com.unboundid.util.args}
168 *       package)</LI>
169 * </UL>
170 */
171@NotMutable()
172@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
173public final class SummarizeAccessLog
174       extends CommandLineTool
175       implements Serializable
176{
177  /**
178   * The column at which long lines should be wrapped.
179   */
180  private static final int WRAP_COLUMN = StaticUtils.TERMINAL_WIDTH_COLUMNS - 1;
181
182
183
184  /**
185   * The serial version UID for this serializable class.
186   */
187  private static final long serialVersionUID = 7189168366509887130L;
188
189
190
191  // Variables used for accessing argument information.
192  @Nullable private ArgumentParser argumentParser;
193
194  // An argument that may be used to indicate that the summarized output should
195  // not be anonymized, and should include attribute values.
196  @Nullable private BooleanArgument doNotAnonymize;
197
198  // An argument that may be used to indicate that the log files are compressed.
199  @Nullable private BooleanArgument isCompressed;
200
201  // An argument that may be used to indicate that the log content is
202  // JSON-formatted rather than text-formatted.
203  @Nullable private BooleanArgument json;
204
205  // An argument used to specify the encryption passphrase.
206  @Nullable private FileArgument encryptionPassphraseFile;
207
208  // An argument used to specify the maximum number of values to report for each
209  // item.
210  @Nullable private IntegerArgument reportCount;
211
212  // The decimal format that will be used for this class.
213  @NotNull private final DecimalFormat decimalFormat;
214
215  // The total duration for log content, in milliseconds.
216  private long logDurationMillis;
217
218  // The total processing time for each type of operation.
219  private double addProcessingDuration;
220  private double bindProcessingDuration;
221  private double compareProcessingDuration;
222  private double deleteProcessingDuration;
223  private double extendedProcessingDuration;
224  private double modifyProcessingDuration;
225  private double modifyDNProcessingDuration;
226  private double searchProcessingDuration;
227
228  // A variable used for tracking total  work queue wait time.
229  private long totalWorkQueueWaitTime;
230
231  // A variable used for counting the number of messages of each type.
232  private long numAbandons;
233  private long numAdds;
234  private long numBinds;
235  private long numCompares;
236  private long numConnects;
237  private long numDeletes;
238  private long numDisconnects;
239  private long numExtended;
240  private long numModifies;
241  private long numModifyDNs;
242  private long numSearches;
243  private long numUnbinds;
244
245  // The number of operations of each type that accessed uncached data.
246  private long numUncachedAdds;
247  private long numUncachedBinds;
248  private long numUncachedCompares;
249  private long numUncachedDeletes;
250  private long numUncachedExtended;
251  private long numUncachedModifies;
252  private long numUncachedModifyDNs;
253  private long numUncachedSearches;
254
255  // The number of unindexed searches processed within the server.
256  private long numUnindexedAttempts;
257  private long numUnindexedFailed;
258  private long numUnindexedSuccessful;
259
260  // The number of request and response controls used.
261  private long numRequestControls;
262  private long numResponseControls;
263
264  // Variables used for maintaining counts for common types of information.
265  @NotNull private final HashMap<Long,AtomicLong> searchEntryCounts;
266  @NotNull private final HashMap<Long,String> ipAddressesByConnectionID;
267  @NotNull private final HashMap<ResultCode,AtomicLong> addResultCodes;
268  @NotNull private final HashMap<ResultCode,AtomicLong> bindResultCodes;
269  @NotNull private final HashMap<ResultCode,AtomicLong> compareResultCodes;
270  @NotNull private final HashMap<ResultCode,AtomicLong> deleteResultCodes;
271  @NotNull private final HashMap<ResultCode,AtomicLong> extendedResultCodes;
272  @NotNull private final HashMap<ResultCode,AtomicLong> modifyResultCodes;
273  @NotNull private final HashMap<ResultCode,AtomicLong> modifyDNResultCodes;
274  @NotNull private final HashMap<ResultCode,AtomicLong> searchResultCodes;
275  @NotNull private final HashMap<SearchScope,AtomicLong> searchScopes;
276  @NotNull private final HashMap<String,AtomicLong> authenticationTypes;
277  @NotNull private final HashMap<String,AtomicLong> authzDNs;
278  @NotNull private final HashMap<String,AtomicLong> bindFailuresByDN;
279  @NotNull private final HashMap<String,AtomicLong> bindFailuresByIPAddress;
280  @NotNull private final HashMap<String,AtomicLong> consecutiveFailedBindsByDN;
281  @NotNull private final HashMap<String,AtomicLong> outstandingFailedBindDNs;
282  @NotNull private final HashMap<String,AtomicLong> successfulBindDNs;
283  @NotNull private final HashMap<String,AtomicLong> clientAddresses;
284  @NotNull private final HashMap<String,AtomicLong> clientConnectionPolicies;
285  @NotNull private final HashMap<String,AtomicLong> disconnectReasons;
286  @NotNull private final HashMap<String,AtomicLong> extendedOperations;
287  @NotNull private final HashMap<String,AtomicLong> filterComponentCounts;
288  @NotNull private final HashMap<String,AtomicLong> filterTypes;
289  @NotNull private final HashMap<String,AtomicLong> mostExpensiveFilters;
290  @NotNull private final HashMap<String,AtomicLong> multiEntryFilters;
291  @NotNull private final HashMap<String,AtomicLong> noEntryFilters;
292  @NotNull private final HashMap<String,AtomicLong> oneEntryFilters;
293  @NotNull private final HashMap<String,AtomicLong> preAuthzPrivilegesUsed;
294  @NotNull private final HashMap<String,AtomicLong> privilegesMissing;
295  @NotNull private final HashMap<String,AtomicLong> privilegesUsed;
296  @NotNull private final HashMap<String,AtomicLong> requestControlOIDs;
297  @NotNull private final HashMap<String,AtomicLong> responseControlOIDs;
298  @NotNull private final HashMap<String,AtomicLong> searchBaseDNs;
299  @NotNull private final HashMap<String,AtomicLong> tlsCipherSuites;
300  @NotNull private final HashMap<String,AtomicLong> tlsProtocols;
301  @NotNull private final HashMap<String,AtomicLong> unindexedFilters;
302  @NotNull private final HashMap<String,String> extendedOperationOIDsToNames;
303  @NotNull private final HashSet<String> processedRequests;
304  @NotNull private final LinkedHashMap<Long,AtomicLong> addProcessingTimes;
305  @NotNull private final LinkedHashMap<Long,AtomicLong> bindProcessingTimes;
306  @NotNull private final LinkedHashMap<Long,AtomicLong> compareProcessingTimes;
307  @NotNull private final LinkedHashMap<Long,AtomicLong> deleteProcessingTimes;
308  @NotNull private final LinkedHashMap<Long,AtomicLong> extendedProcessingTimes;
309  @NotNull private final LinkedHashMap<Long,AtomicLong> modifyProcessingTimes;
310  @NotNull private final LinkedHashMap<Long,AtomicLong> modifyDNProcessingTimes;
311  @NotNull private final LinkedHashMap<Long,AtomicLong> searchProcessingTimes;
312  @NotNull private final LinkedHashMap<Long,AtomicLong> workQueueWaitTimes;
313  @NotNull private final LinkedHashSet<Filter>
314       filtersRepresentingPotentialInjectionAttempt;
315
316
317
318  /**
319   * Parse the provided command line arguments and perform the appropriate
320   * processing.
321   *
322   * @param  args  The command line arguments provided to this program.
323   */
324  public static void main(@NotNull final String[] args)
325  {
326    final ResultCode resultCode = main(args, System.out, System.err);
327    if (resultCode != ResultCode.SUCCESS)
328    {
329      System.exit(resultCode.intValue());
330    }
331  }
332
333
334
335  /**
336   * Parse the provided command line arguments and perform the appropriate
337   * processing.
338   *
339   * @param  args       The command line arguments provided to this program.
340   * @param  outStream  The output stream to which standard out should be
341   *                    written.  It may be {@code null} if output should be
342   *                    suppressed.
343   * @param  errStream  The output stream to which standard error should be
344   *                    written.  It may be {@code null} if error messages
345   *                    should be suppressed.
346   *
347   * @return  A result code indicating whether the processing was successful.
348   */
349  @NotNull()
350  public static ResultCode main(@NotNull final String[] args,
351                                @Nullable final OutputStream outStream,
352                                @Nullable final OutputStream errStream)
353  {
354    final SummarizeAccessLog summarizer =
355         new SummarizeAccessLog(outStream, errStream);
356    return summarizer.runTool(args);
357  }
358
359
360
361  /**
362   * Creates a new instance of this tool.
363   *
364   * @param  outStream  The output stream to which standard out should be
365   *                    written.  It may be {@code null} if output should be
366   *                    suppressed.
367   * @param  errStream  The output stream to which standard error should be
368   *                    written.  It may be {@code null} if error messages
369   *                    should be suppressed.
370   */
371  public SummarizeAccessLog(@Nullable final OutputStream outStream,
372                            @Nullable final OutputStream errStream)
373  {
374    super(outStream, errStream);
375
376    argumentParser = null;
377    doNotAnonymize = null;
378    isCompressed = null;
379    json = null;
380    encryptionPassphraseFile = null;
381    reportCount = null;
382
383    decimalFormat = new DecimalFormat("0.000");
384
385    logDurationMillis = 0L;
386
387    addProcessingDuration = 0.0;
388    bindProcessingDuration = 0.0;
389    compareProcessingDuration = 0.0;
390    deleteProcessingDuration = 0.0;
391    extendedProcessingDuration = 0.0;
392    modifyProcessingDuration = 0.0;
393    modifyDNProcessingDuration = 0.0;
394    searchProcessingDuration = 0.0;
395
396    totalWorkQueueWaitTime = 0L;
397
398    numAbandons = 0L;
399    numAdds = 0L;
400    numBinds = 0L;
401    numCompares = 0L;
402    numConnects = 0L;
403    numDeletes = 0L;
404    numDisconnects = 0L;
405    numExtended = 0L;
406    numModifies = 0L;
407    numModifyDNs = 0L;
408    numSearches = 0L;
409    numUnbinds = 0L;
410
411    numUncachedAdds = 0L;
412    numUncachedBinds = 0L;
413    numUncachedCompares = 0L;
414    numUncachedDeletes = 0L;
415    numUncachedExtended = 0L;
416    numUncachedModifies = 0L;
417    numUncachedModifyDNs = 0L;
418    numUncachedSearches = 0L;
419
420    numUnindexedAttempts = 0L;
421    numUnindexedFailed = 0L;
422    numUnindexedSuccessful = 0L;
423
424    numRequestControls = 0L;
425    numResponseControls = 0L;
426
427    searchEntryCounts = new HashMap<>(StaticUtils.computeMapCapacity(10));
428    ipAddressesByConnectionID =
429         new HashMap<>(StaticUtils.computeMapCapacity(100));
430    addResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
431    bindResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
432    compareResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
433    deleteResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
434    extendedResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
435    modifyResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
436    modifyDNResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
437    searchResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10));
438    searchScopes = new HashMap<>(StaticUtils.computeMapCapacity(4));
439    authenticationTypes = new HashMap<>(StaticUtils.computeMapCapacity(100));
440    authzDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
441    bindFailuresByDN = new HashMap<>(StaticUtils.computeMapCapacity(100));
442    bindFailuresByIPAddress =
443         new HashMap<>(StaticUtils.computeMapCapacity(100));
444    outstandingFailedBindDNs =
445         new HashMap<>(StaticUtils.computeMapCapacity(100));
446    successfulBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
447    clientAddresses = new HashMap<>(StaticUtils.computeMapCapacity(100));
448    clientConnectionPolicies =
449         new HashMap<>(StaticUtils.computeMapCapacity(100));
450    disconnectReasons = new HashMap<>(StaticUtils.computeMapCapacity(100));
451    extendedOperations = new HashMap<>(StaticUtils.computeMapCapacity(10));
452    filterComponentCounts = new HashMap<>(StaticUtils.computeMapCapacity(10));
453    filterTypes = new HashMap<>(StaticUtils.computeMapCapacity(100));
454    mostExpensiveFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
455    multiEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
456    noEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
457    oneEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
458    preAuthzPrivilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100));
459    privilegesMissing = new HashMap<>(StaticUtils.computeMapCapacity(100));
460    privilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100));
461    requestControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100));
462    responseControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100));
463    searchBaseDNs = new HashMap<>(StaticUtils.computeMapCapacity(100));
464    tlsCipherSuites = new HashMap<>(StaticUtils.computeMapCapacity(100));
465    tlsProtocols = new HashMap<>(StaticUtils.computeMapCapacity(100));
466    unindexedFilters = new HashMap<>(StaticUtils.computeMapCapacity(100));
467    consecutiveFailedBindsByDN =
468         new HashMap<>(StaticUtils.computeMapCapacity(100));
469    extendedOperationOIDsToNames =
470         new HashMap<>(StaticUtils.computeMapCapacity(100));
471    processedRequests = new HashSet<>(StaticUtils.computeMapCapacity(100));
472    addProcessingTimes =
473         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
474    bindProcessingTimes =
475         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
476    compareProcessingTimes =
477         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
478    deleteProcessingTimes =
479         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
480    extendedProcessingTimes =
481         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
482    modifyProcessingTimes =
483         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
484    modifyDNProcessingTimes =
485         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
486    searchProcessingTimes =
487         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
488    workQueueWaitTimes =
489         new LinkedHashMap<>(StaticUtils.computeMapCapacity(11));
490    filtersRepresentingPotentialInjectionAttempt =
491         new LinkedHashSet<>(StaticUtils.computeMapCapacity(10));
492
493    populateProcessingTimeMap(addProcessingTimes);
494    populateProcessingTimeMap(bindProcessingTimes);
495    populateProcessingTimeMap(compareProcessingTimes);
496    populateProcessingTimeMap(deleteProcessingTimes);
497    populateProcessingTimeMap(extendedProcessingTimes);
498    populateProcessingTimeMap(modifyProcessingTimes);
499    populateProcessingTimeMap(modifyDNProcessingTimes);
500    populateProcessingTimeMap(searchProcessingTimes);
501    populateProcessingTimeMap(workQueueWaitTimes);
502  }
503
504
505
506  /**
507   * Retrieves the name for this tool.
508   *
509   * @return  The name for this tool.
510   */
511  @Override()
512  @NotNull()
513  public String getToolName()
514  {
515    return "summarize-access-log";
516  }
517
518
519
520  /**
521   * Retrieves the description for this tool.
522   *
523   * @return  The description for this tool.
524   */
525  @Override()
526  @NotNull()
527  public String getToolDescription()
528  {
529    return "Examine one or more access log files from Ping Identity, " +
530         "UnboundID, or Nokia/Alcatel-Lucent 8661 server products to display " +
531         "a number of metrics about operations processed within the server.";
532  }
533
534
535
536  /**
537   * Retrieves the version string for this tool.
538   *
539   * @return  The version string for this tool.
540   */
541  @Override()
542  @NotNull()
543  public String getToolVersion()
544  {
545    return Version.NUMERIC_VERSION_STRING;
546  }
547
548
549
550  /**
551   * Retrieves the minimum number of unnamed trailing arguments that are
552   * required.
553   *
554   * @return  One, to indicate that at least one trailing argument (representing
555   *          the path to an access log file) must be provided.
556   */
557  @Override()
558  public int getMinTrailingArguments()
559  {
560    return 1;
561  }
562
563
564
565  /**
566   * Retrieves the maximum number of unnamed trailing arguments that may be
567   * provided for this tool.
568   *
569   * @return  The maximum number of unnamed trailing arguments that may be
570   *          provided for this tool.
571   */
572  @Override()
573  public int getMaxTrailingArguments()
574  {
575    return -1;
576  }
577
578
579
580  /**
581   * Retrieves a placeholder string that should be used for trailing arguments
582   * in the usage information for this tool.
583   *
584   * @return  A placeholder string that should be used for trailing arguments in
585   *          the usage information for this tool.
586   */
587  @Override()
588  @NotNull()
589  public String getTrailingArgumentsPlaceholder()
590  {
591    return "{path}";
592  }
593
594
595
596  /**
597   * Indicates whether this tool should provide support for an interactive mode,
598   * in which the tool offers a mode in which the arguments can be provided in
599   * a text-driven menu rather than requiring them to be given on the command
600   * line.  If interactive mode is supported, it may be invoked using the
601   * "--interactive" argument.  Alternately, if interactive mode is supported
602   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
603   * interactive mode may be invoked by simply launching the tool without any
604   * arguments.
605   *
606   * @return  {@code true} if this tool supports interactive mode, or
607   *          {@code false} if not.
608   */
609  @Override()
610  public boolean supportsInteractiveMode()
611  {
612    return true;
613  }
614
615
616
617  /**
618   * Indicates whether this tool defaults to launching in interactive mode if
619   * the tool is invoked without any command-line arguments.  This will only be
620   * used if {@link #supportsInteractiveMode()} returns {@code true}.
621   *
622   * @return  {@code true} if this tool defaults to using interactive mode if
623   *          launched without any command-line arguments, or {@code false} if
624   *          not.
625   */
626  @Override()
627  public boolean defaultsToInteractiveMode()
628  {
629    return true;
630  }
631
632
633
634  /**
635   * Indicates whether this tool should provide arguments for redirecting output
636   * to a file.  If this method returns {@code true}, then the tool will offer
637   * an "--outputFile" argument that will specify the path to a file to which
638   * all standard output and standard error content will be written, and it will
639   * also offer a "--teeToStandardOut" argument that can only be used if the
640   * "--outputFile" argument is present and will cause all output to be written
641   * to both the specified output file and to standard output.
642   *
643   * @return  {@code true} if this tool should provide arguments for redirecting
644   *          output to a file, or {@code false} if not.
645   */
646  @Override()
647  protected boolean supportsOutputFile()
648  {
649    return true;
650  }
651
652
653
654  /**
655   * Indicates whether this tool supports the use of a properties file for
656   * specifying default values for arguments that aren't specified on the
657   * command line.
658   *
659   * @return  {@code true} if this tool supports the use of a properties file
660   *          for specifying default values for arguments that aren't specified
661   *          on the command line, or {@code false} if not.
662   */
663  @Override()
664  public boolean supportsPropertiesFile()
665  {
666    return true;
667  }
668
669
670
671  /**
672   * Adds the command-line arguments supported for use with this tool to the
673   * provided argument parser.  The tool may need to retain references to the
674   * arguments (and/or the argument parser, if trailing arguments are allowed)
675   * to it in order to obtain their values for use in later processing.
676   *
677   * @param  parser  The argument parser to which the arguments are to be added.
678   *
679   * @throws  ArgumentException  If a problem occurs while adding any of the
680   *                             tool-specific arguments to the provided
681   *                             argument parser.
682   */
683  @Override()
684  public void addToolArguments(@NotNull final ArgumentParser parser)
685         throws ArgumentException
686  {
687    // We need to save a reference to the argument parser so that we can get
688    // the trailing arguments later.
689    argumentParser = parser;
690
691    // Add an argument that makes it possible to read a JSON-formatted access
692    // log file.
693    String description = "Indicates that the log file contains " +
694         "JSON-formatted log messages rather than text-formatted messages.";
695    json = new BooleanArgument(null, "json", description);
696    parser.addArgument(json);
697
698
699    // Add an argument that makes it possible to read a compressed log file.
700    // Note that this argument is no longer needed for dealing with compressed
701    // files, since the tool will automatically detect whether a file is
702    // compressed.  However, the argument is still provided for the purpose of
703    // backward compatibility.
704    description = "Indicates that the log file is compressed.";
705    isCompressed = new BooleanArgument('c', "isCompressed", description);
706    isCompressed.addLongIdentifier("is-compressed", true);
707    isCompressed.addLongIdentifier("compressed", true);
708    isCompressed.setHidden(true);
709    parser.addArgument(isCompressed);
710
711
712    // Add an argument that indicates that the tool should read the encryption
713    // passphrase from a file.
714    description = "Indicates that the log file is encrypted and that the " +
715         "encryption passphrase is contained in the specified file.  If " +
716         "the log data is encrypted and this argument is not provided, then " +
717         "the tool will interactively prompt for the encryption passphrase.";
718    encryptionPassphraseFile = new FileArgument(null,
719         "encryptionPassphraseFile", false, 1, null, description, true, true,
720         true, false);
721    encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file",
722         true);
723    encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true);
724    encryptionPassphraseFile.addLongIdentifier("encryption-password-file",
725         true);
726    parser.addArgument(encryptionPassphraseFile);
727
728
729    // Add an argument that indicates the number of values to display for each
730    // item being summarized.
731    description = "The number of values to display for each item being " +
732         "summarized.  A value of zero indicates that all items should be " +
733         "displayed.  If this is not provided, a default value of 20 will " +
734         "be used.";
735    reportCount = new IntegerArgument(null, "reportCount", false, 0, null,
736         description, 0, Integer.MAX_VALUE, 20);
737    reportCount.addLongIdentifier("report-count", true);
738    reportCount.addLongIdentifier("maximumCount", true);
739    reportCount.addLongIdentifier("maximum-count", true);
740    reportCount.addLongIdentifier("maxCount", true);
741    reportCount.addLongIdentifier("max-count", true);
742    reportCount.addLongIdentifier("count", true);
743    parser.addArgument(reportCount);
744
745
746    // Add an argument that indicates that the output should not be anonymized.
747    description = "Do not anonymize the output, but include actual attribute " +
748         "values in filters and DNs.  This will also have the effect of " +
749         "de-generifying those values, so output including the most common " +
750         "filters and DNs in some category will be specific instances of " +
751         "those filters and DNs instead of generic patterns.";
752    doNotAnonymize = new BooleanArgument(null, "doNotAnonymize", 1,
753         description);
754    doNotAnonymize.addLongIdentifier("do-not-anonymize", true);
755    doNotAnonymize.addLongIdentifier("deAnonymize", true);
756    doNotAnonymize.addLongIdentifier("de-anonymize", true);
757    parser.addArgument(doNotAnonymize);
758  }
759
760
761
762  /**
763   * Performs any necessary processing that should be done to ensure that the
764   * provided set of command-line arguments were valid.  This method will be
765   * called after the basic argument parsing has been performed and immediately
766   * before the {@link #doToolProcessing} method is invoked.
767   *
768   * @throws  ArgumentException  If there was a problem with the command-line
769   *                             arguments provided to this program.
770   */
771  @Override()
772  public void doExtendedArgumentValidation()
773         throws ArgumentException
774  {
775    // Make sure that at least one access log file path was provided.
776    final List<String> trailingArguments =
777         argumentParser.getTrailingArguments();
778    if ((trailingArguments == null) || trailingArguments.isEmpty())
779    {
780      throw new ArgumentException("No access log file paths were provided.");
781    }
782  }
783
784
785
786  /**
787   * Performs the core set of processing for this tool.
788   *
789   * @return  A result code that indicates whether the processing completed
790   *          successfully.
791   */
792  @Override()
793  @NotNull()
794  public ResultCode doToolProcessing()
795  {
796    int displayCount = reportCount.getValue();
797    if (displayCount <= 0)
798    {
799      displayCount = Integer.MAX_VALUE;
800    }
801
802    String encryptionPassphrase = null;
803    if (encryptionPassphraseFile.isPresent())
804    {
805      try
806      {
807        encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile(
808             encryptionPassphraseFile.getValue());
809      }
810      catch (final LDAPException e)
811      {
812        Debug.debugException(e);
813        err(e.getMessage());
814        return e.getResultCode();
815      }
816    }
817
818
819    long logLines = 0L;
820    for (final String path : argumentParser.getTrailingArguments())
821    {
822      final File f = new File(path);
823      out("Examining access log ", f.getAbsolutePath());
824      AccessLogReader reader = null;
825      InputStream inputStream = null;
826      try
827      {
828        inputStream = new FileInputStream(f);
829
830        final ObjectPair<InputStream,String> p =
831             ToolUtils.getPossiblyPassphraseEncryptedInputStream(inputStream,
832                  encryptionPassphrase,
833                  (! encryptionPassphraseFile.isPresent()),
834                  "Log file '" + path + "' is encrypted.  Please enter the " +
835                       "encryption passphrase:",
836                  "ERROR:  The provided passphrase was incorrect.",
837                  getOut(), getErr());
838        inputStream = p.getFirst();
839        if ((p.getSecond() != null) && (encryptionPassphrase == null))
840        {
841          encryptionPassphrase = p.getSecond();
842        }
843
844        if (isCompressed.isPresent())
845        {
846          inputStream = new GZIPInputStream(inputStream);
847        }
848        else
849        {
850          inputStream =
851               ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream);
852        }
853
854        if (json.isPresent())
855        {
856          reader = new JSONAccessLogReader(inputStream);
857        }
858        else
859        {
860          reader = new TextFormattedAccessLogReader(inputStream);
861        }
862      }
863      catch (final Exception e)
864      {
865        Debug.debugException(e);
866        err("Unable to open access log file ", f.getAbsolutePath(), ":  ",
867            StaticUtils.getExceptionMessage(e));
868        return ResultCode.LOCAL_ERROR;
869      }
870      finally
871      {
872        if ((reader == null) && (inputStream != null))
873        {
874          try
875          {
876            inputStream.close();
877          }
878          catch (final Exception e)
879          {
880            Debug.debugException(e);
881          }
882        }
883      }
884
885      long startTime = 0L;
886      long stopTime  = 0L;
887
888      while (true)
889      {
890        final AccessLogMessage msg;
891        try
892        {
893          msg = reader.readMessage();
894        }
895        catch (final IOException ioe)
896        {
897          Debug.debugException(ioe);
898          err("Error reading from access log file ", f.getAbsolutePath(),
899              ":  ", StaticUtils.getExceptionMessage(ioe));
900
901          if ((ioe.getCause() != null) &&
902               (ioe.getCause() instanceof BadPaddingException))
903          {
904            err("This error is likely because the log is encrypted and the " +
905                 "server still has the log file open.  It is recommended " +
906                 "that you only try to examine encrypted logs after they " +
907                 "have been rotated.  You can use the rotate-log tool to " +
908                 "force a rotation at any time.  Attempting to proceed with " +
909                 "just the data that was successfully read.");
910            break;
911          }
912          else
913          {
914            return ResultCode.LOCAL_ERROR;
915          }
916        }
917        catch (final LogException le)
918        {
919          Debug.debugException(le);
920          err("Encountered an error while attempting to parse a line in" +
921              "access log file ", f.getAbsolutePath(), ":  ",
922              StaticUtils.getExceptionMessage(le));
923          continue;
924        }
925
926        if (msg == null)
927        {
928          break;
929        }
930
931        logLines++;
932        stopTime = msg.getTimestamp().getTime();
933        if (startTime == 0L)
934        {
935          startTime = stopTime;
936        }
937
938        switch (msg.getMessageType())
939        {
940          case CONNECT:
941            processConnect((ConnectAccessLogMessage) msg);
942            break;
943          case SECURITY_NEGOTIATION:
944            processSecurityNegotiation(
945                 (SecurityNegotiationAccessLogMessage) msg);
946            break;
947          case DISCONNECT:
948            processDisconnect((DisconnectAccessLogMessage) msg);
949            break;
950          case REQUEST:
951            switch (((OperationRequestAccessLogMessage) msg).getOperationType())
952            {
953              case ABANDON:
954                processAbandonRequest((AbandonRequestAccessLogMessage) msg);
955                break;
956              case EXTENDED:
957                processExtendedRequest((ExtendedRequestAccessLogMessage) msg);
958                break;
959              case SEARCH:
960                processSearchRequest((SearchRequestAccessLogMessage) msg);
961                break;
962              case UNBIND:
963                processUnbindRequest((UnbindRequestAccessLogMessage) msg);
964                break;
965            }
966            break;
967          case RESULT:
968            switch (((OperationRequestAccessLogMessage) msg).getOperationType())
969            {
970              case ADD:
971                processAddResult((AddResultAccessLogMessage) msg);
972                break;
973              case BIND:
974                processBindResult((BindResultAccessLogMessage) msg);
975                break;
976              case COMPARE:
977                processCompareResult((CompareResultAccessLogMessage) msg);
978                break;
979              case DELETE:
980                processDeleteResult((DeleteResultAccessLogMessage) msg);
981                break;
982              case EXTENDED:
983                processExtendedResult((ExtendedResultAccessLogMessage) msg);
984                break;
985              case MODIFY:
986                processModifyResult((ModifyResultAccessLogMessage) msg);
987                break;
988              case MODDN:
989                processModifyDNResult((ModifyDNResultAccessLogMessage) msg);
990                break;
991              case SEARCH:
992                processSearchResult((SearchResultAccessLogMessage) msg);
993                break;
994            }
995            break;
996
997          case ASSURANCE_COMPLETE:
998          case CLIENT_CERTIFICATE:
999          case ENTRY_REBALANCING_REQUEST:
1000          case ENTRY_REBALANCING_RESULT:
1001          case FORWARD:
1002          case FORWARD_FAILED:
1003          case ENTRY:
1004          case REFERENCE:
1005          default:
1006            // Nothing needs to be done for these message types.
1007        }
1008      }
1009
1010      try
1011      {
1012        reader.close();
1013      }
1014      catch (final Exception e)
1015      {
1016        Debug.debugException(e);
1017      }
1018      logDurationMillis += (stopTime - startTime);
1019
1020
1021      // If there are any outstanding authentication failures, then update the
1022      // set of consecutive failures as appropriate.
1023      for (final Map.Entry<String,AtomicLong> e :
1024           outstandingFailedBindDNs.entrySet())
1025      {
1026        final String dn = e.getKey();
1027        final AtomicLong outstandingFailureCount = e.getValue();
1028        final AtomicLong consecutiveFailures =
1029             consecutiveFailedBindsByDN.get(dn);
1030        if ((consecutiveFailures == null) ||
1031           (outstandingFailureCount.get() > consecutiveFailures.get()))
1032        {
1033          consecutiveFailedBindsByDN.put(dn, outstandingFailureCount);
1034        }
1035      }
1036      outstandingFailedBindDNs.clear();
1037    }
1038
1039
1040    final int numFiles = argumentParser.getTrailingArguments().size();
1041    out();
1042    out("Examined ", logLines, " lines in ", numFiles,
1043        ((numFiles == 1) ? " file" : " files"),
1044        " covering a total duration of ",
1045        StaticUtils.millisToHumanReadableDuration(logDurationMillis));
1046    if (logLines == 0)
1047    {
1048      return ResultCode.SUCCESS;
1049    }
1050
1051    out();
1052
1053    final double logDurationSeconds   = logDurationMillis / 1_000.0;
1054    final double connectsPerSecond    = numConnects / logDurationSeconds;
1055    final double disconnectsPerSecond = numDisconnects / logDurationSeconds;
1056
1057    out("Total connections established:  ", numConnects, " (",
1058        decimalFormat.format(connectsPerSecond), "/second)");
1059    out("Total disconnects:  ", numDisconnects, " (",
1060        decimalFormat.format(disconnectsPerSecond), "/second)");
1061
1062    printCounts(clientAddresses, "Most common client addresses:", "address",
1063         "addresses");
1064
1065    printCounts(clientConnectionPolicies,
1066         "Most common client connection policies:", "policy", "policies");
1067
1068    printCounts(tlsProtocols, "Most common TLS protocol versions:", "version",
1069         "versions");
1070
1071    printCounts(tlsCipherSuites, "Most common TLS cipher suites:",
1072         "cipher suite", "cipher suites");
1073
1074    printCounts(disconnectReasons, "Most common disconnect reasons:", "reason",
1075         "reasons");
1076
1077    final long totalOps = numAbandons + numAdds + numBinds + numCompares +
1078         numDeletes + numExtended + numModifies + numModifyDNs + numSearches +
1079         numUnbinds;
1080    final long totalResults = totalOps - numAbandons - numUnbinds;
1081
1082    if (totalOps > 0)
1083    {
1084      final double percentAbandon  = 100.0 * numAbandons / totalOps;
1085      final double percentAdd      = 100.0 * numAdds / totalOps;
1086      final double percentBind     = 100.0 * numBinds / totalOps;
1087      final double percentCompare  = 100.0 * numCompares / totalOps;
1088      final double percentDelete   = 100.0 * numDeletes / totalOps;
1089      final double percentExtended = 100.0 * numExtended / totalOps;
1090      final double percentModify   = 100.0 * numModifies / totalOps;
1091      final double percentModifyDN = 100.0 * numModifyDNs / totalOps;
1092      final double percentSearch   = 100.0 * numSearches / totalOps;
1093      final double percentUnbind   = 100.0 * numUnbinds / totalOps;
1094
1095      final double abandonsPerSecond  = numAbandons / logDurationSeconds;
1096      final double addsPerSecond      = numAdds / logDurationSeconds;
1097      final double bindsPerSecond     = numBinds / logDurationSeconds;
1098      final double comparesPerSecond  = numCompares / logDurationSeconds;
1099      final double deletesPerSecond   = numDeletes / logDurationSeconds;
1100      final double extendedPerSecond  = numExtended / logDurationSeconds;
1101      final double modifiesPerSecond  = numModifies / logDurationSeconds;
1102      final double modifyDNsPerSecond = numModifyDNs / logDurationSeconds;
1103      final double searchesPerSecond  = numSearches / logDurationSeconds;
1104      final double unbindsPerSecond   = numUnbinds / logDurationSeconds;
1105
1106      out();
1107      out("Total operations examined:  ", totalOps);
1108      out("Abandon operations examined:  ", numAbandons, " (",
1109          decimalFormat.format(percentAbandon), "%, ",
1110          decimalFormat.format(abandonsPerSecond), "/second)");
1111      out("Add operations examined:  ", numAdds, " (",
1112          decimalFormat.format(percentAdd), "%, ",
1113          decimalFormat.format(addsPerSecond), "/second)");
1114      out("Bind operations examined:  ", numBinds, " (",
1115          decimalFormat.format(percentBind), "%, ",
1116          decimalFormat.format(bindsPerSecond), "/second)");
1117      out("Compare operations examined:  ", numCompares, " (",
1118          decimalFormat.format(percentCompare), "%, ",
1119          decimalFormat.format(comparesPerSecond), "/second)");
1120      out("Delete operations examined:  ", numDeletes, " (",
1121          decimalFormat.format(percentDelete), "%, ",
1122          decimalFormat.format(deletesPerSecond), "/second)");
1123      out("Extended operations examined:  ", numExtended, " (",
1124          decimalFormat.format(percentExtended), "%, ",
1125          decimalFormat.format(extendedPerSecond), "/second)");
1126      out("Modify operations examined:  ", numModifies, " (",
1127          decimalFormat.format(percentModify), "%, ",
1128          decimalFormat.format(modifiesPerSecond), "/second)");
1129      out("Modify DN operations examined:  ", numModifyDNs, " (",
1130          decimalFormat.format(percentModifyDN), "%, ",
1131          decimalFormat.format(modifyDNsPerSecond), "/second)");
1132      out("Search operations examined:  ", numSearches, " (",
1133          decimalFormat.format(percentSearch), "%, ",
1134          decimalFormat.format(searchesPerSecond), "/second)");
1135      out("Unbind operations examined:  ", numUnbinds, " (",
1136          decimalFormat.format(percentUnbind), "%, ",
1137          decimalFormat.format(unbindsPerSecond), "/second)");
1138
1139      final double totalProcessingDuration = addProcessingDuration +
1140           bindProcessingDuration + compareProcessingDuration +
1141           deleteProcessingDuration + extendedProcessingDuration +
1142           modifyProcessingDuration + modifyDNProcessingDuration +
1143           searchProcessingDuration;
1144
1145      out();
1146      out("Average operation processing duration:  ",
1147          decimalFormat.format(totalProcessingDuration / totalOps), "ms");
1148
1149      if (numAdds > 0)
1150      {
1151        out("Average add operation processing duration:  ",
1152            decimalFormat.format(addProcessingDuration / numAdds), "ms");
1153      }
1154
1155      if (numBinds > 0)
1156      {
1157        out("Average bind operation processing duration:  ",
1158            decimalFormat.format(bindProcessingDuration / numBinds), "ms");
1159      }
1160
1161      if (numCompares > 0)
1162      {
1163        out("Average compare operation processing duration:  ",
1164            decimalFormat.format(compareProcessingDuration / numCompares),
1165            "ms");
1166      }
1167
1168      if (numDeletes > 0)
1169      {
1170        out("Average delete operation processing duration:  ",
1171            decimalFormat.format(deleteProcessingDuration / numDeletes), "ms");
1172      }
1173
1174      if (numExtended > 0)
1175      {
1176        out("Average extended operation processing duration:  ",
1177            decimalFormat.format(extendedProcessingDuration / numExtended),
1178            "ms");
1179      }
1180
1181      if (numModifies > 0)
1182      {
1183        out("Average modify operation processing duration:  ",
1184            decimalFormat.format(modifyProcessingDuration / numModifies), "ms");
1185      }
1186
1187      if (numModifyDNs > 0)
1188      {
1189        out("Average modify DN operation processing duration:  ",
1190            decimalFormat.format(modifyDNProcessingDuration / numModifyDNs),
1191            "ms");
1192      }
1193
1194      if (numSearches > 0)
1195      {
1196        out("Average search operation processing duration:  ",
1197            decimalFormat.format(searchProcessingDuration / numSearches), "ms");
1198      }
1199
1200      printProcessingTimeHistogram("add", numAdds, addProcessingTimes);
1201      printProcessingTimeHistogram("bind", numBinds, bindProcessingTimes);
1202      printProcessingTimeHistogram("compare", numCompares,
1203                                   compareProcessingTimes);
1204      printProcessingTimeHistogram("delete", numDeletes, deleteProcessingTimes);
1205      printProcessingTimeHistogram("extended", numExtended,
1206                                   extendedProcessingTimes);
1207      printProcessingTimeHistogram("modify", numModifies,
1208                                   modifyProcessingTimes);
1209      printProcessingTimeHistogram("modify DN", numModifyDNs,
1210                                 modifyDNProcessingTimes);
1211      printProcessingTimeHistogram("search", numSearches,
1212                                   searchProcessingTimes);
1213
1214      if (totalWorkQueueWaitTime > 0L)
1215      {
1216        out();
1217        out("Average work queue wait time:  ",
1218             decimalFormat.format(totalWorkQueueWaitTime / totalResults), "ms");
1219        printHistogram("Count of operations by work queue wait time:",
1220             totalResults, workQueueWaitTimes);
1221      }
1222
1223      printResultCodeCounts(addResultCodes, "add");
1224      printResultCodeCounts(bindResultCodes, "bind");
1225      printResultCodeCounts(compareResultCodes, "compare");
1226      printResultCodeCounts(deleteResultCodes, "delete");
1227      printResultCodeCounts(extendedResultCodes, "extended");
1228      printResultCodeCounts(modifyResultCodes, "modify");
1229      printResultCodeCounts(modifyDNResultCodes, "modify DN");
1230      printResultCodeCounts(searchResultCodes, "search");
1231
1232      printCounts(preAuthzPrivilegesUsed,
1233           "Most common pre-authorization privileges used:", "privilege",
1234           "privileges");
1235      printCounts(privilegesUsed, "Most common privileges used:", "privilege",
1236           "privileges");
1237      printCounts(privilegesMissing, "Most common missing privileges:",
1238           "privilege", "privileges");
1239
1240      printCounts(successfulBindDNs,
1241           "Most common bind DNs used in successful authentication attempts:",
1242           "DN", "DNs");
1243      printCounts(bindFailuresByDN,
1244           "Most common bind DNs used in failed authentication attempts:",
1245           "DN", "DNs");
1246      printCounts(bindFailuresByIPAddress,
1247           "Most common IP addresses used in failed authentication attempts:",
1248           "IP", "IPs");
1249      if (doNotAnonymize.isPresent())
1250      {
1251        printCounts(consecutiveFailedBindsByDN,
1252             "Bind DNs with the most consecutive authentication failures:",
1253             "DN", "DNs");
1254      }
1255      printCounts(authenticationTypes, "Most common authentication types:",
1256           "authentication type", "authentication types");
1257
1258      long numResultsWithAuthzID = 0L;
1259      for (final AtomicLong l : authzDNs.values())
1260      {
1261        numResultsWithAuthzID += l.get();
1262      }
1263
1264      out();
1265      final double percentWithAuthzID =
1266           100.0 * numResultsWithAuthzID / totalOps;
1267      out("Number of operations with an alternate authorization identity:  ",
1268           numResultsWithAuthzID, " (",
1269           decimalFormat.format(percentWithAuthzID), "%)");
1270
1271      printCounts(authzDNs, "Most common alternate authorization identity DNs:",
1272           "DN", "DNs");
1273
1274      if (! requestControlOIDs.isEmpty())
1275      {
1276        final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>();
1277        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1278        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1279        getMostCommonElements(requestControlOIDs, controlCounts, displayCount,
1280             skippedWithSameCount, skippedWithLowerCount);
1281
1282        out();
1283        out("Most common request control types:");
1284
1285        long count = -1L;
1286        for (final ObjectPair<String,Long> p : controlCounts)
1287        {
1288          count = p.getSecond();
1289          final double percent = 100.0 * count / numRequestControls;
1290
1291          final String oid = p.getFirst();
1292          final OIDRegistryItem item = OIDRegistry.getDefault().get(oid);
1293          if (item == null)
1294          {
1295            out(p.getFirst(), ":  ", p.getSecond(), " (",
1296                 decimalFormat.format(percent), "%)");
1297          }
1298          else
1299          {
1300            out(p.getFirst(), " (", item.getName(), "):  ", p.getSecond(), " (",
1301                 decimalFormat.format(percent), "%)");
1302          }
1303        }
1304
1305        if (skippedWithSameCount.get() > 0L)
1306        {
1307          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1308               getSingularOrPlural(skippedWithSameCount.get(), "control",
1309                    "controls") +
1310               " with a count of " + count + " }");
1311        }
1312
1313        if (skippedWithLowerCount.get() > 0L)
1314        {
1315          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1316               getSingularOrPlural(skippedWithLowerCount.get(), "control",
1317                    "controls") +
1318               " with a count that is less than " + count + " }");
1319        }
1320      }
1321
1322      if (! responseControlOIDs.isEmpty())
1323      {
1324        final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>();
1325        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1326        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1327        getMostCommonElements(responseControlOIDs, controlCounts, displayCount,
1328             skippedWithSameCount, skippedWithLowerCount);
1329
1330        out();
1331        out("Most common response control types:");
1332
1333        long count = -1L;
1334        for (final ObjectPair<String,Long> p : controlCounts)
1335        {
1336          count = p.getSecond();
1337          final double percent = 100.0 * count / numResponseControls;
1338
1339          final String oid = p.getFirst();
1340          final OIDRegistryItem item = OIDRegistry.getDefault().get(oid);
1341          if (item == null)
1342          {
1343            out(p.getFirst(), ":  ", p.getSecond(), " (",
1344                 decimalFormat.format(percent), "%)");
1345          }
1346          else
1347          {
1348            out(p.getFirst(), " (", item.getName(), "):  ", p.getSecond(), " (",
1349                 decimalFormat.format(percent), "%)");
1350          }
1351        }
1352
1353        if (skippedWithSameCount.get() > 0L)
1354        {
1355          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1356               getSingularOrPlural(skippedWithSameCount.get(), "control",
1357                    "controls") +
1358               " with a count of " + count + " }");
1359        }
1360
1361        if (skippedWithLowerCount.get() > 0L)
1362        {
1363          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1364               getSingularOrPlural(skippedWithLowerCount.get(), "control",
1365                    "controls") +
1366               " with a count that is less than " + count + " }");
1367        }
1368      }
1369
1370      if (! extendedOperations.isEmpty())
1371      {
1372        final List<ObjectPair<String,Long>> extOpCounts = new ArrayList<>();
1373        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1374        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1375        getMostCommonElements(extendedOperations, extOpCounts, displayCount,
1376             skippedWithSameCount, skippedWithLowerCount);
1377
1378        out();
1379        out("Most common extended operation types:");
1380
1381        long count = -1L;
1382        for (final ObjectPair<String,Long> p : extOpCounts)
1383        {
1384          count = p.getSecond();
1385          final double percent = 100.0 * count / numExtended;
1386
1387          final String oid = p.getFirst();
1388          final String name = extendedOperationOIDsToNames.get(oid);
1389          if (name == null)
1390          {
1391            out(p.getFirst(), ":  ", p.getSecond(), " (",
1392                 decimalFormat.format(percent), "%)");
1393          }
1394          else
1395          {
1396            out(p.getFirst(), " (", name, "):  ", p.getSecond(), " (",
1397                 decimalFormat.format(percent), "%)");
1398          }
1399        }
1400
1401        if (skippedWithSameCount.get() > 0L)
1402        {
1403          out("{ Skipped " + skippedWithSameCount.get() +
1404               " additional extended " +
1405               getSingularOrPlural(skippedWithSameCount.get(), "operation",
1406                    "operations") +
1407               " with a count of " + count + " }");
1408        }
1409
1410        if (skippedWithLowerCount.get() > 0L)
1411        {
1412          out("{ Skipped " + skippedWithLowerCount.get() +
1413               " additional extended " +
1414               getSingularOrPlural(skippedWithLowerCount.get(), "operation",
1415                    "operations") +
1416               " with a count that is less than " + count + " }");
1417        }
1418      }
1419
1420      out();
1421      out("Number of unindexed search attempts:  ", numUnindexedAttempts);
1422      out("Number of successfully-completed unindexed searches:  ",
1423           numUnindexedSuccessful);
1424      out("Number of failed unindexed searches:  ", numUnindexedFailed);
1425
1426      printCounts(unindexedFilters, "Most common unindexed search filters:",
1427           "filter", "filters");
1428
1429      if (! searchScopes.isEmpty())
1430      {
1431        final List<ObjectPair<SearchScope,Long>> scopeCounts =
1432             new ArrayList<>();
1433        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1434        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1435        getMostCommonElements(searchScopes, scopeCounts, displayCount,
1436             skippedWithSameCount, skippedWithLowerCount);
1437
1438        out();
1439        out("Most common search scopes:");
1440
1441        long count = -1L;
1442        for (final ObjectPair<SearchScope,Long> p : scopeCounts)
1443        {
1444          count = p.getSecond();
1445          final double percent = 100.0 * count / numSearches;
1446          out(p.getFirst().getName().toLowerCase(), " (",
1447               p.getFirst().intValue(), "):  ", p.getSecond(), " (",
1448               decimalFormat.format(percent), "%)");
1449        }
1450
1451        if (skippedWithSameCount.get() > 0L)
1452        {
1453          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1454               getSingularOrPlural(skippedWithSameCount.get(), "scope",
1455                    "scopes") +
1456               " with a count of " + count + " }");
1457        }
1458
1459        if (skippedWithLowerCount.get() > 0L)
1460        {
1461          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1462               getSingularOrPlural(skippedWithLowerCount.get(), "scope",
1463                    "scopes") +
1464               " with a count that is less than " + count + " }");
1465        }
1466      }
1467
1468      if (! searchEntryCounts.isEmpty())
1469      {
1470        final List<ObjectPair<Long,Long>> entryCounts = new ArrayList<>();
1471        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1472        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1473        getMostCommonElements(searchEntryCounts, entryCounts, displayCount,
1474             skippedWithSameCount, skippedWithLowerCount);
1475
1476        out();
1477        out("Most common search entry counts:");
1478
1479        long count = -1L;
1480        for (final ObjectPair<Long,Long> p : entryCounts)
1481        {
1482          count = p.getSecond();
1483          final double percent = 100.0 * count / numSearches;
1484          out(p.getFirst(), " matching ",
1485               getSingularOrPlural(p.getFirst(), "entry", "entries"),
1486               ":  ", p.getSecond(), " (", decimalFormat.format(percent), "%)");
1487        }
1488
1489        if (skippedWithSameCount.get() > 0L)
1490        {
1491          out("{ Skipped " + skippedWithSameCount.get() + " additional entry " +
1492               getSingularOrPlural(skippedWithSameCount.get(), "count",
1493                    "counts") +
1494               " with a count of " + count + " }");
1495        }
1496
1497        if (skippedWithLowerCount.get() > 0L)
1498        {
1499          out("{ Skipped " + skippedWithLowerCount.get() +
1500               " additional entry " +
1501               getSingularOrPlural(skippedWithLowerCount.get(), "count",
1502                    "counts") +
1503               " with a count that is less than " + count + " }");
1504        }
1505      }
1506
1507      printCounts(searchBaseDNs,
1508           "Most common base DNs for searches with a non-base scope:",
1509           "base DN", "base DNs");
1510
1511      printCounts(filterTypes,
1512           "Most common filters for searches with a non-base scope:",
1513           "filter", "filters");
1514
1515      printCounts(filterComponentCounts,
1516           "Most common search filter component counts:", "filter",
1517           "filters");
1518
1519      if (doNotAnonymize.isPresent() &&
1520           (! filtersRepresentingPotentialInjectionAttempt.isEmpty()))
1521      {
1522        out();
1523        wrapOut(0, WRAP_COLUMN,
1524             "Search filters that may indicate an unsuccessful injection " +
1525                  "attempt.  These include filters with an assertion value " +
1526                  "that contains one or more of the following:  parentheses, " +
1527                  "ampersands, pipes, single quotes, double quotes, or the " +
1528                  "words 'select' and 'from':");
1529        for (final Filter f : filtersRepresentingPotentialInjectionAttempt)
1530        {
1531          out("* " + f.toString());
1532        }
1533      }
1534
1535      if (numSearches > 0L)
1536      {
1537        long numSearchesMatchingNoEntries = 0L;
1538        for (final AtomicLong l : noEntryFilters.values())
1539        {
1540          numSearchesMatchingNoEntries += l.get();
1541        }
1542
1543        out();
1544        final double noEntryPercent =
1545             100.0 * numSearchesMatchingNoEntries / numSearches;
1546        out("Number of searches matching no entries:  ",
1547             numSearchesMatchingNoEntries, " (",
1548             decimalFormat.format(noEntryPercent), "%)");
1549
1550        printCounts(noEntryFilters,
1551             "Most common filters for searches matching no entries:",
1552             "filter", "filters");
1553
1554
1555        long numSearchesMatchingOneEntry = 0L;
1556        for (final AtomicLong l : oneEntryFilters.values())
1557        {
1558          numSearchesMatchingOneEntry += l.get();
1559        }
1560
1561        out();
1562        final double oneEntryPercent =
1563             100.0 * numSearchesMatchingOneEntry / numSearches;
1564        out("Number of searches matching one entry:  ",
1565             numSearchesMatchingOneEntry, " (",
1566             decimalFormat.format(oneEntryPercent), "%)");
1567
1568        printCounts(oneEntryFilters,
1569             "Most common filters for searches matching one entry:",
1570             "filter", "filters");
1571
1572
1573        long numSearchesMatchingMultipleEntries = 0L;
1574        for (final AtomicLong l : multiEntryFilters.values())
1575        {
1576          numSearchesMatchingMultipleEntries += l.get();
1577        }
1578
1579        out();
1580        final double multiEntryPercent =
1581             100.0 * numSearchesMatchingMultipleEntries / numSearches;
1582        out("Number of searches matching multiple entries:  ",
1583             numSearchesMatchingMultipleEntries, " (",
1584             decimalFormat.format(multiEntryPercent), "%)");
1585
1586        printCounts(multiEntryFilters,
1587             "Most common filters for searches matching multiple entries:",
1588             "filter", "filters");
1589      }
1590    }
1591
1592    if (! mostExpensiveFilters.isEmpty())
1593    {
1594        final List<ObjectPair<String,Long>> filterDurations = new ArrayList<>();
1595        final AtomicLong skippedWithSameCount = new AtomicLong(0L);
1596        final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
1597        getMostCommonElements(mostExpensiveFilters, filterDurations,
1598             displayCount, skippedWithSameCount, skippedWithLowerCount);
1599
1600        out();
1601        out("Filters for searches with the longest processing times:");
1602
1603        String durationStr = "";
1604        for (final ObjectPair<String,Long> p : filterDurations)
1605        {
1606          final long durationMicros = p.getSecond();
1607          final double durationMillis = durationMicros / 1_000.0;
1608          durationStr = decimalFormat.format(durationMillis) + " ms";
1609          out(p.getFirst(), ":  ", durationStr);
1610        }
1611
1612        if (skippedWithSameCount.get() > 0L)
1613        {
1614          out("{ Skipped " + skippedWithSameCount.get() + " additional " +
1615               getSingularOrPlural(skippedWithSameCount.get(), "filter",
1616                    "filters") +
1617               " with a duration of " + durationStr + " }");
1618        }
1619
1620        if (skippedWithLowerCount.get() > 0L)
1621        {
1622          out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
1623               getSingularOrPlural(skippedWithLowerCount.get(), "filter",
1624                    "filters") +
1625               " with a duration that is less than " + durationStr + " }");
1626        }
1627    }
1628
1629    final long totalUncached = numUncachedAdds + numUncachedBinds +
1630         numUncachedCompares + numUncachedDeletes + numUncachedExtended +
1631         numUncachedModifies + numUncachedModifyDNs + numUncachedSearches;
1632    if (totalUncached > 0L)
1633    {
1634      out();
1635      out("Operations accessing uncached data:");
1636      printUncached("Add", numUncachedAdds, numAdds);
1637      printUncached("Bind", numUncachedBinds, numBinds);
1638      printUncached("Compare", numUncachedCompares, numCompares);
1639      printUncached("Delete", numUncachedDeletes, numDeletes);
1640      printUncached("Extended", numUncachedExtended, numExtended);
1641      printUncached("Modify", numUncachedModifies, numModifies);
1642      printUncached("Modify DN", numUncachedModifyDNs, numModifyDNs);
1643      printUncached("Search", numUncachedSearches, numSearches);
1644    }
1645
1646
1647    return ResultCode.SUCCESS;
1648  }
1649
1650
1651
1652  /**
1653   * Retrieves a set of information that may be used to generate example usage
1654   * information.  Each element in the returned map should consist of a map
1655   * between an example set of arguments and a string that describes the
1656   * behavior of the tool when invoked with that set of arguments.
1657   *
1658   * @return  A set of information that may be used to generate example usage
1659   *          information.  It may be {@code null} or empty if no example usage
1660   *          information is available.
1661   */
1662  @Override()
1663  @NotNull()
1664  public LinkedHashMap<String[],String> getExampleUsages()
1665  {
1666    final LinkedHashMap<String[],String> examples =
1667         new LinkedHashMap<>(StaticUtils.computeMapCapacity(1));
1668
1669    final String[] args =
1670    {
1671      "/ds/logs/access"
1672    };
1673    final String description =
1674         "Analyze the contents of the /ds/logs/access access log file.";
1675    examples.put(args, description);
1676
1677    return examples;
1678  }
1679
1680
1681
1682  /**
1683   * Populates the provided processing time map with an initial set of values.
1684   *
1685   * @param  m  The processing time map to be populated.
1686   */
1687  private static void populateProcessingTimeMap(
1688                           @NotNull final HashMap<Long,AtomicLong> m)
1689  {
1690    m.put(1L, new AtomicLong(0L));
1691    m.put(2L, new AtomicLong(0L));
1692    m.put(3L, new AtomicLong(0L));
1693    m.put(5L, new AtomicLong(0L));
1694    m.put(10L, new AtomicLong(0L));
1695    m.put(20L, new AtomicLong(0L));
1696    m.put(30L, new AtomicLong(0L));
1697    m.put(50L, new AtomicLong(0L));
1698    m.put(100L, new AtomicLong(0L));
1699    m.put(1_000L, new AtomicLong(0L));
1700    m.put(2_000L, new AtomicLong(0L));
1701    m.put(3_000L, new AtomicLong(0L));
1702    m.put(5_000L, new AtomicLong(0L));
1703    m.put(10_000L, new AtomicLong(0L));
1704    m.put(20_000L, new AtomicLong(0L));
1705    m.put(30_000L, new AtomicLong(0L));
1706    m.put(60_000L, new AtomicLong(0L));
1707    m.put(Long.MAX_VALUE, new AtomicLong(0L));
1708  }
1709
1710
1711
1712  /**
1713   * Performs any necessary processing for a connect message.
1714   *
1715   * @param  m  The log message to be processed.
1716   */
1717  private void processConnect(@NotNull final ConnectAccessLogMessage m)
1718  {
1719    numConnects++;
1720
1721    final String clientAddr = m.getSourceAddress();
1722    if (clientAddr != null)
1723    {
1724      final Long connectionID = m.getConnectionID();
1725      if (connectionID != null)
1726      {
1727        ipAddressesByConnectionID.put(connectionID, clientAddr);
1728      }
1729
1730      AtomicLong count = clientAddresses.get(clientAddr);
1731      if (count == null)
1732      {
1733        count = new AtomicLong(0L);
1734        clientAddresses.put(clientAddr, count);
1735      }
1736      count.incrementAndGet();
1737    }
1738
1739    final String ccp = m.getClientConnectionPolicy();
1740    if (ccp != null)
1741    {
1742      AtomicLong l = clientConnectionPolicies.get(ccp);
1743      if (l == null)
1744      {
1745        l = new AtomicLong(0L);
1746        clientConnectionPolicies.put(ccp, l);
1747      }
1748      l.incrementAndGet();
1749    }
1750  }
1751
1752
1753
1754  /**
1755   * Performs any necessary processing for a security negotiation message.
1756   *
1757   * @param  m  The log message to be processed.
1758   */
1759  private void processSecurityNegotiation(
1760                    @NotNull final SecurityNegotiationAccessLogMessage m)
1761  {
1762    final String protocol = m.getProtocol();
1763    if (protocol != null)
1764    {
1765      AtomicLong l = tlsProtocols.get(protocol);
1766      if (l == null)
1767      {
1768        l = new AtomicLong(0L);
1769        tlsProtocols.put(protocol, l);
1770      }
1771      l.incrementAndGet();
1772    }
1773
1774    final String cipherSuite = m.getCipher();
1775    if (cipherSuite != null)
1776    {
1777      AtomicLong l = tlsCipherSuites.get(cipherSuite);
1778      if (l == null)
1779      {
1780        l = new AtomicLong(0L);
1781        tlsCipherSuites.put(cipherSuite, l);
1782      }
1783      l.incrementAndGet();
1784    }
1785  }
1786
1787
1788
1789  /**
1790   * Performs any necessary processing for a disconnect message.
1791   *
1792   * @param  m  The log message to be processed.
1793   */
1794  private void processDisconnect(@NotNull final DisconnectAccessLogMessage m)
1795  {
1796    numDisconnects++;
1797
1798    final Long connectionID = m.getConnectionID();
1799    if (connectionID != null)
1800    {
1801      ipAddressesByConnectionID.remove(connectionID);
1802    }
1803
1804    final String reason = m.getDisconnectReason();
1805    if (reason != null)
1806    {
1807      AtomicLong l = disconnectReasons.get(reason);
1808      if (l == null)
1809      {
1810        l = new AtomicLong(0L);
1811        disconnectReasons.put(reason, l);
1812      }
1813      l.incrementAndGet();
1814    }
1815  }
1816
1817
1818
1819  /**
1820   * Performs any necessary processing for an abandon request message.
1821   *
1822   * @param  m  The log message to be processed.
1823   */
1824  private void processAbandonRequest(
1825                    @NotNull final AbandonRequestAccessLogMessage m)
1826  {
1827    numAbandons++;
1828  }
1829
1830
1831
1832  /**
1833   * Performs any necessary processing for an extended request message.
1834   *
1835   * @param  m  The log message to be processed.
1836   */
1837  private void processExtendedRequest(
1838                    @NotNull final ExtendedRequestAccessLogMessage m)
1839  {
1840    processedRequests.add(m.getConnectionID() + "-" + m.getOperationID());
1841    processExtendedRequestInternal(m);
1842  }
1843
1844
1845
1846  /**
1847   * Performs the internal processing for an extended request message.
1848   *
1849   * @param  m  The log message to be processed.
1850   */
1851  private void processExtendedRequestInternal(
1852                    @NotNull final ExtendedRequestAccessLogMessage m)
1853  {
1854    final String oid = m.getRequestOID();
1855    if (oid != null)
1856    {
1857      AtomicLong l = extendedOperations.get(oid);
1858      if (l == null)
1859      {
1860        l  = new AtomicLong(0L);
1861        extendedOperations.put(oid, l);
1862      }
1863      l.incrementAndGet();
1864
1865      final String requestType = m.getRequestType();
1866      if ((requestType != null) &&
1867           (! extendedOperationOIDsToNames.containsKey(oid)))
1868      {
1869        extendedOperationOIDsToNames.put(oid, requestType);
1870      }
1871    }
1872  }
1873
1874
1875
1876  /**
1877   * Performs any necessary processing for a search request message.
1878   *
1879   * @param  m  The log message to be processed.
1880   */
1881  private void processSearchRequest(
1882                    @NotNull final SearchRequestAccessLogMessage m)
1883  {
1884    processedRequests.add(m.getConnectionID() + "-" + m.getOperationID());
1885    processSearchRequestInternal(m);
1886  }
1887
1888
1889
1890  /**
1891   * Performs any necessary processing for a search request message.
1892   *
1893   * @param  m  The log message to be processed.
1894   */
1895  private void processSearchRequestInternal(
1896                    @NotNull final SearchRequestAccessLogMessage m)
1897  {
1898    final SearchScope scope = m.getScope();
1899    if (scope != null)
1900    {
1901      AtomicLong scopeCount = searchScopes.get(scope);
1902      if (scopeCount == null)
1903      {
1904        scopeCount = new AtomicLong(0L);
1905        searchScopes.put(scope, scopeCount);
1906      }
1907      scopeCount.incrementAndGet();
1908
1909      if (! scope.equals(SearchScope.BASE))
1910      {
1911        final String filterString = prepareFilter(m.getFilter());
1912        if (filterString != null)
1913        {
1914          AtomicLong filterCount = filterTypes.get(filterString);
1915          if (filterCount == null)
1916          {
1917            filterCount = new AtomicLong(0L);
1918            filterTypes.put(filterString, filterCount);
1919          }
1920          filterCount.incrementAndGet();
1921
1922
1923          final String baseDN = getDNString(m.getBaseDN());
1924          if (baseDN != null)
1925          {
1926            AtomicLong baseDNCount = searchBaseDNs.get(baseDN);
1927            if (baseDNCount == null)
1928            {
1929              baseDNCount = new AtomicLong(0L);
1930              searchBaseDNs.put(baseDN, baseDNCount);
1931            }
1932            baseDNCount.incrementAndGet();
1933          }
1934        }
1935      }
1936    }
1937
1938    final String filterString = m.getFilter();
1939    if (filterString != null)
1940    {
1941      try
1942      {
1943        final Filter filter = Filter.create(filterString);
1944        if (mayRepresentInjectionAttempt(filter))
1945        {
1946          filtersRepresentingPotentialInjectionAttempt.add(filter);
1947        }
1948
1949
1950        final int numComponents = countComponents(filter);
1951        final String label;
1952        if (numComponents == 1)
1953        {
1954          label = "1 component";
1955        }
1956        else
1957        {
1958          label = numComponents + " components";
1959        }
1960
1961        AtomicLong count = filterComponentCounts.get(label);
1962        if (count == null)
1963        {
1964          count = new AtomicLong(0L);
1965          filterComponentCounts.put(label, count);
1966        }
1967
1968        count.incrementAndGet();
1969      }
1970      catch (final Exception e)
1971      {
1972        Debug.debugException(e);
1973      }
1974    }
1975  }
1976
1977
1978
1979  /**
1980   * Indicates whether the provided search filter may represent an injection
1981   * attempt.  Filters that may represent injection attempts include:
1982   * <UL>
1983   *   <LI>Filters with assertion values that contain parentheses, ampersands,
1984   *       pipes, or single or double quotes.</LI>
1985   *   <LI>Filters that contain the words "select" and "from".</LI>
1986   * </UL>
1987   *
1988   * @param  filter  The filter to examine.  It must not be {@code null}.
1989   *
1990   * @return  {@code true} if the provided filter may represent an injection
1991   *          attempt, or {@code false} if not.
1992   */
1993  static boolean mayRepresentInjectionAttempt(@NotNull final Filter filter)
1994  {
1995    switch (filter.getFilterType())
1996    {
1997      case Filter.FILTER_TYPE_AND:
1998      case Filter.FILTER_TYPE_OR:
1999        for (final Filter f : filter.getComponents())
2000        {
2001          if (mayRepresentInjectionAttempt(f))
2002          {
2003            return true;
2004          }
2005        }
2006        return false;
2007
2008      case Filter.FILTER_TYPE_NOT:
2009        return mayRepresentInjectionAttempt(filter.getNOTComponent());
2010
2011      case Filter.FILTER_TYPE_EQUALITY:
2012      case Filter.FILTER_TYPE_GREATER_OR_EQUAL:
2013      case Filter.FILTER_TYPE_LESS_OR_EQUAL:
2014      case Filter.FILTER_TYPE_APPROXIMATE_MATCH:
2015      case Filter.FILTER_TYPE_EXTENSIBLE_MATCH:
2016        return mayRepresentInjectionAttempt(filter.getAssertionValue());
2017
2018      case Filter.FILTER_TYPE_SUBSTRING:
2019        final String[] subAnyStrings = filter.getSubAnyStrings();
2020        if (subAnyStrings != null)
2021        {
2022          for (final String subAnyString : subAnyStrings)
2023          {
2024            if (mayRepresentInjectionAttempt(subAnyString))
2025            {
2026              return true;
2027            }
2028          }
2029        }
2030
2031        return mayRepresentInjectionAttempt(filter.getSubInitialString()) ||
2032             mayRepresentInjectionAttempt(filter.getSubFinalString());
2033
2034      case Filter.FILTER_TYPE_PRESENCE:
2035      default:
2036        return false;
2037    }
2038  }
2039
2040
2041
2042  /**
2043   * Indicates whether the provided string (which should be a filter assertion
2044   * value or substring component) may represent an injection attempt.
2045   *
2046   * @param  value  The value for which to make the determination.  It may
2047   *                optionally be {@code null}.
2048   *
2049   * @return  {@code true} if the provided value may represent an injection
2050   *          attempt, or {@code false} if not.
2051   */
2052  private static boolean mayRepresentInjectionAttempt(
2053               @Nullable final String value)
2054  {
2055    if (value == null)
2056    {
2057      return false;
2058    }
2059
2060    final String lowerValue = StaticUtils.toLowerCase(value);
2061    return (lowerValue.contains("(") ||
2062         lowerValue.contains(")") ||
2063         lowerValue.contains("&") ||
2064         lowerValue.contains("|") ||
2065         lowerValue.contains("\"") ||
2066         lowerValue.contains("'") ||
2067         ((lowerValue.contains("select") && lowerValue.contains("from"))));
2068  }
2069
2070
2071
2072  /**
2073   * Counts the number of components in the specified filter.  Presence,
2074   * equality, substring, greater-or-equal, less-or-equal, approximate-match,
2075   * and extensible-match filters will all be considered a single component.
2076   * AND and OR filters will be one plus the aggregate component count for each
2077   * of the components they contain.  NOT filters will be one plus the component
2078   * count for the filter it contains.
2079   *
2080   * @param  filter  The filter for which to count the number of components.  It
2081   *                 must not be {@code null}.
2082   *
2083   * @return  The number of components in the specified filter.
2084   */
2085  static int countComponents(@NotNull final Filter filter)
2086  {
2087    switch (filter.getFilterType())
2088    {
2089      case Filter.FILTER_TYPE_AND:
2090      case Filter.FILTER_TYPE_OR:
2091        int count = 1;
2092        for (final Filter f : filter.getComponents())
2093        {
2094          count += countComponents(f);
2095        }
2096        return count;
2097
2098      case Filter.FILTER_TYPE_NOT:
2099        return 1 + countComponents(filter.getNOTComponent());
2100
2101      case Filter.FILTER_TYPE_PRESENCE:
2102      case Filter.FILTER_TYPE_EQUALITY:
2103      case Filter.FILTER_TYPE_SUBSTRING:
2104      case Filter.FILTER_TYPE_GREATER_OR_EQUAL:
2105      case Filter.FILTER_TYPE_LESS_OR_EQUAL:
2106      case Filter.FILTER_TYPE_APPROXIMATE_MATCH:
2107      case Filter.FILTER_TYPE_EXTENSIBLE_MATCH:
2108      default:
2109        return 1;
2110    }
2111  }
2112
2113
2114
2115  /**
2116   * Performs any necessary processing for an unbind request message.
2117   *
2118   * @param  m  The log message to be processed.
2119   */
2120  private void processUnbindRequest(
2121                    @NotNull final UnbindRequestAccessLogMessage m)
2122  {
2123    numUnbinds++;
2124  }
2125
2126
2127
2128  /**
2129   * Performs any necessary processing for an add result message.
2130   *
2131   * @param  m  The log message to be processed.
2132   */
2133  private void processAddResult(@NotNull final AddResultAccessLogMessage m)
2134  {
2135    numAdds++;
2136
2137    updateCommonResult(m);
2138
2139    updateResultCodeCount(m.getResultCode(), addResultCodes);
2140    addProcessingDuration +=
2141         doubleValue(m.getProcessingTimeMillis(), addProcessingTimes);
2142
2143    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2144    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2145    {
2146      numUncachedAdds++;
2147    }
2148
2149    updateAuthzCount(m.getAlternateAuthorizationDN());
2150  }
2151
2152
2153
2154  /**
2155   * Performs any necessary processing for a bind result message.
2156   *
2157   * @param  m  The log message to be processed.
2158   */
2159  private void processBindResult(@NotNull final BindResultAccessLogMessage m)
2160  {
2161    numBinds++;
2162
2163    updateCommonResult(m);
2164
2165    if (m.getAuthenticationType() != null)
2166    {
2167      final String authType;
2168      switch (m.getAuthenticationType())
2169      {
2170        case SIMPLE:
2171          authType = "Simple";
2172          break;
2173
2174        case SASL:
2175          final String saslMechanism = m.getSASLMechanismName();
2176          if (saslMechanism == null)
2177          {
2178            authType = "SASL {unknown mechanism}";
2179          }
2180          else
2181          {
2182            authType = "SASL " + saslMechanism;
2183          }
2184          break;
2185
2186        case INTERNAL:
2187          authType = "Internal";
2188          break;
2189
2190        default:
2191          authType = m.getAuthenticationType().name();
2192          break;
2193      }
2194
2195      AtomicLong l = authenticationTypes.get(authType);
2196      if (l == null)
2197      {
2198        l = new AtomicLong(0L);
2199        authenticationTypes.put(authType, l);
2200      }
2201      l.incrementAndGet();
2202    }
2203
2204    updateResultCodeCount(m.getResultCode(), bindResultCodes);
2205    bindProcessingDuration +=
2206         doubleValue(m.getProcessingTimeMillis(), bindProcessingTimes);
2207
2208    String authenticationDN = getDNString(m.getAuthenticationDN());
2209    if (m.getResultCode() == ResultCode.SUCCESS)
2210    {
2211      if (authenticationDN != null)
2212      {
2213        AtomicLong l = successfulBindDNs.get(authenticationDN);
2214        if (l == null)
2215        {
2216          l = new AtomicLong(0L);
2217          successfulBindDNs.put(authenticationDN, l);
2218        }
2219        l.incrementAndGet();
2220
2221        final AtomicLong outstandingFailures =
2222             outstandingFailedBindDNs.remove(authenticationDN);
2223        if (outstandingFailures != null)
2224        {
2225          final AtomicLong consecutiveFailures =
2226               consecutiveFailedBindsByDN.get(authenticationDN);
2227          if ((consecutiveFailures == null) ||
2228             (outstandingFailures.get() > consecutiveFailures.get()))
2229          {
2230            consecutiveFailedBindsByDN.put(authenticationDN,
2231                 new AtomicLong(outstandingFailures.get()));
2232          }
2233        }
2234      }
2235
2236      final String ccp = m.getClientConnectionPolicy();
2237      if (ccp != null)
2238      {
2239        AtomicLong l = clientConnectionPolicies.get(ccp);
2240        if (l == null)
2241        {
2242          l = new AtomicLong(0L);
2243          clientConnectionPolicies.put(ccp, l);
2244        }
2245        l.incrementAndGet();
2246      }
2247    }
2248    else if ((m.getResultCode() != ResultCode.SASL_BIND_IN_PROGRESS) &&
2249         (m.getResultCode() != ResultCode.REFERRAL))
2250    {
2251      if (authenticationDN == null)
2252      {
2253        authenticationDN = getDNString(m.getDN());
2254      }
2255
2256      if (authenticationDN != null)
2257      {
2258        AtomicLong l = bindFailuresByDN.get(authenticationDN);
2259        if (l == null)
2260        {
2261          l = new AtomicLong(0L);
2262          bindFailuresByDN.put(authenticationDN, l);
2263        }
2264        l.incrementAndGet();
2265
2266        l = outstandingFailedBindDNs.get(authenticationDN);
2267        if (l == null)
2268        {
2269          l = new AtomicLong(0L);
2270          outstandingFailedBindDNs.put(authenticationDN, l);
2271        }
2272        l.incrementAndGet();
2273      }
2274
2275      String ipAddress = m.getRequesterIPAddress();
2276      if (ipAddress == null)
2277      {
2278        final Long connectionID = m.getConnectionID();
2279        if (connectionID != null)
2280        {
2281          ipAddress = ipAddressesByConnectionID.get(connectionID);
2282        }
2283      }
2284
2285      if (ipAddress != null)
2286      {
2287        AtomicLong l = bindFailuresByIPAddress.get(ipAddress);
2288        if (l == null)
2289        {
2290          l = new AtomicLong(0L);
2291          bindFailuresByIPAddress.put(ipAddress, l);
2292        }
2293        l.incrementAndGet();
2294      }
2295    }
2296
2297    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2298    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2299    {
2300      numUncachedBinds++;
2301    }
2302
2303    updateAuthzCount(m.getAuthorizationDN());
2304  }
2305
2306
2307
2308  /**
2309   * Performs any necessary processing for a compare result message.
2310   *
2311   * @param  m  The log message to be processed.
2312   */
2313  private void processCompareResult(
2314                    @NotNull final CompareResultAccessLogMessage m)
2315  {
2316    numCompares++;
2317
2318    updateCommonResult(m);
2319
2320    updateResultCodeCount(m.getResultCode(), compareResultCodes);
2321    compareProcessingDuration +=
2322         doubleValue(m.getProcessingTimeMillis(), compareProcessingTimes);
2323
2324    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2325    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2326    {
2327      numUncachedCompares++;
2328    }
2329
2330    updateAuthzCount(m.getAlternateAuthorizationDN());
2331  }
2332
2333
2334
2335  /**
2336   * Performs any necessary processing for a delete result message.
2337   *
2338   * @param  m  The log message to be processed.
2339   */
2340  private void processDeleteResult(
2341                    @NotNull final DeleteResultAccessLogMessage m)
2342  {
2343    numDeletes++;
2344
2345    updateCommonResult(m);
2346
2347    updateResultCodeCount(m.getResultCode(), deleteResultCodes);
2348    deleteProcessingDuration +=
2349         doubleValue(m.getProcessingTimeMillis(), deleteProcessingTimes);
2350
2351    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2352    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2353    {
2354      numUncachedDeletes++;
2355    }
2356
2357    updateAuthzCount(m.getAlternateAuthorizationDN());
2358  }
2359
2360
2361
2362  /**
2363   * Performs any necessary processing for an extended result message.
2364   *
2365   * @param  m  The log message to be processed.
2366   */
2367  private void processExtendedResult(
2368                    @NotNull final ExtendedResultAccessLogMessage m)
2369  {
2370    numExtended++;
2371
2372    updateCommonResult(m);
2373
2374    final String id = m.getConnectionID() + "-" + m.getOperationID();
2375    if (!processedRequests.remove(id))
2376    {
2377      processExtendedRequestInternal(m);
2378    }
2379
2380    updateResultCodeCount(m.getResultCode(), extendedResultCodes);
2381    extendedProcessingDuration +=
2382         doubleValue(m.getProcessingTimeMillis(), extendedProcessingTimes);
2383
2384    final String ccp = m.getClientConnectionPolicy();
2385    if (ccp != null)
2386    {
2387      AtomicLong l = clientConnectionPolicies.get(ccp);
2388      if (l == null)
2389      {
2390        l = new AtomicLong(0L);
2391        clientConnectionPolicies.put(ccp, l);
2392      }
2393      l.incrementAndGet();
2394    }
2395
2396    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2397    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2398    {
2399      numUncachedExtended++;
2400    }
2401  }
2402
2403
2404
2405  /**
2406   * Performs any necessary processing for a modify result message.
2407   *
2408   * @param  m  The log message to be processed.
2409   */
2410  private void processModifyResult(
2411                    @NotNull final ModifyResultAccessLogMessage m)
2412  {
2413    numModifies++;
2414
2415    updateCommonResult(m);
2416
2417    updateResultCodeCount(m.getResultCode(), modifyResultCodes);
2418    modifyProcessingDuration +=
2419         doubleValue(m.getProcessingTimeMillis(), modifyProcessingTimes);
2420
2421    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2422    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2423    {
2424      numUncachedModifies++;
2425    }
2426
2427    updateAuthzCount(m.getAlternateAuthorizationDN());
2428  }
2429
2430
2431
2432  /**
2433   * Performs any necessary processing for a modify DN result message.
2434   *
2435   * @param  m  The log message to be processed.
2436   */
2437  private void processModifyDNResult(
2438                    @NotNull final ModifyDNResultAccessLogMessage m)
2439  {
2440    numModifyDNs++;
2441
2442    updateCommonResult(m);
2443
2444    updateResultCodeCount(m.getResultCode(), modifyDNResultCodes);
2445    modifyDNProcessingDuration +=
2446         doubleValue(m.getProcessingTimeMillis(), modifyDNProcessingTimes);
2447
2448    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2449    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2450    {
2451      numUncachedModifyDNs++;
2452    }
2453
2454    updateAuthzCount(m.getAlternateAuthorizationDN());
2455  }
2456
2457
2458
2459  /**
2460   * Performs any necessary processing for a search result message.
2461   *
2462   * @param  m  The log message to be processed.
2463   */
2464  private void processSearchResult(
2465                    @NotNull final SearchResultAccessLogMessage m)
2466  {
2467    numSearches++;
2468
2469    updateCommonResult(m);
2470
2471    final String id = m.getConnectionID() + "-" + m.getOperationID();
2472    if (! processedRequests.remove(id))
2473    {
2474      processSearchRequestInternal(m);
2475    }
2476
2477    final ResultCode resultCode = m.getResultCode();
2478    updateResultCodeCount(resultCode, searchResultCodes);
2479    searchProcessingDuration +=
2480         doubleValue(m.getProcessingTimeMillis(), searchProcessingTimes);
2481
2482    final String filterString = prepareFilter(m.getFilter());
2483
2484    final Long entryCount = m.getEntriesReturned();
2485    if (entryCount != null)
2486    {
2487      AtomicLong l = searchEntryCounts.get(entryCount);
2488      if (l == null)
2489      {
2490        l = new AtomicLong(0L);
2491        searchEntryCounts.put(entryCount, l);
2492      }
2493      l.incrementAndGet();
2494
2495      final Map<String,AtomicLong> filterCountMap;
2496      switch (entryCount.intValue())
2497      {
2498        case 0:
2499          filterCountMap = noEntryFilters;
2500          break;
2501        case 1:
2502          filterCountMap = oneEntryFilters;
2503          break;
2504        default:
2505          filterCountMap = multiEntryFilters;
2506          break;
2507      }
2508
2509      if (filterString != null)
2510      {
2511        AtomicLong filterCount = filterCountMap.get(filterString);
2512        if (filterCount == null)
2513        {
2514          filterCount = new AtomicLong(0L);
2515          filterCountMap.put(filterString, filterCount);
2516        }
2517        filterCount.incrementAndGet();
2518      }
2519    }
2520
2521    final Boolean isUnindexed = m.getUnindexed();
2522    if ((isUnindexed != null) && isUnindexed)
2523    {
2524      numUnindexedAttempts++;
2525      if (resultCode == ResultCode.SUCCESS)
2526      {
2527        numUnindexedSuccessful++;
2528      }
2529      else
2530      {
2531        numUnindexedFailed++;
2532      }
2533
2534      if (filterString != null)
2535      {
2536        AtomicLong l = unindexedFilters.get(filterString);
2537        if (l == null)
2538        {
2539          l = new AtomicLong(0L);
2540          unindexedFilters.put(filterString, l);
2541        }
2542        l.incrementAndGet();
2543      }
2544    }
2545
2546    final Boolean uncachedDataAccessed = m.getUncachedDataAccessed();
2547    if ((uncachedDataAccessed != null) && uncachedDataAccessed)
2548    {
2549      numUncachedSearches++;
2550    }
2551
2552    updateAuthzCount(m.getAlternateAuthorizationDN());
2553
2554    final Double processingTimeMillis = m.getProcessingTimeMillis();
2555    if ((processingTimeMillis != null) && (filterString != null))
2556    {
2557      final long processingTimeMicros =
2558           Math.round(processingTimeMillis * 1_000.0);
2559
2560      AtomicLong l = mostExpensiveFilters.get(filterString);
2561      if (l == null)
2562      {
2563        l = new AtomicLong(processingTimeMicros);
2564        mostExpensiveFilters.put(filterString, l);
2565      }
2566      else
2567      {
2568        final long previousProcessingTimeMicros = l.get();
2569        if (processingTimeMicros > previousProcessingTimeMicros)
2570        {
2571          l.set(processingTimeMicros);
2572        }
2573      }
2574    }
2575  }
2576
2577
2578
2579  /**
2580   * Updates a number of statistics that are common to all types of result log
2581   * messages.
2582   *
2583   * @param  m  The result log message to examine.
2584   */
2585  private void updateCommonResult(
2586                    @NotNull final OperationResultAccessLogMessage m)
2587  {
2588    // Handle the work queue wait time.
2589    totalWorkQueueWaitTime +=
2590         doubleValue(m.getWorkQueueWaitTimeMillis(), workQueueWaitTimes);
2591
2592
2593    // Handle request and response control OIDs.
2594    for (final String oid : m.getRequestControlOIDs())
2595    {
2596      numRequestControls++;
2597      updateCount(requestControlOIDs, oid);
2598    }
2599
2600    for (final String oid : m.getResponseControlOIDs())
2601    {
2602      numResponseControls++;
2603      updateCount(responseControlOIDs, oid);
2604    }
2605
2606
2607    // Handle used and missing privileges.
2608    for (final String privilegeName : m.getPreAuthorizationUsedPrivileges())
2609    {
2610      updateCount(preAuthzPrivilegesUsed, privilegeName);
2611    }
2612
2613    for (final String privilegeName : m.getUsedPrivileges())
2614    {
2615      updateCount(privilegesUsed, privilegeName);
2616    }
2617
2618    for (final String privilegeName : m.getMissingPrivileges())
2619    {
2620      updateCount(privilegesMissing, privilegeName);
2621    }
2622  }
2623
2624
2625
2626  /**
2627   * Updates the counter for the given key in the provided map.  If the key does
2628   * not exist, it will be added to the map.
2629   *
2630   * @param  m    The map to be updated.
2631   * @param  key  The key for which to update the count.
2632   */
2633  private static void updateCount(@NotNull final Map<String,AtomicLong> m,
2634                                  @NotNull final String key)
2635  {
2636    AtomicLong count = m.get(key);
2637    if (count == null)
2638    {
2639      count = new AtomicLong(0L);
2640      m.put(key, count);
2641    }
2642
2643    count.incrementAndGet();
2644  }
2645
2646
2647
2648  /**
2649   * Updates the count for the provided result code in the given map.
2650   *
2651   * @param  rc  The result code for which to update the count.
2652   * @param  m   The map used to hold counts by result code.
2653   */
2654  private static void updateResultCodeCount(@Nullable final ResultCode rc,
2655                           @NotNull final HashMap<ResultCode,AtomicLong> m)
2656  {
2657    if (rc == null)
2658    {
2659      return;
2660    }
2661
2662    AtomicLong l = m.get(rc);
2663    if (l == null)
2664    {
2665      l = new AtomicLong(0L);
2666      m.put(rc, l);
2667    }
2668    l.incrementAndGet();
2669  }
2670
2671
2672
2673  /**
2674   * Retrieves the double value for the provided {@code Double} object.
2675   *
2676   * @param  d  The {@code Double} object for which to retrieve the value.
2677   * @param  m  The processing time histogram map to be updated.
2678   *
2679   * @return  The double value of the provided {@code Double} object if it was
2680   *          non-{@code null}, or 0.0 if it was {@code null}.
2681   */
2682  private static double doubleValue(@Nullable final Double d,
2683                                    @NotNull final HashMap<Long,AtomicLong> m)
2684  {
2685    if (d == null)
2686    {
2687      return 0.0;
2688    }
2689    else
2690    {
2691      for (final Map.Entry<Long,AtomicLong> e : m.entrySet())
2692      {
2693        if (d <= e.getKey())
2694        {
2695          e.getValue().incrementAndGet();
2696          break;
2697        }
2698      }
2699
2700      return d;
2701    }
2702  }
2703
2704
2705
2706  /**
2707   * Updates the provided list with the most frequently-occurring elements in
2708   * the provided map, paired with the number of times each value occurred.
2709   *
2710   * @param  <K>                    The type of object used as the key for the
2711   *                                provided map.
2712   * @param  countMap               The map to be examined.  It is expected that
2713   *                                the values of the map will be the count of
2714   *                                occurrences for the keys.
2715   * @param  mostCommonElementList  The list to which the values will be
2716   *                                updated.  It must not be {@code null}, must
2717   *                                be empty, and must be updatable.
2718   * @param  maxListSize            The maximum number of items to add to the
2719   *                                provided list.  It must be greater than
2720   *                                zero.
2721   * @param  skippedWithSameCount   A counter that will be incremented for each
2722   *                                map entry that is skipped with the same
2723   *                                count as a value that was not skipped.  It
2724   *                                must not be {@code null} and must initially
2725   *                                be zero.
2726   * @param  skippedWithLowerCount  A counter that will be incremented for each
2727   *                                map entry that is skipped with a lower count
2728   *                                as the last value that was not skipped.  It
2729   *                                must not be {@code null} and must initially
2730   *                                be zero.
2731   *
2732   * @return  A list of the most frequently-occurring elements in the provided
2733   *          map.
2734   */
2735  @NotNull()
2736  private static <K> List<ObjectPair<K,Long>> getMostCommonElements(
2737               @NotNull final Map<K,AtomicLong> countMap,
2738               @NotNull final List<ObjectPair<K,Long>> mostCommonElementList,
2739               final int maxListSize,
2740               @NotNull final AtomicLong skippedWithSameCount,
2741               @NotNull final AtomicLong skippedWithLowerCount)
2742  {
2743    final TreeMap<Long,List<K>> reverseMap =
2744         new TreeMap<>(new ReverseComparator<Long>());
2745    for (final Map.Entry<K,AtomicLong> e : countMap.entrySet())
2746    {
2747      final Long count = e.getValue().get();
2748      List<K> list = reverseMap.get(count);
2749      if (list == null)
2750      {
2751        list = new ArrayList<>();
2752        reverseMap.put(count, list);
2753      }
2754      list.add(e.getKey());
2755    }
2756
2757    for (final Map.Entry<Long,List<K>> e : reverseMap.entrySet())
2758    {
2759      final Long l = e.getKey();
2760      int numNotSkipped = 0;
2761      for (final K k : e.getValue())
2762      {
2763        if (mostCommonElementList.size() >= maxListSize)
2764        {
2765          if (numNotSkipped > 0)
2766          {
2767            skippedWithSameCount.incrementAndGet();
2768          }
2769          else
2770          {
2771            skippedWithLowerCount.incrementAndGet();
2772          }
2773        }
2774        else
2775        {
2776          numNotSkipped++;
2777          mostCommonElementList.add(new ObjectPair<>(k, l));
2778        }
2779      }
2780    }
2781
2782    return mostCommonElementList;
2783  }
2784
2785
2786
2787  /**
2788   * Updates the count of alternate authorization identities for the provided
2789   * DN.
2790   *
2791   * @param  authzDN  The DN of the alternate authorization identity that was
2792   *                  used.  It may be {@code null} if no alternate
2793   *                  authorization identity was used.
2794   */
2795  private void updateAuthzCount(@Nullable final String authzDN)
2796  {
2797    if (authzDN == null)
2798    {
2799      return;
2800    }
2801
2802    final String dnString = getDNString(authzDN);
2803
2804    AtomicLong l = authzDNs.get(dnString);
2805    if (l == null)
2806    {
2807      l = new AtomicLong(0L);
2808      authzDNs.put(dnString, l);
2809    }
2810  }
2811
2812
2813
2814  /**
2815   * Retrieves a string representation of the provided DN.  It may either be
2816   * anonymized, using question marks in place of specific attribute values, or
2817   * it may be the actual string representation of the given DN.
2818   *
2819   * @param  dn  The DN for which to retrieve the string representation.
2820   *
2821   * @return  A string representation of the provided DN, or {@code null} if the
2822   *          given DN was {@code null}.
2823   */
2824  @Nullable()
2825  private String getDNString(@Nullable final String dn)
2826  {
2827    if (dn == null)
2828    {
2829      return null;
2830    }
2831
2832    final DN parsedDN;
2833    try
2834    {
2835      parsedDN = new DN(dn);
2836    }
2837    catch (final Exception e)
2838    {
2839      Debug.debugException(e);
2840      return dn.toLowerCase();
2841    }
2842
2843    if (parsedDN.isNullDN())
2844    {
2845      return "{Null DN}";
2846    }
2847
2848    if (doNotAnonymize.isPresent())
2849    {
2850      return parsedDN.toNormalizedString();
2851    }
2852
2853    final StringBuilder buffer = new StringBuilder();
2854    final RDN[] rdns = parsedDN.getRDNs();
2855    for (int i=0; i < rdns.length; i++)
2856    {
2857      if (i > 0)
2858      {
2859        buffer.append(',');
2860      }
2861
2862      final RDN rdn = rdns[i];
2863      final String[] attributeNames = rdn.getAttributeNames();
2864      for (int j=0; j < attributeNames.length; j++)
2865      {
2866        if (j > 0)
2867        {
2868          buffer.append('+');
2869        }
2870        buffer.append(attributeNames[j].toLowerCase());
2871        buffer.append("=?");
2872      }
2873    }
2874
2875    return buffer.toString();
2876  }
2877
2878
2879
2880  /**
2881   * Retrieves a prepared string representation of the provided search filter.
2882   * It may potentially be de-anonymized to include specific values.
2883   *
2884   * @param  filterString  The string representation of the filter to prepare.
2885   *                       It may be {@code null} if the log message does not
2886   *                       have a filter.
2887   *
2888   * @return  A string representation of the provided filter (which may or may
2889   *          not be anonymized), or {@code null} if the provided filter is
2890   *          {@code null} or cannot be prepared.
2891   */
2892  @Nullable()
2893  private String prepareFilter(@Nullable final String filterString)
2894  {
2895    if (filterString == null)
2896    {
2897      return null;
2898    }
2899
2900    if (doNotAnonymize.isPresent())
2901    {
2902      return filterString.toLowerCase();
2903    }
2904
2905    try
2906    {
2907      return new GenericFilter(Filter.create(filterString)).toString().
2908           toLowerCase();
2909    }
2910    catch (final Exception e)
2911    {
2912      Debug.debugException(e);
2913      return null;
2914    }
2915  }
2916
2917
2918
2919  /**
2920   * Writes a breakdown of the processing times for a specified type of
2921   * operation.
2922   *
2923   * @param  t  The name of the operation type.
2924   * @param  n  The total number of operations of the specified type that were
2925   *            processed by the server.
2926   * @param  m  The map of operation counts by processing time bucket.
2927   */
2928  private void printProcessingTimeHistogram(@NotNull final String t,
2929                    final long n,
2930                    @NotNull final LinkedHashMap<Long,AtomicLong> m)
2931  {
2932    printHistogram("Count of " + t + " operations by processing time:", n, m);
2933  }
2934
2935
2936
2937  /**
2938   * Writes a breakdown of the processing times for a specified type of
2939   * operation.
2940   *
2941   * @param  h  The header to display at the beginning of the histogram.
2942   * @param  n  The total number of operations that were processed by the
2943   *            server.
2944   * @param  m  The map of operation counts by processing time bucket.
2945   */
2946  private void printHistogram(@NotNull final String h,
2947                    final long n,
2948                    @NotNull final LinkedHashMap<Long,AtomicLong> m)
2949  {
2950    if (n <= 0)
2951    {
2952      return;
2953    }
2954
2955    out();
2956    out(h);
2957
2958    long lowerBound = 0;
2959    long accumulatedCount = 0;
2960    final Iterator<Map.Entry<Long,AtomicLong>> i = m.entrySet().iterator();
2961    while (i.hasNext())
2962    {
2963      final Map.Entry<Long,AtomicLong> e = i.next();
2964      final long upperBound = e.getKey();
2965      final long count = e.getValue().get();
2966      final double categoryPercent = 100.0 * count / n;
2967
2968      accumulatedCount += count;
2969      final double accumulatedPercent = 100.0 * accumulatedCount / n;
2970
2971      if (i.hasNext())
2972      {
2973        final String lowerBoundString;
2974        if (lowerBound == 0L)
2975        {
2976          lowerBoundString = "0 milliseconds";
2977        }
2978        else
2979        {
2980          final long lowerBoundNanos = lowerBound * 1_000_000L;
2981          lowerBoundString = DurationArgument.nanosToDuration(lowerBoundNanos);
2982        }
2983
2984        final long upperBoundNanos = upperBound * 1_000_000L;
2985        final String upperBoundString =
2986             DurationArgument.nanosToDuration(upperBoundNanos);
2987
2988
2989        out("Between ", lowerBoundString, " and ", upperBoundString, ":  ",
2990            count, " (", decimalFormat.format(categoryPercent), "%, ",
2991            decimalFormat.format(accumulatedPercent), "% accumulated)");
2992        lowerBound = upperBound;
2993      }
2994      else
2995      {
2996        final long lowerBoundNanos = lowerBound * 1_000_000L;
2997        final String lowerBoundString =
2998             DurationArgument.nanosToDuration(lowerBoundNanos);
2999
3000        out("Greater than ", lowerBoundString, ":  ", count, " (",
3001            decimalFormat.format(categoryPercent), "%, ",
3002            decimalFormat.format(accumulatedPercent), "% accumulated)");
3003      }
3004    }
3005  }
3006
3007
3008
3009  /**
3010   * Optionally prints information about the number and percent of operations of
3011   * the specified type that involved access to uncached data.
3012   *
3013   * @param  operationType  The type of operation.
3014   * @param  numUncached    The number of operations of the specified type that
3015   *                        involved access to uncached data.
3016   * @param  numTotal       The total number of operations of the specified
3017   *                        type.
3018   */
3019  private void printUncached(@NotNull final String operationType,
3020                             final long numUncached,
3021                             final long numTotal)
3022  {
3023    if (numUncached == 0)
3024    {
3025      return;
3026    }
3027
3028    out(operationType, ":  ", numUncached, " (",
3029         decimalFormat.format(100.0 * numUncached / numTotal), "%)");
3030  }
3031
3032
3033
3034  /**
3035   * Prints data from the provided map of counts.
3036   *
3037   * @param  countMap      The map containing the data to print.
3038   * @param  heading       The heading to display before printing the contents
3039   *                       of the map.
3040   * @param  singularItem  The name to use for a single item represented by the
3041   *                       key of the given map.
3042   * @param  pluralItem    The name to use for zero or multiple items
3043   *                       represented by the key of the given map.
3044   */
3045  private void printCounts(@Nullable final Map<String,AtomicLong> countMap,
3046                           @NotNull final String heading,
3047                           @NotNull final String singularItem,
3048                           @NotNull final String pluralItem)
3049  {
3050    if ((countMap == null) || countMap.isEmpty())
3051    {
3052      return;
3053    }
3054
3055    long totalCount = 0L;
3056    for (final AtomicLong l : countMap.values())
3057    {
3058      totalCount += l.get();
3059    }
3060
3061    out();
3062    out(heading);
3063
3064    int displayCount = reportCount.getValue();
3065    if (displayCount <= 0L)
3066    {
3067      displayCount = Integer.MAX_VALUE;
3068    }
3069
3070    final List<ObjectPair<String,Long>> countList = new ArrayList<>();
3071    final AtomicLong skippedWithSameCount = new AtomicLong(0L);
3072    final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
3073    getMostCommonElements(countMap, countList, displayCount,
3074         skippedWithSameCount, skippedWithLowerCount);
3075
3076    long count = -1L;
3077    for (final ObjectPair<String,Long> p : countList)
3078    {
3079      count = p.getSecond();
3080
3081      if (totalCount > 0L)
3082      {
3083        final double percent = 100.0 * count / totalCount;
3084        out(p.getFirst(), ":  ", count, " (", decimalFormat.format(percent),
3085             ")");
3086      }
3087      else
3088      {
3089        out(p.getFirst(), ":  ", count);
3090      }
3091    }
3092
3093    if (skippedWithSameCount.get() > 0L)
3094    {
3095      out("{ Skipped " + skippedWithSameCount.get() + " additional " +
3096           getSingularOrPlural(skippedWithSameCount.get(), singularItem,
3097                pluralItem) +
3098           " with a count of " + count + " }");
3099    }
3100
3101    if (skippedWithLowerCount.get() > 0L)
3102    {
3103      out("{ Skipped " + skippedWithLowerCount.get() + " additional " +
3104           getSingularOrPlural(skippedWithLowerCount.get(), singularItem,
3105                pluralItem) +
3106           " with a count that is less than " + count + " }");
3107    }
3108  }
3109
3110
3111
3112  /**
3113   * Prints data from the provided map of counts.
3114   *
3115   * @param  countMap       The map containing the data to print.
3116   * @param  operationType  The type of operation represented by the keys of
3117   *                        the map.
3118   */
3119  private void printResultCodeCounts(
3120                    @Nullable final Map<ResultCode,AtomicLong> countMap,
3121                    @NotNull final String operationType)
3122  {
3123    if ((countMap == null) || countMap.isEmpty())
3124    {
3125      return;
3126    }
3127
3128    long totalCount = 0L;
3129    for (final AtomicLong l : countMap.values())
3130    {
3131      totalCount += l.get();
3132    }
3133
3134    out();
3135    out("Most common " + operationType + " operation result codes:");
3136
3137    int displayCount = reportCount.getValue();
3138    if (displayCount <= 0L)
3139    {
3140      displayCount = Integer.MAX_VALUE;
3141    }
3142
3143    final List<ObjectPair<ResultCode,Long>> resultCodeList = new ArrayList<>();
3144    final AtomicLong skippedWithSameCount = new AtomicLong(0L);
3145    final AtomicLong skippedWithLowerCount = new AtomicLong(0L);
3146    getMostCommonElements(countMap, resultCodeList, displayCount,
3147         skippedWithSameCount, skippedWithLowerCount);
3148
3149    long count = -1L;
3150    for (final ObjectPair<ResultCode,Long> p : resultCodeList)
3151    {
3152      count = p.getSecond();
3153
3154      if (totalCount > 0L)
3155      {
3156        final double percent = 100.0 * count / totalCount;
3157        out(p.getFirst().getName(), " (", p.getFirst().intValue(), "):  ",
3158             count, " (", decimalFormat.format(percent), ")");
3159      }
3160      else
3161      {
3162        out(p.getFirst(), ":  ", count);
3163      }
3164    }
3165
3166    if (skippedWithSameCount.get() > 0L)
3167    {
3168      out("{ Skipped " + skippedWithSameCount.get() + " additional result " +
3169           getSingularOrPlural(skippedWithSameCount.get(), "code", "codes") +
3170           " with a count of " + count + " }");
3171    }
3172
3173    if (skippedWithLowerCount.get() > 0L)
3174    {
3175      out("{ Skipped " + skippedWithLowerCount.get() + " additional result " +
3176           getSingularOrPlural(skippedWithLowerCount.get(), "code", "codes") +
3177           " with a count that is less than " + count + " }");
3178    }
3179  }
3180
3181
3182
3183  /**
3184   * Retrieves the appropriate singular or plural form based on the given
3185   * value.
3186   *
3187   * @param  count     The count that will be used to determine whether to
3188   *                   retrieve the singular or plural form.
3189   * @param  singular  The singular form for the value to return.
3190   * @param  plural    The plural form for the value to return.
3191   *
3192   * @return  The singular form if the count is 1, or the plural form if the
3193   *          count is any other value.
3194   */
3195  @NotNull()
3196  private String getSingularOrPlural(final long count,
3197                                     @NotNull final String singular,
3198                                     @NotNull final String plural)
3199  {
3200    if (count == 1L)
3201    {
3202      return singular;
3203    }
3204    else
3205    {
3206      return plural;
3207    }
3208  }
3209}