001/* 002 * Copyright 2009-2024 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2009-2024 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2009-2024 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.unboundidds.examples; 037 038 039 040import java.io.File; 041import java.io.FileInputStream; 042import java.io.InputStream; 043import java.io.IOException; 044import java.io.OutputStream; 045import java.io.Serializable; 046import java.text.DecimalFormat; 047import java.util.ArrayList; 048import java.util.HashMap; 049import java.util.HashSet; 050import java.util.Iterator; 051import java.util.LinkedHashMap; 052import java.util.LinkedHashSet; 053import java.util.List; 054import java.util.Map; 055import java.util.TreeMap; 056import java.util.concurrent.atomic.AtomicLong; 057import java.util.zip.GZIPInputStream; 058import javax.crypto.BadPaddingException; 059 060import com.unboundid.ldap.sdk.DN; 061import com.unboundid.ldap.sdk.Filter; 062import com.unboundid.ldap.sdk.LDAPException; 063import com.unboundid.ldap.sdk.RDN; 064import com.unboundid.ldap.sdk.ResultCode; 065import com.unboundid.ldap.sdk.SearchScope; 066import com.unboundid.ldap.sdk.Version; 067import com.unboundid.ldap.sdk.unboundidds.logs.LogException; 068import com.unboundid.ldap.sdk.unboundidds.logs.v2. 069 AbandonRequestAccessLogMessage; 070import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogMessage; 071import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogReader; 072import com.unboundid.ldap.sdk.unboundidds.logs.v2.AddResultAccessLogMessage; 073import com.unboundid.ldap.sdk.unboundidds.logs.v2.BindResultAccessLogMessage; 074import com.unboundid.ldap.sdk.unboundidds.logs.v2.CompareResultAccessLogMessage; 075import com.unboundid.ldap.sdk.unboundidds.logs.v2.ConnectAccessLogMessage; 076import com.unboundid.ldap.sdk.unboundidds.logs.v2.DeleteResultAccessLogMessage; 077import com.unboundid.ldap.sdk.unboundidds.logs.v2.DisconnectAccessLogMessage; 078import com.unboundid.ldap.sdk.unboundidds.logs.v2. 079 ExtendedRequestAccessLogMessage; 080import com.unboundid.ldap.sdk.unboundidds.logs.v2. 081 ExtendedResultAccessLogMessage; 082import com.unboundid.ldap.sdk.unboundidds.logs.v2. 083 ModifyDNResultAccessLogMessage; 084import com.unboundid.ldap.sdk.unboundidds.logs.v2.ModifyResultAccessLogMessage; 085import com.unboundid.ldap.sdk.unboundidds.logs.v2. 086 OperationRequestAccessLogMessage; 087import com.unboundid.ldap.sdk.unboundidds.logs.v2. 088 OperationResultAccessLogMessage; 089import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchRequestAccessLogMessage; 090import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchResultAccessLogMessage; 091import com.unboundid.ldap.sdk.unboundidds.logs.v2. 092 SecurityNegotiationAccessLogMessage; 093import com.unboundid.ldap.sdk.unboundidds.logs.v2.UnbindRequestAccessLogMessage; 094import com.unboundid.ldap.sdk.unboundidds.logs.v2.json.JSONAccessLogReader; 095import com.unboundid.ldap.sdk.unboundidds.logs.v2.text. 096 TextFormattedAccessLogReader; 097import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 098import com.unboundid.util.CommandLineTool; 099import com.unboundid.util.Debug; 100import com.unboundid.util.NotMutable; 101import com.unboundid.util.NotNull; 102import com.unboundid.util.Nullable; 103import com.unboundid.util.OIDRegistry; 104import com.unboundid.util.OIDRegistryItem; 105import com.unboundid.util.ObjectPair; 106import com.unboundid.util.ReverseComparator; 107import com.unboundid.util.StaticUtils; 108import com.unboundid.util.ThreadSafety; 109import com.unboundid.util.ThreadSafetyLevel; 110import com.unboundid.util.args.ArgumentException; 111import com.unboundid.util.args.ArgumentParser; 112import com.unboundid.util.args.BooleanArgument; 113import com.unboundid.util.args.DurationArgument; 114import com.unboundid.util.args.FileArgument; 115import com.unboundid.util.args.IntegerArgument; 116 117 118 119/** 120 * This class provides a tool that may be used to read and summarize the 121 * contents of one or more access log files from Ping Identity, UnboundID and 122 * Nokia/Alcatel-Lucent 8661 server products. 123 * <BR> 124 * <BLOCKQUOTE> 125 * <B>NOTE:</B> This class, and other classes within the 126 * {@code com.unboundid.ldap.sdk.unboundidds} package structure, are only 127 * supported for use against Ping Identity, UnboundID, and 128 * Nokia/Alcatel-Lucent 8661 server products. These classes provide support 129 * for proprietary functionality or for external specifications that are not 130 * considered stable or mature enough to be guaranteed to work in an 131 * interoperable way with other types of LDAP servers. 132 * </BLOCKQUOTE> 133 * Information that will be reported includes: 134 * <UL> 135 * <LI>The total length of time covered by the log files.</LI> 136 * <LI>The number of connections established and disconnected, the addresses 137 * of the most commonly-connecting clients, and the average rate of 138 * connects and disconnects per second.</LI> 139 * <LI>The number of operations processed, overall and by operation type, 140 * and the average rate of operations per second.</LI> 141 * <LI>The average duration for operations processed, overall and by operation 142 * type.</LI> 143 * <LI>A breakdown of operation processing times into a number of predefined 144 * categories, ranging from less than one millisecond to over one 145 * minute.</LI> 146 * <LI>A breakdown of the most common result codes for each type of operation 147 * and their relative frequencies.</LI> 148 * <LI>The most common types of extended requests processed and their 149 * relative frequencies.</LI> 150 * <LI>The number of unindexed search operations processed and the most common 151 * types of filters used in unindexed searches.</LI> 152 * <LI>A breakdown of the relative frequencies for each type of search 153 * scope.</LI> 154 * <LI>The most common types of search filters used for search 155 * operations and their relative frequencies.</LI> 156 * </UL> 157 * It is designed to work with access log files using either the default log 158 * format with separate request and response messages, as well as log files 159 * in which the request and response details have been combined on the same 160 * line. The log files to be processed should be provided as command-line 161 * arguments. 162 * <BR><BR> 163 * The APIs demonstrated by this example include: 164 * <UL> 165 * <LI>Access log parsing (from the 166 * {@code com.unboundid.ldap.sdk.unboundidds.logs} package)</LI> 167 * <LI>Argument parsing (from the {@code com.unboundid.util.args} 168 * package)</LI> 169 * </UL> 170 */ 171@NotMutable() 172@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 173public final class SummarizeAccessLog 174 extends CommandLineTool 175 implements Serializable 176{ 177 /** 178 * The column at which long lines should be wrapped. 179 */ 180 private static final int WRAP_COLUMN = StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 181 182 183 184 /** 185 * The serial version UID for this serializable class. 186 */ 187 private static final long serialVersionUID = 7189168366509887130L; 188 189 190 191 // Variables used for accessing argument information. 192 @Nullable private ArgumentParser argumentParser; 193 194 // An argument that may be used to indicate that the summarized output should 195 // not be anonymized, and should include attribute values. 196 @Nullable private BooleanArgument doNotAnonymize; 197 198 // An argument that may be used to indicate that the log files are compressed. 199 @Nullable private BooleanArgument isCompressed; 200 201 // An argument that may be used to indicate that the log content is 202 // JSON-formatted rather than text-formatted. 203 @Nullable private BooleanArgument json; 204 205 // An argument used to specify the encryption passphrase. 206 @Nullable private FileArgument encryptionPassphraseFile; 207 208 // An argument used to specify the maximum number of values to report for each 209 // item. 210 @Nullable private IntegerArgument reportCount; 211 212 // The decimal format that will be used for this class. 213 @NotNull private final DecimalFormat decimalFormat; 214 215 // The total duration for log content, in milliseconds. 216 private long logDurationMillis; 217 218 // The total processing time for each type of operation. 219 private double addProcessingDuration; 220 private double bindProcessingDuration; 221 private double compareProcessingDuration; 222 private double deleteProcessingDuration; 223 private double extendedProcessingDuration; 224 private double modifyProcessingDuration; 225 private double modifyDNProcessingDuration; 226 private double searchProcessingDuration; 227 228 // A variable used for tracking total work queue wait time. 229 private long totalWorkQueueWaitTime; 230 231 // A variable used for counting the number of messages of each type. 232 private long numAbandons; 233 private long numAdds; 234 private long numBinds; 235 private long numCompares; 236 private long numConnects; 237 private long numDeletes; 238 private long numDisconnects; 239 private long numExtended; 240 private long numModifies; 241 private long numModifyDNs; 242 private long numSearches; 243 private long numUnbinds; 244 245 // The number of operations of each type that accessed uncached data. 246 private long numUncachedAdds; 247 private long numUncachedBinds; 248 private long numUncachedCompares; 249 private long numUncachedDeletes; 250 private long numUncachedExtended; 251 private long numUncachedModifies; 252 private long numUncachedModifyDNs; 253 private long numUncachedSearches; 254 255 // The number of unindexed searches processed within the server. 256 private long numUnindexedAttempts; 257 private long numUnindexedFailed; 258 private long numUnindexedSuccessful; 259 260 // The number of request and response controls used. 261 private long numRequestControls; 262 private long numResponseControls; 263 264 // Variables used for maintaining counts for common types of information. 265 @NotNull private final HashMap<Long,AtomicLong> searchEntryCounts; 266 @NotNull private final HashMap<Long,String> ipAddressesByConnectionID; 267 @NotNull private final HashMap<ResultCode,AtomicLong> addResultCodes; 268 @NotNull private final HashMap<ResultCode,AtomicLong> bindResultCodes; 269 @NotNull private final HashMap<ResultCode,AtomicLong> compareResultCodes; 270 @NotNull private final HashMap<ResultCode,AtomicLong> deleteResultCodes; 271 @NotNull private final HashMap<ResultCode,AtomicLong> extendedResultCodes; 272 @NotNull private final HashMap<ResultCode,AtomicLong> modifyResultCodes; 273 @NotNull private final HashMap<ResultCode,AtomicLong> modifyDNResultCodes; 274 @NotNull private final HashMap<ResultCode,AtomicLong> searchResultCodes; 275 @NotNull private final HashMap<SearchScope,AtomicLong> searchScopes; 276 @NotNull private final HashMap<String,AtomicLong> authenticationTypes; 277 @NotNull private final HashMap<String,AtomicLong> authzDNs; 278 @NotNull private final HashMap<String,AtomicLong> bindFailuresByDN; 279 @NotNull private final HashMap<String,AtomicLong> bindFailuresByIPAddress; 280 @NotNull private final HashMap<String,AtomicLong> consecutiveFailedBindsByDN; 281 @NotNull private final HashMap<String,AtomicLong> outstandingFailedBindDNs; 282 @NotNull private final HashMap<String,AtomicLong> successfulBindDNs; 283 @NotNull private final HashMap<String,AtomicLong> clientAddresses; 284 @NotNull private final HashMap<String,AtomicLong> clientConnectionPolicies; 285 @NotNull private final HashMap<String,AtomicLong> disconnectReasons; 286 @NotNull private final HashMap<String,AtomicLong> extendedOperations; 287 @NotNull private final HashMap<String,AtomicLong> filterComponentCounts; 288 @NotNull private final HashMap<String,AtomicLong> filterTypes; 289 @NotNull private final HashMap<String,AtomicLong> mostExpensiveFilters; 290 @NotNull private final HashMap<String,AtomicLong> multiEntryFilters; 291 @NotNull private final HashMap<String,AtomicLong> noEntryFilters; 292 @NotNull private final HashMap<String,AtomicLong> oneEntryFilters; 293 @NotNull private final HashMap<String,AtomicLong> preAuthzPrivilegesUsed; 294 @NotNull private final HashMap<String,AtomicLong> privilegesMissing; 295 @NotNull private final HashMap<String,AtomicLong> privilegesUsed; 296 @NotNull private final HashMap<String,AtomicLong> requestControlOIDs; 297 @NotNull private final HashMap<String,AtomicLong> responseControlOIDs; 298 @NotNull private final HashMap<String,AtomicLong> searchBaseDNs; 299 @NotNull private final HashMap<String,AtomicLong> tlsCipherSuites; 300 @NotNull private final HashMap<String,AtomicLong> tlsProtocols; 301 @NotNull private final HashMap<String,AtomicLong> unindexedFilters; 302 @NotNull private final HashMap<String,String> extendedOperationOIDsToNames; 303 @NotNull private final HashSet<String> processedRequests; 304 @NotNull private final LinkedHashMap<Long,AtomicLong> addProcessingTimes; 305 @NotNull private final LinkedHashMap<Long,AtomicLong> bindProcessingTimes; 306 @NotNull private final LinkedHashMap<Long,AtomicLong> compareProcessingTimes; 307 @NotNull private final LinkedHashMap<Long,AtomicLong> deleteProcessingTimes; 308 @NotNull private final LinkedHashMap<Long,AtomicLong> extendedProcessingTimes; 309 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyProcessingTimes; 310 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyDNProcessingTimes; 311 @NotNull private final LinkedHashMap<Long,AtomicLong> searchProcessingTimes; 312 @NotNull private final LinkedHashMap<Long,AtomicLong> workQueueWaitTimes; 313 @NotNull private final LinkedHashSet<Filter> 314 filtersRepresentingPotentialInjectionAttempt; 315 316 317 318 /** 319 * Parse the provided command line arguments and perform the appropriate 320 * processing. 321 * 322 * @param args The command line arguments provided to this program. 323 */ 324 public static void main(@NotNull final String[] args) 325 { 326 final ResultCode resultCode = main(args, System.out, System.err); 327 if (resultCode != ResultCode.SUCCESS) 328 { 329 System.exit(resultCode.intValue()); 330 } 331 } 332 333 334 335 /** 336 * Parse the provided command line arguments and perform the appropriate 337 * processing. 338 * 339 * @param args The command line arguments provided to this program. 340 * @param outStream The output stream to which standard out should be 341 * written. It may be {@code null} if output should be 342 * suppressed. 343 * @param errStream The output stream to which standard error should be 344 * written. It may be {@code null} if error messages 345 * should be suppressed. 346 * 347 * @return A result code indicating whether the processing was successful. 348 */ 349 @NotNull() 350 public static ResultCode main(@NotNull final String[] args, 351 @Nullable final OutputStream outStream, 352 @Nullable final OutputStream errStream) 353 { 354 final SummarizeAccessLog summarizer = 355 new SummarizeAccessLog(outStream, errStream); 356 return summarizer.runTool(args); 357 } 358 359 360 361 /** 362 * Creates a new instance of this tool. 363 * 364 * @param outStream The output stream to which standard out should be 365 * written. It may be {@code null} if output should be 366 * suppressed. 367 * @param errStream The output stream to which standard error should be 368 * written. It may be {@code null} if error messages 369 * should be suppressed. 370 */ 371 public SummarizeAccessLog(@Nullable final OutputStream outStream, 372 @Nullable final OutputStream errStream) 373 { 374 super(outStream, errStream); 375 376 argumentParser = null; 377 doNotAnonymize = null; 378 isCompressed = null; 379 json = null; 380 encryptionPassphraseFile = null; 381 reportCount = null; 382 383 decimalFormat = new DecimalFormat("0.000"); 384 385 logDurationMillis = 0L; 386 387 addProcessingDuration = 0.0; 388 bindProcessingDuration = 0.0; 389 compareProcessingDuration = 0.0; 390 deleteProcessingDuration = 0.0; 391 extendedProcessingDuration = 0.0; 392 modifyProcessingDuration = 0.0; 393 modifyDNProcessingDuration = 0.0; 394 searchProcessingDuration = 0.0; 395 396 totalWorkQueueWaitTime = 0L; 397 398 numAbandons = 0L; 399 numAdds = 0L; 400 numBinds = 0L; 401 numCompares = 0L; 402 numConnects = 0L; 403 numDeletes = 0L; 404 numDisconnects = 0L; 405 numExtended = 0L; 406 numModifies = 0L; 407 numModifyDNs = 0L; 408 numSearches = 0L; 409 numUnbinds = 0L; 410 411 numUncachedAdds = 0L; 412 numUncachedBinds = 0L; 413 numUncachedCompares = 0L; 414 numUncachedDeletes = 0L; 415 numUncachedExtended = 0L; 416 numUncachedModifies = 0L; 417 numUncachedModifyDNs = 0L; 418 numUncachedSearches = 0L; 419 420 numUnindexedAttempts = 0L; 421 numUnindexedFailed = 0L; 422 numUnindexedSuccessful = 0L; 423 424 numRequestControls = 0L; 425 numResponseControls = 0L; 426 427 searchEntryCounts = new HashMap<>(StaticUtils.computeMapCapacity(10)); 428 ipAddressesByConnectionID = 429 new HashMap<>(StaticUtils.computeMapCapacity(100)); 430 addResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 431 bindResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 432 compareResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 433 deleteResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 434 extendedResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 435 modifyResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 436 modifyDNResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 437 searchResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 438 searchScopes = new HashMap<>(StaticUtils.computeMapCapacity(4)); 439 authenticationTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 440 authzDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 441 bindFailuresByDN = new HashMap<>(StaticUtils.computeMapCapacity(100)); 442 bindFailuresByIPAddress = 443 new HashMap<>(StaticUtils.computeMapCapacity(100)); 444 outstandingFailedBindDNs = 445 new HashMap<>(StaticUtils.computeMapCapacity(100)); 446 successfulBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 447 clientAddresses = new HashMap<>(StaticUtils.computeMapCapacity(100)); 448 clientConnectionPolicies = 449 new HashMap<>(StaticUtils.computeMapCapacity(100)); 450 disconnectReasons = new HashMap<>(StaticUtils.computeMapCapacity(100)); 451 extendedOperations = new HashMap<>(StaticUtils.computeMapCapacity(10)); 452 filterComponentCounts = new HashMap<>(StaticUtils.computeMapCapacity(10)); 453 filterTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 454 mostExpensiveFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 455 multiEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 456 noEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 457 oneEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 458 preAuthzPrivilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100)); 459 privilegesMissing = new HashMap<>(StaticUtils.computeMapCapacity(100)); 460 privilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100)); 461 requestControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 462 responseControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 463 searchBaseDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 464 tlsCipherSuites = new HashMap<>(StaticUtils.computeMapCapacity(100)); 465 tlsProtocols = new HashMap<>(StaticUtils.computeMapCapacity(100)); 466 unindexedFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 467 consecutiveFailedBindsByDN = 468 new HashMap<>(StaticUtils.computeMapCapacity(100)); 469 extendedOperationOIDsToNames = 470 new HashMap<>(StaticUtils.computeMapCapacity(100)); 471 processedRequests = new HashSet<>(StaticUtils.computeMapCapacity(100)); 472 addProcessingTimes = 473 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 474 bindProcessingTimes = 475 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 476 compareProcessingTimes = 477 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 478 deleteProcessingTimes = 479 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 480 extendedProcessingTimes = 481 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 482 modifyProcessingTimes = 483 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 484 modifyDNProcessingTimes = 485 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 486 searchProcessingTimes = 487 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 488 workQueueWaitTimes = 489 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 490 filtersRepresentingPotentialInjectionAttempt = 491 new LinkedHashSet<>(StaticUtils.computeMapCapacity(10)); 492 493 populateProcessingTimeMap(addProcessingTimes); 494 populateProcessingTimeMap(bindProcessingTimes); 495 populateProcessingTimeMap(compareProcessingTimes); 496 populateProcessingTimeMap(deleteProcessingTimes); 497 populateProcessingTimeMap(extendedProcessingTimes); 498 populateProcessingTimeMap(modifyProcessingTimes); 499 populateProcessingTimeMap(modifyDNProcessingTimes); 500 populateProcessingTimeMap(searchProcessingTimes); 501 populateProcessingTimeMap(workQueueWaitTimes); 502 } 503 504 505 506 /** 507 * Retrieves the name for this tool. 508 * 509 * @return The name for this tool. 510 */ 511 @Override() 512 @NotNull() 513 public String getToolName() 514 { 515 return "summarize-access-log"; 516 } 517 518 519 520 /** 521 * Retrieves the description for this tool. 522 * 523 * @return The description for this tool. 524 */ 525 @Override() 526 @NotNull() 527 public String getToolDescription() 528 { 529 return "Examine one or more access log files from Ping Identity, " + 530 "UnboundID, or Nokia/Alcatel-Lucent 8661 server products to display " + 531 "a number of metrics about operations processed within the server."; 532 } 533 534 535 536 /** 537 * Retrieves the version string for this tool. 538 * 539 * @return The version string for this tool. 540 */ 541 @Override() 542 @NotNull() 543 public String getToolVersion() 544 { 545 return Version.NUMERIC_VERSION_STRING; 546 } 547 548 549 550 /** 551 * Retrieves the minimum number of unnamed trailing arguments that are 552 * required. 553 * 554 * @return One, to indicate that at least one trailing argument (representing 555 * the path to an access log file) must be provided. 556 */ 557 @Override() 558 public int getMinTrailingArguments() 559 { 560 return 1; 561 } 562 563 564 565 /** 566 * Retrieves the maximum number of unnamed trailing arguments that may be 567 * provided for this tool. 568 * 569 * @return The maximum number of unnamed trailing arguments that may be 570 * provided for this tool. 571 */ 572 @Override() 573 public int getMaxTrailingArguments() 574 { 575 return -1; 576 } 577 578 579 580 /** 581 * Retrieves a placeholder string that should be used for trailing arguments 582 * in the usage information for this tool. 583 * 584 * @return A placeholder string that should be used for trailing arguments in 585 * the usage information for this tool. 586 */ 587 @Override() 588 @NotNull() 589 public String getTrailingArgumentsPlaceholder() 590 { 591 return "{path}"; 592 } 593 594 595 596 /** 597 * Indicates whether this tool should provide support for an interactive mode, 598 * in which the tool offers a mode in which the arguments can be provided in 599 * a text-driven menu rather than requiring them to be given on the command 600 * line. If interactive mode is supported, it may be invoked using the 601 * "--interactive" argument. Alternately, if interactive mode is supported 602 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 603 * interactive mode may be invoked by simply launching the tool without any 604 * arguments. 605 * 606 * @return {@code true} if this tool supports interactive mode, or 607 * {@code false} if not. 608 */ 609 @Override() 610 public boolean supportsInteractiveMode() 611 { 612 return true; 613 } 614 615 616 617 /** 618 * Indicates whether this tool defaults to launching in interactive mode if 619 * the tool is invoked without any command-line arguments. This will only be 620 * used if {@link #supportsInteractiveMode()} returns {@code true}. 621 * 622 * @return {@code true} if this tool defaults to using interactive mode if 623 * launched without any command-line arguments, or {@code false} if 624 * not. 625 */ 626 @Override() 627 public boolean defaultsToInteractiveMode() 628 { 629 return true; 630 } 631 632 633 634 /** 635 * Indicates whether this tool should provide arguments for redirecting output 636 * to a file. If this method returns {@code true}, then the tool will offer 637 * an "--outputFile" argument that will specify the path to a file to which 638 * all standard output and standard error content will be written, and it will 639 * also offer a "--teeToStandardOut" argument that can only be used if the 640 * "--outputFile" argument is present and will cause all output to be written 641 * to both the specified output file and to standard output. 642 * 643 * @return {@code true} if this tool should provide arguments for redirecting 644 * output to a file, or {@code false} if not. 645 */ 646 @Override() 647 protected boolean supportsOutputFile() 648 { 649 return true; 650 } 651 652 653 654 /** 655 * Indicates whether this tool supports the use of a properties file for 656 * specifying default values for arguments that aren't specified on the 657 * command line. 658 * 659 * @return {@code true} if this tool supports the use of a properties file 660 * for specifying default values for arguments that aren't specified 661 * on the command line, or {@code false} if not. 662 */ 663 @Override() 664 public boolean supportsPropertiesFile() 665 { 666 return true; 667 } 668 669 670 671 /** 672 * Indicates whether this tool supports the ability to generate a debug log 673 * file. If this method returns {@code true}, then the tool will expose 674 * additional arguments that can control debug logging. 675 * 676 * @return {@code true} if this tool supports the ability to generate a debug 677 * log file, or {@code false} if not. 678 */ 679 @Override() 680 protected boolean supportsDebugLogging() 681 { 682 return true; 683 } 684 685 686 687 /** 688 * Adds the command-line arguments supported for use with this tool to the 689 * provided argument parser. The tool may need to retain references to the 690 * arguments (and/or the argument parser, if trailing arguments are allowed) 691 * to it in order to obtain their values for use in later processing. 692 * 693 * @param parser The argument parser to which the arguments are to be added. 694 * 695 * @throws ArgumentException If a problem occurs while adding any of the 696 * tool-specific arguments to the provided 697 * argument parser. 698 */ 699 @Override() 700 public void addToolArguments(@NotNull final ArgumentParser parser) 701 throws ArgumentException 702 { 703 // We need to save a reference to the argument parser so that we can get 704 // the trailing arguments later. 705 argumentParser = parser; 706 707 // Add an argument that makes it possible to read a JSON-formatted access 708 // log file. 709 String description = "Indicates that the log file contains " + 710 "JSON-formatted log messages rather than text-formatted messages."; 711 json = new BooleanArgument(null, "json", description); 712 parser.addArgument(json); 713 714 715 // Add an argument that makes it possible to read a compressed log file. 716 // Note that this argument is no longer needed for dealing with compressed 717 // files, since the tool will automatically detect whether a file is 718 // compressed. However, the argument is still provided for the purpose of 719 // backward compatibility. 720 description = "Indicates that the log file is compressed."; 721 isCompressed = new BooleanArgument('c', "isCompressed", description); 722 isCompressed.addLongIdentifier("is-compressed", true); 723 isCompressed.addLongIdentifier("compressed", true); 724 isCompressed.setHidden(true); 725 parser.addArgument(isCompressed); 726 727 728 // Add an argument that indicates that the tool should read the encryption 729 // passphrase from a file. 730 description = "Indicates that the log file is encrypted and that the " + 731 "encryption passphrase is contained in the specified file. If " + 732 "the log data is encrypted and this argument is not provided, then " + 733 "the tool will interactively prompt for the encryption passphrase."; 734 encryptionPassphraseFile = new FileArgument(null, 735 "encryptionPassphraseFile", false, 1, null, description, true, true, 736 true, false); 737 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 738 true); 739 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 740 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 741 true); 742 parser.addArgument(encryptionPassphraseFile); 743 744 745 // Add an argument that indicates the number of values to display for each 746 // item being summarized. 747 description = "The number of values to display for each item being " + 748 "summarized. A value of zero indicates that all items should be " + 749 "displayed. If this is not provided, a default value of 20 will " + 750 "be used."; 751 reportCount = new IntegerArgument(null, "reportCount", false, 0, null, 752 description, 0, Integer.MAX_VALUE, 20); 753 reportCount.addLongIdentifier("report-count", true); 754 reportCount.addLongIdentifier("maximumCount", true); 755 reportCount.addLongIdentifier("maximum-count", true); 756 reportCount.addLongIdentifier("maxCount", true); 757 reportCount.addLongIdentifier("max-count", true); 758 reportCount.addLongIdentifier("count", true); 759 parser.addArgument(reportCount); 760 761 762 // Add an argument that indicates that the output should not be anonymized. 763 description = "Do not anonymize the output, but include actual attribute " + 764 "values in filters and DNs. This will also have the effect of " + 765 "de-generifying those values, so output including the most common " + 766 "filters and DNs in some category will be specific instances of " + 767 "those filters and DNs instead of generic patterns."; 768 doNotAnonymize = new BooleanArgument(null, "doNotAnonymize", 1, 769 description); 770 doNotAnonymize.addLongIdentifier("do-not-anonymize", true); 771 doNotAnonymize.addLongIdentifier("deAnonymize", true); 772 doNotAnonymize.addLongIdentifier("de-anonymize", true); 773 parser.addArgument(doNotAnonymize); 774 } 775 776 777 778 /** 779 * Performs any necessary processing that should be done to ensure that the 780 * provided set of command-line arguments were valid. This method will be 781 * called after the basic argument parsing has been performed and immediately 782 * before the {@link #doToolProcessing} method is invoked. 783 * 784 * @throws ArgumentException If there was a problem with the command-line 785 * arguments provided to this program. 786 */ 787 @Override() 788 public void doExtendedArgumentValidation() 789 throws ArgumentException 790 { 791 // Make sure that at least one access log file path was provided. 792 final List<String> trailingArguments = 793 argumentParser.getTrailingArguments(); 794 if ((trailingArguments == null) || trailingArguments.isEmpty()) 795 { 796 throw new ArgumentException("No access log file paths were provided."); 797 } 798 } 799 800 801 802 /** 803 * Performs the core set of processing for this tool. 804 * 805 * @return A result code that indicates whether the processing completed 806 * successfully. 807 */ 808 @Override() 809 @NotNull() 810 public ResultCode doToolProcessing() 811 { 812 int displayCount = reportCount.getValue(); 813 if (displayCount <= 0) 814 { 815 displayCount = Integer.MAX_VALUE; 816 } 817 818 String encryptionPassphrase = null; 819 if (encryptionPassphraseFile.isPresent()) 820 { 821 try 822 { 823 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 824 encryptionPassphraseFile.getValue()); 825 } 826 catch (final LDAPException e) 827 { 828 Debug.debugException(e); 829 err(e.getMessage()); 830 return e.getResultCode(); 831 } 832 } 833 834 835 long logLines = 0L; 836 for (final String path : argumentParser.getTrailingArguments()) 837 { 838 final File f = new File(path); 839 out("Examining access log ", f.getAbsolutePath()); 840 AccessLogReader reader = null; 841 InputStream inputStream = null; 842 try 843 { 844 inputStream = new FileInputStream(f); 845 846 final ObjectPair<InputStream,String> p = 847 ToolUtils.getPossiblyPassphraseEncryptedInputStream(inputStream, 848 encryptionPassphrase, 849 (! encryptionPassphraseFile.isPresent()), 850 "Log file '" + path + "' is encrypted. Please enter the " + 851 "encryption passphrase:", 852 "ERROR: The provided passphrase was incorrect.", 853 getOut(), getErr()); 854 inputStream = p.getFirst(); 855 if ((p.getSecond() != null) && (encryptionPassphrase == null)) 856 { 857 encryptionPassphrase = p.getSecond(); 858 } 859 860 if (isCompressed.isPresent()) 861 { 862 inputStream = new GZIPInputStream(inputStream); 863 } 864 else 865 { 866 inputStream = 867 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 868 } 869 870 if (json.isPresent()) 871 { 872 reader = new JSONAccessLogReader(inputStream); 873 } 874 else 875 { 876 reader = new TextFormattedAccessLogReader(inputStream); 877 } 878 } 879 catch (final Exception e) 880 { 881 Debug.debugException(e); 882 err("Unable to open access log file ", f.getAbsolutePath(), ": ", 883 StaticUtils.getExceptionMessage(e)); 884 return ResultCode.LOCAL_ERROR; 885 } 886 finally 887 { 888 if ((reader == null) && (inputStream != null)) 889 { 890 try 891 { 892 inputStream.close(); 893 } 894 catch (final Exception e) 895 { 896 Debug.debugException(e); 897 } 898 } 899 } 900 901 long startTime = 0L; 902 long stopTime = 0L; 903 904 while (true) 905 { 906 final AccessLogMessage msg; 907 try 908 { 909 msg = reader.readMessage(); 910 } 911 catch (final IOException ioe) 912 { 913 Debug.debugException(ioe); 914 err("Error reading from access log file ", f.getAbsolutePath(), 915 ": ", StaticUtils.getExceptionMessage(ioe)); 916 917 if ((ioe.getCause() != null) && 918 (ioe.getCause() instanceof BadPaddingException)) 919 { 920 err("This error is likely because the log is encrypted and the " + 921 "server still has the log file open. It is recommended " + 922 "that you only try to examine encrypted logs after they " + 923 "have been rotated. You can use the rotate-log tool to " + 924 "force a rotation at any time. Attempting to proceed with " + 925 "just the data that was successfully read."); 926 break; 927 } 928 else 929 { 930 return ResultCode.LOCAL_ERROR; 931 } 932 } 933 catch (final LogException le) 934 { 935 Debug.debugException(le); 936 err("Encountered an error while attempting to parse a line in" + 937 "access log file ", f.getAbsolutePath(), ": ", 938 StaticUtils.getExceptionMessage(le)); 939 continue; 940 } 941 942 if (msg == null) 943 { 944 break; 945 } 946 947 logLines++; 948 stopTime = msg.getTimestamp().getTime(); 949 if (startTime == 0L) 950 { 951 startTime = stopTime; 952 } 953 954 switch (msg.getMessageType()) 955 { 956 case CONNECT: 957 processConnect((ConnectAccessLogMessage) msg); 958 break; 959 case SECURITY_NEGOTIATION: 960 processSecurityNegotiation( 961 (SecurityNegotiationAccessLogMessage) msg); 962 break; 963 case DISCONNECT: 964 processDisconnect((DisconnectAccessLogMessage) msg); 965 break; 966 case REQUEST: 967 switch (((OperationRequestAccessLogMessage) msg).getOperationType()) 968 { 969 case ABANDON: 970 processAbandonRequest((AbandonRequestAccessLogMessage) msg); 971 break; 972 case EXTENDED: 973 processExtendedRequest((ExtendedRequestAccessLogMessage) msg); 974 break; 975 case SEARCH: 976 processSearchRequest((SearchRequestAccessLogMessage) msg); 977 break; 978 case UNBIND: 979 processUnbindRequest((UnbindRequestAccessLogMessage) msg); 980 break; 981 } 982 break; 983 case RESULT: 984 switch (((OperationRequestAccessLogMessage) msg).getOperationType()) 985 { 986 case ADD: 987 processAddResult((AddResultAccessLogMessage) msg); 988 break; 989 case BIND: 990 processBindResult((BindResultAccessLogMessage) msg); 991 break; 992 case COMPARE: 993 processCompareResult((CompareResultAccessLogMessage) msg); 994 break; 995 case DELETE: 996 processDeleteResult((DeleteResultAccessLogMessage) msg); 997 break; 998 case EXTENDED: 999 processExtendedResult((ExtendedResultAccessLogMessage) msg); 1000 break; 1001 case MODIFY: 1002 processModifyResult((ModifyResultAccessLogMessage) msg); 1003 break; 1004 case MODDN: 1005 processModifyDNResult((ModifyDNResultAccessLogMessage) msg); 1006 break; 1007 case SEARCH: 1008 processSearchResult((SearchResultAccessLogMessage) msg); 1009 break; 1010 } 1011 break; 1012 1013 case ASSURANCE_COMPLETE: 1014 case CLIENT_CERTIFICATE: 1015 case ENTRY_REBALANCING_REQUEST: 1016 case ENTRY_REBALANCING_RESULT: 1017 case FORWARD: 1018 case FORWARD_FAILED: 1019 case ENTRY: 1020 case REFERENCE: 1021 default: 1022 // Nothing needs to be done for these message types. 1023 } 1024 } 1025 1026 try 1027 { 1028 reader.close(); 1029 } 1030 catch (final Exception e) 1031 { 1032 Debug.debugException(e); 1033 } 1034 logDurationMillis += (stopTime - startTime); 1035 1036 1037 // If there are any outstanding authentication failures, then update the 1038 // set of consecutive failures as appropriate. 1039 for (final Map.Entry<String,AtomicLong> e : 1040 outstandingFailedBindDNs.entrySet()) 1041 { 1042 final String dn = e.getKey(); 1043 final AtomicLong outstandingFailureCount = e.getValue(); 1044 final AtomicLong consecutiveFailures = 1045 consecutiveFailedBindsByDN.get(dn); 1046 if ((consecutiveFailures == null) || 1047 (outstandingFailureCount.get() > consecutiveFailures.get())) 1048 { 1049 consecutiveFailedBindsByDN.put(dn, outstandingFailureCount); 1050 } 1051 } 1052 outstandingFailedBindDNs.clear(); 1053 } 1054 1055 1056 final int numFiles = argumentParser.getTrailingArguments().size(); 1057 out(); 1058 out("Examined ", logLines, " lines in ", numFiles, 1059 ((numFiles == 1) ? " file" : " files"), 1060 " covering a total duration of ", 1061 StaticUtils.millisToHumanReadableDuration(logDurationMillis)); 1062 if (logLines == 0) 1063 { 1064 return ResultCode.SUCCESS; 1065 } 1066 1067 out(); 1068 1069 final double logDurationSeconds = logDurationMillis / 1_000.0; 1070 final double connectsPerSecond = numConnects / logDurationSeconds; 1071 final double disconnectsPerSecond = numDisconnects / logDurationSeconds; 1072 1073 out("Total connections established: ", numConnects, " (", 1074 decimalFormat.format(connectsPerSecond), "/second)"); 1075 out("Total disconnects: ", numDisconnects, " (", 1076 decimalFormat.format(disconnectsPerSecond), "/second)"); 1077 1078 printCounts(clientAddresses, "Most common client addresses:", "address", 1079 "addresses"); 1080 1081 printCounts(clientConnectionPolicies, 1082 "Most common client connection policies:", "policy", "policies"); 1083 1084 printCounts(tlsProtocols, "Most common TLS protocol versions:", "version", 1085 "versions"); 1086 1087 printCounts(tlsCipherSuites, "Most common TLS cipher suites:", 1088 "cipher suite", "cipher suites"); 1089 1090 printCounts(disconnectReasons, "Most common disconnect reasons:", "reason", 1091 "reasons"); 1092 1093 final long totalOps = numAbandons + numAdds + numBinds + numCompares + 1094 numDeletes + numExtended + numModifies + numModifyDNs + numSearches + 1095 numUnbinds; 1096 final long totalResults = totalOps - numAbandons - numUnbinds; 1097 1098 if (totalOps > 0) 1099 { 1100 final double percentAbandon = 100.0 * numAbandons / totalOps; 1101 final double percentAdd = 100.0 * numAdds / totalOps; 1102 final double percentBind = 100.0 * numBinds / totalOps; 1103 final double percentCompare = 100.0 * numCompares / totalOps; 1104 final double percentDelete = 100.0 * numDeletes / totalOps; 1105 final double percentExtended = 100.0 * numExtended / totalOps; 1106 final double percentModify = 100.0 * numModifies / totalOps; 1107 final double percentModifyDN = 100.0 * numModifyDNs / totalOps; 1108 final double percentSearch = 100.0 * numSearches / totalOps; 1109 final double percentUnbind = 100.0 * numUnbinds / totalOps; 1110 1111 final double abandonsPerSecond = numAbandons / logDurationSeconds; 1112 final double addsPerSecond = numAdds / logDurationSeconds; 1113 final double bindsPerSecond = numBinds / logDurationSeconds; 1114 final double comparesPerSecond = numCompares / logDurationSeconds; 1115 final double deletesPerSecond = numDeletes / logDurationSeconds; 1116 final double extendedPerSecond = numExtended / logDurationSeconds; 1117 final double modifiesPerSecond = numModifies / logDurationSeconds; 1118 final double modifyDNsPerSecond = numModifyDNs / logDurationSeconds; 1119 final double searchesPerSecond = numSearches / logDurationSeconds; 1120 final double unbindsPerSecond = numUnbinds / logDurationSeconds; 1121 1122 out(); 1123 out("Total operations examined: ", totalOps); 1124 out("Abandon operations examined: ", numAbandons, " (", 1125 decimalFormat.format(percentAbandon), "%, ", 1126 decimalFormat.format(abandonsPerSecond), "/second)"); 1127 out("Add operations examined: ", numAdds, " (", 1128 decimalFormat.format(percentAdd), "%, ", 1129 decimalFormat.format(addsPerSecond), "/second)"); 1130 out("Bind operations examined: ", numBinds, " (", 1131 decimalFormat.format(percentBind), "%, ", 1132 decimalFormat.format(bindsPerSecond), "/second)"); 1133 out("Compare operations examined: ", numCompares, " (", 1134 decimalFormat.format(percentCompare), "%, ", 1135 decimalFormat.format(comparesPerSecond), "/second)"); 1136 out("Delete operations examined: ", numDeletes, " (", 1137 decimalFormat.format(percentDelete), "%, ", 1138 decimalFormat.format(deletesPerSecond), "/second)"); 1139 out("Extended operations examined: ", numExtended, " (", 1140 decimalFormat.format(percentExtended), "%, ", 1141 decimalFormat.format(extendedPerSecond), "/second)"); 1142 out("Modify operations examined: ", numModifies, " (", 1143 decimalFormat.format(percentModify), "%, ", 1144 decimalFormat.format(modifiesPerSecond), "/second)"); 1145 out("Modify DN operations examined: ", numModifyDNs, " (", 1146 decimalFormat.format(percentModifyDN), "%, ", 1147 decimalFormat.format(modifyDNsPerSecond), "/second)"); 1148 out("Search operations examined: ", numSearches, " (", 1149 decimalFormat.format(percentSearch), "%, ", 1150 decimalFormat.format(searchesPerSecond), "/second)"); 1151 out("Unbind operations examined: ", numUnbinds, " (", 1152 decimalFormat.format(percentUnbind), "%, ", 1153 decimalFormat.format(unbindsPerSecond), "/second)"); 1154 1155 final double totalProcessingDuration = addProcessingDuration + 1156 bindProcessingDuration + compareProcessingDuration + 1157 deleteProcessingDuration + extendedProcessingDuration + 1158 modifyProcessingDuration + modifyDNProcessingDuration + 1159 searchProcessingDuration; 1160 1161 out(); 1162 out("Average operation processing duration: ", 1163 decimalFormat.format(totalProcessingDuration / totalOps), "ms"); 1164 1165 if (numAdds > 0) 1166 { 1167 out("Average add operation processing duration: ", 1168 decimalFormat.format(addProcessingDuration / numAdds), "ms"); 1169 } 1170 1171 if (numBinds > 0) 1172 { 1173 out("Average bind operation processing duration: ", 1174 decimalFormat.format(bindProcessingDuration / numBinds), "ms"); 1175 } 1176 1177 if (numCompares > 0) 1178 { 1179 out("Average compare operation processing duration: ", 1180 decimalFormat.format(compareProcessingDuration / numCompares), 1181 "ms"); 1182 } 1183 1184 if (numDeletes > 0) 1185 { 1186 out("Average delete operation processing duration: ", 1187 decimalFormat.format(deleteProcessingDuration / numDeletes), "ms"); 1188 } 1189 1190 if (numExtended > 0) 1191 { 1192 out("Average extended operation processing duration: ", 1193 decimalFormat.format(extendedProcessingDuration / numExtended), 1194 "ms"); 1195 } 1196 1197 if (numModifies > 0) 1198 { 1199 out("Average modify operation processing duration: ", 1200 decimalFormat.format(modifyProcessingDuration / numModifies), "ms"); 1201 } 1202 1203 if (numModifyDNs > 0) 1204 { 1205 out("Average modify DN operation processing duration: ", 1206 decimalFormat.format(modifyDNProcessingDuration / numModifyDNs), 1207 "ms"); 1208 } 1209 1210 if (numSearches > 0) 1211 { 1212 out("Average search operation processing duration: ", 1213 decimalFormat.format(searchProcessingDuration / numSearches), "ms"); 1214 } 1215 1216 printProcessingTimeHistogram("add", numAdds, addProcessingTimes); 1217 printProcessingTimeHistogram("bind", numBinds, bindProcessingTimes); 1218 printProcessingTimeHistogram("compare", numCompares, 1219 compareProcessingTimes); 1220 printProcessingTimeHistogram("delete", numDeletes, deleteProcessingTimes); 1221 printProcessingTimeHistogram("extended", numExtended, 1222 extendedProcessingTimes); 1223 printProcessingTimeHistogram("modify", numModifies, 1224 modifyProcessingTimes); 1225 printProcessingTimeHistogram("modify DN", numModifyDNs, 1226 modifyDNProcessingTimes); 1227 printProcessingTimeHistogram("search", numSearches, 1228 searchProcessingTimes); 1229 1230 if (totalWorkQueueWaitTime > 0L) 1231 { 1232 out(); 1233 out("Average work queue wait time: ", 1234 decimalFormat.format(totalWorkQueueWaitTime / totalResults), "ms"); 1235 printHistogram("Count of operations by work queue wait time:", 1236 totalResults, workQueueWaitTimes); 1237 } 1238 1239 printResultCodeCounts(addResultCodes, "add"); 1240 printResultCodeCounts(bindResultCodes, "bind"); 1241 printResultCodeCounts(compareResultCodes, "compare"); 1242 printResultCodeCounts(deleteResultCodes, "delete"); 1243 printResultCodeCounts(extendedResultCodes, "extended"); 1244 printResultCodeCounts(modifyResultCodes, "modify"); 1245 printResultCodeCounts(modifyDNResultCodes, "modify DN"); 1246 printResultCodeCounts(searchResultCodes, "search"); 1247 1248 printCounts(preAuthzPrivilegesUsed, 1249 "Most common pre-authorization privileges used:", "privilege", 1250 "privileges"); 1251 printCounts(privilegesUsed, "Most common privileges used:", "privilege", 1252 "privileges"); 1253 printCounts(privilegesMissing, "Most common missing privileges:", 1254 "privilege", "privileges"); 1255 1256 printCounts(successfulBindDNs, 1257 "Most common bind DNs used in successful authentication attempts:", 1258 "DN", "DNs"); 1259 printCounts(bindFailuresByDN, 1260 "Most common bind DNs used in failed authentication attempts:", 1261 "DN", "DNs"); 1262 printCounts(bindFailuresByIPAddress, 1263 "Most common IP addresses used in failed authentication attempts:", 1264 "IP", "IPs"); 1265 if (doNotAnonymize.isPresent()) 1266 { 1267 printCounts(consecutiveFailedBindsByDN, 1268 "Bind DNs with the most consecutive authentication failures:", 1269 "DN", "DNs"); 1270 } 1271 printCounts(authenticationTypes, "Most common authentication types:", 1272 "authentication type", "authentication types"); 1273 1274 long numResultsWithAuthzID = 0L; 1275 for (final AtomicLong l : authzDNs.values()) 1276 { 1277 numResultsWithAuthzID += l.get(); 1278 } 1279 1280 out(); 1281 final double percentWithAuthzID = 1282 100.0 * numResultsWithAuthzID / totalOps; 1283 out("Number of operations with an alternate authorization identity: ", 1284 numResultsWithAuthzID, " (", 1285 decimalFormat.format(percentWithAuthzID), "%)"); 1286 1287 printCounts(authzDNs, "Most common alternate authorization identity DNs:", 1288 "DN", "DNs"); 1289 1290 if (! requestControlOIDs.isEmpty()) 1291 { 1292 final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>(); 1293 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1294 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1295 getMostCommonElements(requestControlOIDs, controlCounts, displayCount, 1296 skippedWithSameCount, skippedWithLowerCount); 1297 1298 out(); 1299 out("Most common request control types:"); 1300 1301 long count = -1L; 1302 for (final ObjectPair<String,Long> p : controlCounts) 1303 { 1304 count = p.getSecond(); 1305 final double percent = 100.0 * count / numRequestControls; 1306 1307 final String oid = p.getFirst(); 1308 final OIDRegistryItem item = OIDRegistry.getDefault().get(oid); 1309 if (item == null) 1310 { 1311 out(p.getFirst(), ": ", p.getSecond(), " (", 1312 decimalFormat.format(percent), "%)"); 1313 } 1314 else 1315 { 1316 out(p.getFirst(), " (", item.getName(), "): ", p.getSecond(), " (", 1317 decimalFormat.format(percent), "%)"); 1318 } 1319 } 1320 1321 if (skippedWithSameCount.get() > 0L) 1322 { 1323 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1324 getSingularOrPlural(skippedWithSameCount.get(), "control", 1325 "controls") + 1326 " with a count of " + count + " }"); 1327 } 1328 1329 if (skippedWithLowerCount.get() > 0L) 1330 { 1331 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1332 getSingularOrPlural(skippedWithLowerCount.get(), "control", 1333 "controls") + 1334 " with a count that is less than " + count + " }"); 1335 } 1336 } 1337 1338 if (! responseControlOIDs.isEmpty()) 1339 { 1340 final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>(); 1341 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1342 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1343 getMostCommonElements(responseControlOIDs, controlCounts, displayCount, 1344 skippedWithSameCount, skippedWithLowerCount); 1345 1346 out(); 1347 out("Most common response control types:"); 1348 1349 long count = -1L; 1350 for (final ObjectPair<String,Long> p : controlCounts) 1351 { 1352 count = p.getSecond(); 1353 final double percent = 100.0 * count / numResponseControls; 1354 1355 final String oid = p.getFirst(); 1356 final OIDRegistryItem item = OIDRegistry.getDefault().get(oid); 1357 if (item == null) 1358 { 1359 out(p.getFirst(), ": ", p.getSecond(), " (", 1360 decimalFormat.format(percent), "%)"); 1361 } 1362 else 1363 { 1364 out(p.getFirst(), " (", item.getName(), "): ", p.getSecond(), " (", 1365 decimalFormat.format(percent), "%)"); 1366 } 1367 } 1368 1369 if (skippedWithSameCount.get() > 0L) 1370 { 1371 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1372 getSingularOrPlural(skippedWithSameCount.get(), "control", 1373 "controls") + 1374 " with a count of " + count + " }"); 1375 } 1376 1377 if (skippedWithLowerCount.get() > 0L) 1378 { 1379 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1380 getSingularOrPlural(skippedWithLowerCount.get(), "control", 1381 "controls") + 1382 " with a count that is less than " + count + " }"); 1383 } 1384 } 1385 1386 if (! extendedOperations.isEmpty()) 1387 { 1388 final List<ObjectPair<String,Long>> extOpCounts = new ArrayList<>(); 1389 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1390 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1391 getMostCommonElements(extendedOperations, extOpCounts, displayCount, 1392 skippedWithSameCount, skippedWithLowerCount); 1393 1394 out(); 1395 out("Most common extended operation types:"); 1396 1397 long count = -1L; 1398 for (final ObjectPair<String,Long> p : extOpCounts) 1399 { 1400 count = p.getSecond(); 1401 final double percent = 100.0 * count / numExtended; 1402 1403 final String oid = p.getFirst(); 1404 final String name = extendedOperationOIDsToNames.get(oid); 1405 if (name == null) 1406 { 1407 out(p.getFirst(), ": ", p.getSecond(), " (", 1408 decimalFormat.format(percent), "%)"); 1409 } 1410 else 1411 { 1412 out(p.getFirst(), " (", name, "): ", p.getSecond(), " (", 1413 decimalFormat.format(percent), "%)"); 1414 } 1415 } 1416 1417 if (skippedWithSameCount.get() > 0L) 1418 { 1419 out("{ Skipped " + skippedWithSameCount.get() + 1420 " additional extended " + 1421 getSingularOrPlural(skippedWithSameCount.get(), "operation", 1422 "operations") + 1423 " with a count of " + count + " }"); 1424 } 1425 1426 if (skippedWithLowerCount.get() > 0L) 1427 { 1428 out("{ Skipped " + skippedWithLowerCount.get() + 1429 " additional extended " + 1430 getSingularOrPlural(skippedWithLowerCount.get(), "operation", 1431 "operations") + 1432 " with a count that is less than " + count + " }"); 1433 } 1434 } 1435 1436 out(); 1437 out("Number of unindexed search attempts: ", numUnindexedAttempts); 1438 out("Number of successfully-completed unindexed searches: ", 1439 numUnindexedSuccessful); 1440 out("Number of failed unindexed searches: ", numUnindexedFailed); 1441 1442 printCounts(unindexedFilters, "Most common unindexed search filters:", 1443 "filter", "filters"); 1444 1445 if (! searchScopes.isEmpty()) 1446 { 1447 final List<ObjectPair<SearchScope,Long>> scopeCounts = 1448 new ArrayList<>(); 1449 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1450 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1451 getMostCommonElements(searchScopes, scopeCounts, displayCount, 1452 skippedWithSameCount, skippedWithLowerCount); 1453 1454 out(); 1455 out("Most common search scopes:"); 1456 1457 long count = -1L; 1458 for (final ObjectPair<SearchScope,Long> p : scopeCounts) 1459 { 1460 count = p.getSecond(); 1461 final double percent = 100.0 * count / numSearches; 1462 out(p.getFirst().getName().toLowerCase(), " (", 1463 p.getFirst().intValue(), "): ", p.getSecond(), " (", 1464 decimalFormat.format(percent), "%)"); 1465 } 1466 1467 if (skippedWithSameCount.get() > 0L) 1468 { 1469 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1470 getSingularOrPlural(skippedWithSameCount.get(), "scope", 1471 "scopes") + 1472 " with a count of " + count + " }"); 1473 } 1474 1475 if (skippedWithLowerCount.get() > 0L) 1476 { 1477 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1478 getSingularOrPlural(skippedWithLowerCount.get(), "scope", 1479 "scopes") + 1480 " with a count that is less than " + count + " }"); 1481 } 1482 } 1483 1484 if (! searchEntryCounts.isEmpty()) 1485 { 1486 final List<ObjectPair<Long,Long>> entryCounts = new ArrayList<>(); 1487 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1488 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1489 getMostCommonElements(searchEntryCounts, entryCounts, displayCount, 1490 skippedWithSameCount, skippedWithLowerCount); 1491 1492 out(); 1493 out("Most common search entry counts:"); 1494 1495 long count = -1L; 1496 for (final ObjectPair<Long,Long> p : entryCounts) 1497 { 1498 count = p.getSecond(); 1499 final double percent = 100.0 * count / numSearches; 1500 out(p.getFirst(), " matching ", 1501 getSingularOrPlural(p.getFirst(), "entry", "entries"), 1502 ": ", p.getSecond(), " (", decimalFormat.format(percent), "%)"); 1503 } 1504 1505 if (skippedWithSameCount.get() > 0L) 1506 { 1507 out("{ Skipped " + skippedWithSameCount.get() + " additional entry " + 1508 getSingularOrPlural(skippedWithSameCount.get(), "count", 1509 "counts") + 1510 " with a count of " + count + " }"); 1511 } 1512 1513 if (skippedWithLowerCount.get() > 0L) 1514 { 1515 out("{ Skipped " + skippedWithLowerCount.get() + 1516 " additional entry " + 1517 getSingularOrPlural(skippedWithLowerCount.get(), "count", 1518 "counts") + 1519 " with a count that is less than " + count + " }"); 1520 } 1521 } 1522 1523 printCounts(searchBaseDNs, 1524 "Most common base DNs for searches with a non-base scope:", 1525 "base DN", "base DNs"); 1526 1527 printCounts(filterTypes, 1528 "Most common filters for searches with a non-base scope:", 1529 "filter", "filters"); 1530 1531 printCounts(filterComponentCounts, 1532 "Most common search filter component counts:", "filter", 1533 "filters"); 1534 1535 if (doNotAnonymize.isPresent() && 1536 (! filtersRepresentingPotentialInjectionAttempt.isEmpty())) 1537 { 1538 out(); 1539 wrapOut(0, WRAP_COLUMN, 1540 "Search filters that may indicate an unsuccessful injection " + 1541 "attempt. These include filters with an assertion value " + 1542 "that contains one or more of the following: parentheses, " + 1543 "ampersands, pipes, single quotes, double quotes, or the " + 1544 "words 'select' and 'from':"); 1545 for (final Filter f : filtersRepresentingPotentialInjectionAttempt) 1546 { 1547 out("* " + f.toString()); 1548 } 1549 } 1550 1551 if (numSearches > 0L) 1552 { 1553 long numSearchesMatchingNoEntries = 0L; 1554 for (final AtomicLong l : noEntryFilters.values()) 1555 { 1556 numSearchesMatchingNoEntries += l.get(); 1557 } 1558 1559 out(); 1560 final double noEntryPercent = 1561 100.0 * numSearchesMatchingNoEntries / numSearches; 1562 out("Number of searches matching no entries: ", 1563 numSearchesMatchingNoEntries, " (", 1564 decimalFormat.format(noEntryPercent), "%)"); 1565 1566 printCounts(noEntryFilters, 1567 "Most common filters for searches matching no entries:", 1568 "filter", "filters"); 1569 1570 1571 long numSearchesMatchingOneEntry = 0L; 1572 for (final AtomicLong l : oneEntryFilters.values()) 1573 { 1574 numSearchesMatchingOneEntry += l.get(); 1575 } 1576 1577 out(); 1578 final double oneEntryPercent = 1579 100.0 * numSearchesMatchingOneEntry / numSearches; 1580 out("Number of searches matching one entry: ", 1581 numSearchesMatchingOneEntry, " (", 1582 decimalFormat.format(oneEntryPercent), "%)"); 1583 1584 printCounts(oneEntryFilters, 1585 "Most common filters for searches matching one entry:", 1586 "filter", "filters"); 1587 1588 1589 long numSearchesMatchingMultipleEntries = 0L; 1590 for (final AtomicLong l : multiEntryFilters.values()) 1591 { 1592 numSearchesMatchingMultipleEntries += l.get(); 1593 } 1594 1595 out(); 1596 final double multiEntryPercent = 1597 100.0 * numSearchesMatchingMultipleEntries / numSearches; 1598 out("Number of searches matching multiple entries: ", 1599 numSearchesMatchingMultipleEntries, " (", 1600 decimalFormat.format(multiEntryPercent), "%)"); 1601 1602 printCounts(multiEntryFilters, 1603 "Most common filters for searches matching multiple entries:", 1604 "filter", "filters"); 1605 } 1606 } 1607 1608 if (! mostExpensiveFilters.isEmpty()) 1609 { 1610 final List<ObjectPair<String,Long>> filterDurations = new ArrayList<>(); 1611 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1612 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1613 getMostCommonElements(mostExpensiveFilters, filterDurations, 1614 displayCount, skippedWithSameCount, skippedWithLowerCount); 1615 1616 out(); 1617 out("Filters for searches with the longest processing times:"); 1618 1619 String durationStr = ""; 1620 for (final ObjectPair<String,Long> p : filterDurations) 1621 { 1622 final long durationMicros = p.getSecond(); 1623 final double durationMillis = durationMicros / 1_000.0; 1624 durationStr = decimalFormat.format(durationMillis) + " ms"; 1625 out(p.getFirst(), ": ", durationStr); 1626 } 1627 1628 if (skippedWithSameCount.get() > 0L) 1629 { 1630 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1631 getSingularOrPlural(skippedWithSameCount.get(), "filter", 1632 "filters") + 1633 " with a duration of " + durationStr + " }"); 1634 } 1635 1636 if (skippedWithLowerCount.get() > 0L) 1637 { 1638 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1639 getSingularOrPlural(skippedWithLowerCount.get(), "filter", 1640 "filters") + 1641 " with a duration that is less than " + durationStr + " }"); 1642 } 1643 } 1644 1645 final long totalUncached = numUncachedAdds + numUncachedBinds + 1646 numUncachedCompares + numUncachedDeletes + numUncachedExtended + 1647 numUncachedModifies + numUncachedModifyDNs + numUncachedSearches; 1648 if (totalUncached > 0L) 1649 { 1650 out(); 1651 out("Operations accessing uncached data:"); 1652 printUncached("Add", numUncachedAdds, numAdds); 1653 printUncached("Bind", numUncachedBinds, numBinds); 1654 printUncached("Compare", numUncachedCompares, numCompares); 1655 printUncached("Delete", numUncachedDeletes, numDeletes); 1656 printUncached("Extended", numUncachedExtended, numExtended); 1657 printUncached("Modify", numUncachedModifies, numModifies); 1658 printUncached("Modify DN", numUncachedModifyDNs, numModifyDNs); 1659 printUncached("Search", numUncachedSearches, numSearches); 1660 } 1661 1662 1663 return ResultCode.SUCCESS; 1664 } 1665 1666 1667 1668 /** 1669 * Retrieves a set of information that may be used to generate example usage 1670 * information. Each element in the returned map should consist of a map 1671 * between an example set of arguments and a string that describes the 1672 * behavior of the tool when invoked with that set of arguments. 1673 * 1674 * @return A set of information that may be used to generate example usage 1675 * information. It may be {@code null} or empty if no example usage 1676 * information is available. 1677 */ 1678 @Override() 1679 @NotNull() 1680 public LinkedHashMap<String[],String> getExampleUsages() 1681 { 1682 final LinkedHashMap<String[],String> examples = 1683 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 1684 1685 final String[] args = 1686 { 1687 "/ds/logs/access" 1688 }; 1689 final String description = 1690 "Analyze the contents of the /ds/logs/access access log file."; 1691 examples.put(args, description); 1692 1693 return examples; 1694 } 1695 1696 1697 1698 /** 1699 * Populates the provided processing time map with an initial set of values. 1700 * 1701 * @param m The processing time map to be populated. 1702 */ 1703 private static void populateProcessingTimeMap( 1704 @NotNull final HashMap<Long,AtomicLong> m) 1705 { 1706 m.put(1L, new AtomicLong(0L)); 1707 m.put(2L, new AtomicLong(0L)); 1708 m.put(3L, new AtomicLong(0L)); 1709 m.put(5L, new AtomicLong(0L)); 1710 m.put(10L, new AtomicLong(0L)); 1711 m.put(20L, new AtomicLong(0L)); 1712 m.put(30L, new AtomicLong(0L)); 1713 m.put(50L, new AtomicLong(0L)); 1714 m.put(100L, new AtomicLong(0L)); 1715 m.put(1_000L, new AtomicLong(0L)); 1716 m.put(2_000L, new AtomicLong(0L)); 1717 m.put(3_000L, new AtomicLong(0L)); 1718 m.put(5_000L, new AtomicLong(0L)); 1719 m.put(10_000L, new AtomicLong(0L)); 1720 m.put(20_000L, new AtomicLong(0L)); 1721 m.put(30_000L, new AtomicLong(0L)); 1722 m.put(60_000L, new AtomicLong(0L)); 1723 m.put(Long.MAX_VALUE, new AtomicLong(0L)); 1724 } 1725 1726 1727 1728 /** 1729 * Performs any necessary processing for a connect message. 1730 * 1731 * @param m The log message to be processed. 1732 */ 1733 private void processConnect(@NotNull final ConnectAccessLogMessage m) 1734 { 1735 numConnects++; 1736 1737 final String clientAddr = m.getSourceAddress(); 1738 if (clientAddr != null) 1739 { 1740 final Long connectionID = m.getConnectionID(); 1741 if (connectionID != null) 1742 { 1743 ipAddressesByConnectionID.put(connectionID, clientAddr); 1744 } 1745 1746 AtomicLong count = clientAddresses.get(clientAddr); 1747 if (count == null) 1748 { 1749 count = new AtomicLong(0L); 1750 clientAddresses.put(clientAddr, count); 1751 } 1752 count.incrementAndGet(); 1753 } 1754 1755 final String ccp = m.getClientConnectionPolicy(); 1756 if (ccp != null) 1757 { 1758 AtomicLong l = clientConnectionPolicies.get(ccp); 1759 if (l == null) 1760 { 1761 l = new AtomicLong(0L); 1762 clientConnectionPolicies.put(ccp, l); 1763 } 1764 l.incrementAndGet(); 1765 } 1766 } 1767 1768 1769 1770 /** 1771 * Performs any necessary processing for a security negotiation message. 1772 * 1773 * @param m The log message to be processed. 1774 */ 1775 private void processSecurityNegotiation( 1776 @NotNull final SecurityNegotiationAccessLogMessage m) 1777 { 1778 final String protocol = m.getProtocol(); 1779 if (protocol != null) 1780 { 1781 AtomicLong l = tlsProtocols.get(protocol); 1782 if (l == null) 1783 { 1784 l = new AtomicLong(0L); 1785 tlsProtocols.put(protocol, l); 1786 } 1787 l.incrementAndGet(); 1788 } 1789 1790 final String cipherSuite = m.getCipher(); 1791 if (cipherSuite != null) 1792 { 1793 AtomicLong l = tlsCipherSuites.get(cipherSuite); 1794 if (l == null) 1795 { 1796 l = new AtomicLong(0L); 1797 tlsCipherSuites.put(cipherSuite, l); 1798 } 1799 l.incrementAndGet(); 1800 } 1801 } 1802 1803 1804 1805 /** 1806 * Performs any necessary processing for a disconnect message. 1807 * 1808 * @param m The log message to be processed. 1809 */ 1810 private void processDisconnect(@NotNull final DisconnectAccessLogMessage m) 1811 { 1812 numDisconnects++; 1813 1814 final Long connectionID = m.getConnectionID(); 1815 if (connectionID != null) 1816 { 1817 ipAddressesByConnectionID.remove(connectionID); 1818 } 1819 1820 final String reason = m.getDisconnectReason(); 1821 if (reason != null) 1822 { 1823 AtomicLong l = disconnectReasons.get(reason); 1824 if (l == null) 1825 { 1826 l = new AtomicLong(0L); 1827 disconnectReasons.put(reason, l); 1828 } 1829 l.incrementAndGet(); 1830 } 1831 } 1832 1833 1834 1835 /** 1836 * Performs any necessary processing for an abandon request message. 1837 * 1838 * @param m The log message to be processed. 1839 */ 1840 private void processAbandonRequest( 1841 @NotNull final AbandonRequestAccessLogMessage m) 1842 { 1843 numAbandons++; 1844 } 1845 1846 1847 1848 /** 1849 * Performs any necessary processing for an extended request message. 1850 * 1851 * @param m The log message to be processed. 1852 */ 1853 private void processExtendedRequest( 1854 @NotNull final ExtendedRequestAccessLogMessage m) 1855 { 1856 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1857 processExtendedRequestInternal(m); 1858 } 1859 1860 1861 1862 /** 1863 * Performs the internal processing for an extended request message. 1864 * 1865 * @param m The log message to be processed. 1866 */ 1867 private void processExtendedRequestInternal( 1868 @NotNull final ExtendedRequestAccessLogMessage m) 1869 { 1870 final String oid = m.getRequestOID(); 1871 if (oid != null) 1872 { 1873 AtomicLong l = extendedOperations.get(oid); 1874 if (l == null) 1875 { 1876 l = new AtomicLong(0L); 1877 extendedOperations.put(oid, l); 1878 } 1879 l.incrementAndGet(); 1880 1881 final String requestType = m.getRequestType(); 1882 if ((requestType != null) && 1883 (! extendedOperationOIDsToNames.containsKey(oid))) 1884 { 1885 extendedOperationOIDsToNames.put(oid, requestType); 1886 } 1887 } 1888 } 1889 1890 1891 1892 /** 1893 * Performs any necessary processing for a search request message. 1894 * 1895 * @param m The log message to be processed. 1896 */ 1897 private void processSearchRequest( 1898 @NotNull final SearchRequestAccessLogMessage m) 1899 { 1900 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1901 processSearchRequestInternal(m); 1902 } 1903 1904 1905 1906 /** 1907 * Performs any necessary processing for a search request message. 1908 * 1909 * @param m The log message to be processed. 1910 */ 1911 private void processSearchRequestInternal( 1912 @NotNull final SearchRequestAccessLogMessage m) 1913 { 1914 final SearchScope scope = m.getScope(); 1915 if (scope != null) 1916 { 1917 AtomicLong scopeCount = searchScopes.get(scope); 1918 if (scopeCount == null) 1919 { 1920 scopeCount = new AtomicLong(0L); 1921 searchScopes.put(scope, scopeCount); 1922 } 1923 scopeCount.incrementAndGet(); 1924 1925 if (! scope.equals(SearchScope.BASE)) 1926 { 1927 final String filterString = prepareFilter(m.getFilter()); 1928 if (filterString != null) 1929 { 1930 AtomicLong filterCount = filterTypes.get(filterString); 1931 if (filterCount == null) 1932 { 1933 filterCount = new AtomicLong(0L); 1934 filterTypes.put(filterString, filterCount); 1935 } 1936 filterCount.incrementAndGet(); 1937 1938 1939 final String baseDN = getDNString(m.getBaseDN()); 1940 if (baseDN != null) 1941 { 1942 AtomicLong baseDNCount = searchBaseDNs.get(baseDN); 1943 if (baseDNCount == null) 1944 { 1945 baseDNCount = new AtomicLong(0L); 1946 searchBaseDNs.put(baseDN, baseDNCount); 1947 } 1948 baseDNCount.incrementAndGet(); 1949 } 1950 } 1951 } 1952 } 1953 1954 final String filterString = m.getFilter(); 1955 if (filterString != null) 1956 { 1957 try 1958 { 1959 final Filter filter = Filter.create(filterString); 1960 if (mayRepresentInjectionAttempt(filter)) 1961 { 1962 filtersRepresentingPotentialInjectionAttempt.add(filter); 1963 } 1964 1965 1966 final int numComponents = countComponents(filter); 1967 final String label; 1968 if (numComponents == 1) 1969 { 1970 label = "1 component"; 1971 } 1972 else 1973 { 1974 label = numComponents + " components"; 1975 } 1976 1977 AtomicLong count = filterComponentCounts.get(label); 1978 if (count == null) 1979 { 1980 count = new AtomicLong(0L); 1981 filterComponentCounts.put(label, count); 1982 } 1983 1984 count.incrementAndGet(); 1985 } 1986 catch (final Exception e) 1987 { 1988 Debug.debugException(e); 1989 } 1990 } 1991 } 1992 1993 1994 1995 /** 1996 * Indicates whether the provided search filter may represent an injection 1997 * attempt. Filters that may represent injection attempts include: 1998 * <UL> 1999 * <LI>Filters with assertion values that contain parentheses, ampersands, 2000 * pipes, or single or double quotes.</LI> 2001 * <LI>Filters that contain the words "select" and "from".</LI> 2002 * </UL> 2003 * 2004 * @param filter The filter to examine. It must not be {@code null}. 2005 * 2006 * @return {@code true} if the provided filter may represent an injection 2007 * attempt, or {@code false} if not. 2008 */ 2009 static boolean mayRepresentInjectionAttempt(@NotNull final Filter filter) 2010 { 2011 switch (filter.getFilterType()) 2012 { 2013 case Filter.FILTER_TYPE_AND: 2014 case Filter.FILTER_TYPE_OR: 2015 for (final Filter f : filter.getComponents()) 2016 { 2017 if (mayRepresentInjectionAttempt(f)) 2018 { 2019 return true; 2020 } 2021 } 2022 return false; 2023 2024 case Filter.FILTER_TYPE_NOT: 2025 return mayRepresentInjectionAttempt(filter.getNOTComponent()); 2026 2027 case Filter.FILTER_TYPE_EQUALITY: 2028 case Filter.FILTER_TYPE_GREATER_OR_EQUAL: 2029 case Filter.FILTER_TYPE_LESS_OR_EQUAL: 2030 case Filter.FILTER_TYPE_APPROXIMATE_MATCH: 2031 case Filter.FILTER_TYPE_EXTENSIBLE_MATCH: 2032 return mayRepresentInjectionAttempt(filter.getAssertionValue()); 2033 2034 case Filter.FILTER_TYPE_SUBSTRING: 2035 final String[] subAnyStrings = filter.getSubAnyStrings(); 2036 if (subAnyStrings != null) 2037 { 2038 for (final String subAnyString : subAnyStrings) 2039 { 2040 if (mayRepresentInjectionAttempt(subAnyString)) 2041 { 2042 return true; 2043 } 2044 } 2045 } 2046 2047 return mayRepresentInjectionAttempt(filter.getSubInitialString()) || 2048 mayRepresentInjectionAttempt(filter.getSubFinalString()); 2049 2050 case Filter.FILTER_TYPE_PRESENCE: 2051 default: 2052 return false; 2053 } 2054 } 2055 2056 2057 2058 /** 2059 * Indicates whether the provided string (which should be a filter assertion 2060 * value or substring component) may represent an injection attempt. 2061 * 2062 * @param value The value for which to make the determination. It may 2063 * optionally be {@code null}. 2064 * 2065 * @return {@code true} if the provided value may represent an injection 2066 * attempt, or {@code false} if not. 2067 */ 2068 private static boolean mayRepresentInjectionAttempt( 2069 @Nullable final String value) 2070 { 2071 if (value == null) 2072 { 2073 return false; 2074 } 2075 2076 final String lowerValue = StaticUtils.toLowerCase(value); 2077 return (lowerValue.contains("(") || 2078 lowerValue.contains(")") || 2079 lowerValue.contains("&") || 2080 lowerValue.contains("|") || 2081 lowerValue.contains("\"") || 2082 lowerValue.contains("'") || 2083 ((lowerValue.contains("select") && lowerValue.contains("from")))); 2084 } 2085 2086 2087 2088 /** 2089 * Counts the number of components in the specified filter. Presence, 2090 * equality, substring, greater-or-equal, less-or-equal, approximate-match, 2091 * and extensible-match filters will all be considered a single component. 2092 * AND and OR filters will be one plus the aggregate component count for each 2093 * of the components they contain. NOT filters will be one plus the component 2094 * count for the filter it contains. 2095 * 2096 * @param filter The filter for which to count the number of components. It 2097 * must not be {@code null}. 2098 * 2099 * @return The number of components in the specified filter. 2100 */ 2101 static int countComponents(@NotNull final Filter filter) 2102 { 2103 switch (filter.getFilterType()) 2104 { 2105 case Filter.FILTER_TYPE_AND: 2106 case Filter.FILTER_TYPE_OR: 2107 int count = 1; 2108 for (final Filter f : filter.getComponents()) 2109 { 2110 count += countComponents(f); 2111 } 2112 return count; 2113 2114 case Filter.FILTER_TYPE_NOT: 2115 return 1 + countComponents(filter.getNOTComponent()); 2116 2117 case Filter.FILTER_TYPE_PRESENCE: 2118 case Filter.FILTER_TYPE_EQUALITY: 2119 case Filter.FILTER_TYPE_SUBSTRING: 2120 case Filter.FILTER_TYPE_GREATER_OR_EQUAL: 2121 case Filter.FILTER_TYPE_LESS_OR_EQUAL: 2122 case Filter.FILTER_TYPE_APPROXIMATE_MATCH: 2123 case Filter.FILTER_TYPE_EXTENSIBLE_MATCH: 2124 default: 2125 return 1; 2126 } 2127 } 2128 2129 2130 2131 /** 2132 * Performs any necessary processing for an unbind request message. 2133 * 2134 * @param m The log message to be processed. 2135 */ 2136 private void processUnbindRequest( 2137 @NotNull final UnbindRequestAccessLogMessage m) 2138 { 2139 numUnbinds++; 2140 } 2141 2142 2143 2144 /** 2145 * Performs any necessary processing for an add result message. 2146 * 2147 * @param m The log message to be processed. 2148 */ 2149 private void processAddResult(@NotNull final AddResultAccessLogMessage m) 2150 { 2151 numAdds++; 2152 2153 updateCommonResult(m); 2154 2155 updateResultCodeCount(m.getResultCode(), addResultCodes); 2156 addProcessingDuration += 2157 doubleValue(m.getProcessingTimeMillis(), addProcessingTimes); 2158 2159 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2160 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2161 { 2162 numUncachedAdds++; 2163 } 2164 2165 updateAuthzCount(m.getAlternateAuthorizationDN()); 2166 } 2167 2168 2169 2170 /** 2171 * Performs any necessary processing for a bind result message. 2172 * 2173 * @param m The log message to be processed. 2174 */ 2175 private void processBindResult(@NotNull final BindResultAccessLogMessage m) 2176 { 2177 numBinds++; 2178 2179 updateCommonResult(m); 2180 2181 if (m.getAuthenticationType() != null) 2182 { 2183 final String authType; 2184 switch (m.getAuthenticationType()) 2185 { 2186 case SIMPLE: 2187 authType = "Simple"; 2188 break; 2189 2190 case SASL: 2191 final String saslMechanism = m.getSASLMechanismName(); 2192 if (saslMechanism == null) 2193 { 2194 authType = "SASL {unknown mechanism}"; 2195 } 2196 else 2197 { 2198 authType = "SASL " + saslMechanism; 2199 } 2200 break; 2201 2202 case INTERNAL: 2203 authType = "Internal"; 2204 break; 2205 2206 default: 2207 authType = m.getAuthenticationType().name(); 2208 break; 2209 } 2210 2211 AtomicLong l = authenticationTypes.get(authType); 2212 if (l == null) 2213 { 2214 l = new AtomicLong(0L); 2215 authenticationTypes.put(authType, l); 2216 } 2217 l.incrementAndGet(); 2218 } 2219 2220 updateResultCodeCount(m.getResultCode(), bindResultCodes); 2221 bindProcessingDuration += 2222 doubleValue(m.getProcessingTimeMillis(), bindProcessingTimes); 2223 2224 String authenticationDN = getDNString(m.getAuthenticationDN()); 2225 if (m.getResultCode() == ResultCode.SUCCESS) 2226 { 2227 if (authenticationDN != null) 2228 { 2229 AtomicLong l = successfulBindDNs.get(authenticationDN); 2230 if (l == null) 2231 { 2232 l = new AtomicLong(0L); 2233 successfulBindDNs.put(authenticationDN, l); 2234 } 2235 l.incrementAndGet(); 2236 2237 final AtomicLong outstandingFailures = 2238 outstandingFailedBindDNs.remove(authenticationDN); 2239 if (outstandingFailures != null) 2240 { 2241 final AtomicLong consecutiveFailures = 2242 consecutiveFailedBindsByDN.get(authenticationDN); 2243 if ((consecutiveFailures == null) || 2244 (outstandingFailures.get() > consecutiveFailures.get())) 2245 { 2246 consecutiveFailedBindsByDN.put(authenticationDN, 2247 new AtomicLong(outstandingFailures.get())); 2248 } 2249 } 2250 } 2251 2252 final String ccp = m.getClientConnectionPolicy(); 2253 if (ccp != null) 2254 { 2255 AtomicLong l = clientConnectionPolicies.get(ccp); 2256 if (l == null) 2257 { 2258 l = new AtomicLong(0L); 2259 clientConnectionPolicies.put(ccp, l); 2260 } 2261 l.incrementAndGet(); 2262 } 2263 } 2264 else if ((m.getResultCode() != ResultCode.SASL_BIND_IN_PROGRESS) && 2265 (m.getResultCode() != ResultCode.REFERRAL)) 2266 { 2267 if (authenticationDN == null) 2268 { 2269 authenticationDN = getDNString(m.getDN()); 2270 } 2271 2272 if (authenticationDN != null) 2273 { 2274 AtomicLong l = bindFailuresByDN.get(authenticationDN); 2275 if (l == null) 2276 { 2277 l = new AtomicLong(0L); 2278 bindFailuresByDN.put(authenticationDN, l); 2279 } 2280 l.incrementAndGet(); 2281 2282 l = outstandingFailedBindDNs.get(authenticationDN); 2283 if (l == null) 2284 { 2285 l = new AtomicLong(0L); 2286 outstandingFailedBindDNs.put(authenticationDN, l); 2287 } 2288 l.incrementAndGet(); 2289 } 2290 2291 String ipAddress = m.getRequesterIPAddress(); 2292 if (ipAddress == null) 2293 { 2294 final Long connectionID = m.getConnectionID(); 2295 if (connectionID != null) 2296 { 2297 ipAddress = ipAddressesByConnectionID.get(connectionID); 2298 } 2299 } 2300 2301 if (ipAddress != null) 2302 { 2303 AtomicLong l = bindFailuresByIPAddress.get(ipAddress); 2304 if (l == null) 2305 { 2306 l = new AtomicLong(0L); 2307 bindFailuresByIPAddress.put(ipAddress, l); 2308 } 2309 l.incrementAndGet(); 2310 } 2311 } 2312 2313 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2314 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2315 { 2316 numUncachedBinds++; 2317 } 2318 2319 updateAuthzCount(m.getAuthorizationDN()); 2320 } 2321 2322 2323 2324 /** 2325 * Performs any necessary processing for a compare result message. 2326 * 2327 * @param m The log message to be processed. 2328 */ 2329 private void processCompareResult( 2330 @NotNull final CompareResultAccessLogMessage m) 2331 { 2332 numCompares++; 2333 2334 updateCommonResult(m); 2335 2336 updateResultCodeCount(m.getResultCode(), compareResultCodes); 2337 compareProcessingDuration += 2338 doubleValue(m.getProcessingTimeMillis(), compareProcessingTimes); 2339 2340 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2341 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2342 { 2343 numUncachedCompares++; 2344 } 2345 2346 updateAuthzCount(m.getAlternateAuthorizationDN()); 2347 } 2348 2349 2350 2351 /** 2352 * Performs any necessary processing for a delete result message. 2353 * 2354 * @param m The log message to be processed. 2355 */ 2356 private void processDeleteResult( 2357 @NotNull final DeleteResultAccessLogMessage m) 2358 { 2359 numDeletes++; 2360 2361 updateCommonResult(m); 2362 2363 updateResultCodeCount(m.getResultCode(), deleteResultCodes); 2364 deleteProcessingDuration += 2365 doubleValue(m.getProcessingTimeMillis(), deleteProcessingTimes); 2366 2367 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2368 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2369 { 2370 numUncachedDeletes++; 2371 } 2372 2373 updateAuthzCount(m.getAlternateAuthorizationDN()); 2374 } 2375 2376 2377 2378 /** 2379 * Performs any necessary processing for an extended result message. 2380 * 2381 * @param m The log message to be processed. 2382 */ 2383 private void processExtendedResult( 2384 @NotNull final ExtendedResultAccessLogMessage m) 2385 { 2386 numExtended++; 2387 2388 updateCommonResult(m); 2389 2390 final String id = m.getConnectionID() + "-" + m.getOperationID(); 2391 if (!processedRequests.remove(id)) 2392 { 2393 processExtendedRequestInternal(m); 2394 } 2395 2396 updateResultCodeCount(m.getResultCode(), extendedResultCodes); 2397 extendedProcessingDuration += 2398 doubleValue(m.getProcessingTimeMillis(), extendedProcessingTimes); 2399 2400 final String ccp = m.getClientConnectionPolicy(); 2401 if (ccp != null) 2402 { 2403 AtomicLong l = clientConnectionPolicies.get(ccp); 2404 if (l == null) 2405 { 2406 l = new AtomicLong(0L); 2407 clientConnectionPolicies.put(ccp, l); 2408 } 2409 l.incrementAndGet(); 2410 } 2411 2412 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2413 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2414 { 2415 numUncachedExtended++; 2416 } 2417 } 2418 2419 2420 2421 /** 2422 * Performs any necessary processing for a modify result message. 2423 * 2424 * @param m The log message to be processed. 2425 */ 2426 private void processModifyResult( 2427 @NotNull final ModifyResultAccessLogMessage m) 2428 { 2429 numModifies++; 2430 2431 updateCommonResult(m); 2432 2433 updateResultCodeCount(m.getResultCode(), modifyResultCodes); 2434 modifyProcessingDuration += 2435 doubleValue(m.getProcessingTimeMillis(), modifyProcessingTimes); 2436 2437 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2438 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2439 { 2440 numUncachedModifies++; 2441 } 2442 2443 updateAuthzCount(m.getAlternateAuthorizationDN()); 2444 } 2445 2446 2447 2448 /** 2449 * Performs any necessary processing for a modify DN result message. 2450 * 2451 * @param m The log message to be processed. 2452 */ 2453 private void processModifyDNResult( 2454 @NotNull final ModifyDNResultAccessLogMessage m) 2455 { 2456 numModifyDNs++; 2457 2458 updateCommonResult(m); 2459 2460 updateResultCodeCount(m.getResultCode(), modifyDNResultCodes); 2461 modifyDNProcessingDuration += 2462 doubleValue(m.getProcessingTimeMillis(), modifyDNProcessingTimes); 2463 2464 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2465 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2466 { 2467 numUncachedModifyDNs++; 2468 } 2469 2470 updateAuthzCount(m.getAlternateAuthorizationDN()); 2471 } 2472 2473 2474 2475 /** 2476 * Performs any necessary processing for a search result message. 2477 * 2478 * @param m The log message to be processed. 2479 */ 2480 private void processSearchResult( 2481 @NotNull final SearchResultAccessLogMessage m) 2482 { 2483 numSearches++; 2484 2485 updateCommonResult(m); 2486 2487 final String id = m.getConnectionID() + "-" + m.getOperationID(); 2488 if (! processedRequests.remove(id)) 2489 { 2490 processSearchRequestInternal(m); 2491 } 2492 2493 final ResultCode resultCode = m.getResultCode(); 2494 updateResultCodeCount(resultCode, searchResultCodes); 2495 searchProcessingDuration += 2496 doubleValue(m.getProcessingTimeMillis(), searchProcessingTimes); 2497 2498 final String filterString = prepareFilter(m.getFilter()); 2499 2500 final Long entryCount = m.getEntriesReturned(); 2501 if (entryCount != null) 2502 { 2503 AtomicLong l = searchEntryCounts.get(entryCount); 2504 if (l == null) 2505 { 2506 l = new AtomicLong(0L); 2507 searchEntryCounts.put(entryCount, l); 2508 } 2509 l.incrementAndGet(); 2510 2511 final Map<String,AtomicLong> filterCountMap; 2512 switch (entryCount.intValue()) 2513 { 2514 case 0: 2515 filterCountMap = noEntryFilters; 2516 break; 2517 case 1: 2518 filterCountMap = oneEntryFilters; 2519 break; 2520 default: 2521 filterCountMap = multiEntryFilters; 2522 break; 2523 } 2524 2525 if (filterString != null) 2526 { 2527 AtomicLong filterCount = filterCountMap.get(filterString); 2528 if (filterCount == null) 2529 { 2530 filterCount = new AtomicLong(0L); 2531 filterCountMap.put(filterString, filterCount); 2532 } 2533 filterCount.incrementAndGet(); 2534 } 2535 } 2536 2537 final Boolean isUnindexed = m.getUnindexed(); 2538 if ((isUnindexed != null) && isUnindexed) 2539 { 2540 numUnindexedAttempts++; 2541 if (resultCode == ResultCode.SUCCESS) 2542 { 2543 numUnindexedSuccessful++; 2544 } 2545 else 2546 { 2547 numUnindexedFailed++; 2548 } 2549 2550 if (filterString != null) 2551 { 2552 AtomicLong l = unindexedFilters.get(filterString); 2553 if (l == null) 2554 { 2555 l = new AtomicLong(0L); 2556 unindexedFilters.put(filterString, l); 2557 } 2558 l.incrementAndGet(); 2559 } 2560 } 2561 2562 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2563 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2564 { 2565 numUncachedSearches++; 2566 } 2567 2568 updateAuthzCount(m.getAlternateAuthorizationDN()); 2569 2570 final Double processingTimeMillis = m.getProcessingTimeMillis(); 2571 if ((processingTimeMillis != null) && (filterString != null)) 2572 { 2573 final long processingTimeMicros = 2574 Math.round(processingTimeMillis * 1_000.0); 2575 2576 AtomicLong l = mostExpensiveFilters.get(filterString); 2577 if (l == null) 2578 { 2579 l = new AtomicLong(processingTimeMicros); 2580 mostExpensiveFilters.put(filterString, l); 2581 } 2582 else 2583 { 2584 final long previousProcessingTimeMicros = l.get(); 2585 if (processingTimeMicros > previousProcessingTimeMicros) 2586 { 2587 l.set(processingTimeMicros); 2588 } 2589 } 2590 } 2591 } 2592 2593 2594 2595 /** 2596 * Updates a number of statistics that are common to all types of result log 2597 * messages. 2598 * 2599 * @param m The result log message to examine. 2600 */ 2601 private void updateCommonResult( 2602 @NotNull final OperationResultAccessLogMessage m) 2603 { 2604 // Handle the work queue wait time. 2605 totalWorkQueueWaitTime += 2606 doubleValue(m.getWorkQueueWaitTimeMillis(), workQueueWaitTimes); 2607 2608 2609 // Handle request and response control OIDs. 2610 for (final String oid : m.getRequestControlOIDs()) 2611 { 2612 numRequestControls++; 2613 updateCount(requestControlOIDs, oid); 2614 } 2615 2616 for (final String oid : m.getResponseControlOIDs()) 2617 { 2618 numResponseControls++; 2619 updateCount(responseControlOIDs, oid); 2620 } 2621 2622 2623 // Handle used and missing privileges. 2624 for (final String privilegeName : m.getPreAuthorizationUsedPrivileges()) 2625 { 2626 updateCount(preAuthzPrivilegesUsed, privilegeName); 2627 } 2628 2629 for (final String privilegeName : m.getUsedPrivileges()) 2630 { 2631 updateCount(privilegesUsed, privilegeName); 2632 } 2633 2634 for (final String privilegeName : m.getMissingPrivileges()) 2635 { 2636 updateCount(privilegesMissing, privilegeName); 2637 } 2638 } 2639 2640 2641 2642 /** 2643 * Updates the counter for the given key in the provided map. If the key does 2644 * not exist, it will be added to the map. 2645 * 2646 * @param m The map to be updated. 2647 * @param key The key for which to update the count. 2648 */ 2649 private static void updateCount(@NotNull final Map<String,AtomicLong> m, 2650 @NotNull final String key) 2651 { 2652 AtomicLong count = m.get(key); 2653 if (count == null) 2654 { 2655 count = new AtomicLong(0L); 2656 m.put(key, count); 2657 } 2658 2659 count.incrementAndGet(); 2660 } 2661 2662 2663 2664 /** 2665 * Updates the count for the provided result code in the given map. 2666 * 2667 * @param rc The result code for which to update the count. 2668 * @param m The map used to hold counts by result code. 2669 */ 2670 private static void updateResultCodeCount(@Nullable final ResultCode rc, 2671 @NotNull final HashMap<ResultCode,AtomicLong> m) 2672 { 2673 if (rc == null) 2674 { 2675 return; 2676 } 2677 2678 AtomicLong l = m.get(rc); 2679 if (l == null) 2680 { 2681 l = new AtomicLong(0L); 2682 m.put(rc, l); 2683 } 2684 l.incrementAndGet(); 2685 } 2686 2687 2688 2689 /** 2690 * Retrieves the double value for the provided {@code Double} object. 2691 * 2692 * @param d The {@code Double} object for which to retrieve the value. 2693 * @param m The processing time histogram map to be updated. 2694 * 2695 * @return The double value of the provided {@code Double} object if it was 2696 * non-{@code null}, or 0.0 if it was {@code null}. 2697 */ 2698 private static double doubleValue(@Nullable final Double d, 2699 @NotNull final HashMap<Long,AtomicLong> m) 2700 { 2701 if (d == null) 2702 { 2703 return 0.0; 2704 } 2705 else 2706 { 2707 for (final Map.Entry<Long,AtomicLong> e : m.entrySet()) 2708 { 2709 if (d <= e.getKey()) 2710 { 2711 e.getValue().incrementAndGet(); 2712 break; 2713 } 2714 } 2715 2716 return d; 2717 } 2718 } 2719 2720 2721 2722 /** 2723 * Updates the provided list with the most frequently-occurring elements in 2724 * the provided map, paired with the number of times each value occurred. 2725 * 2726 * @param <K> The type of object used as the key for the 2727 * provided map. 2728 * @param countMap The map to be examined. It is expected that 2729 * the values of the map will be the count of 2730 * occurrences for the keys. 2731 * @param mostCommonElementList The list to which the values will be 2732 * updated. It must not be {@code null}, must 2733 * be empty, and must be updatable. 2734 * @param maxListSize The maximum number of items to add to the 2735 * provided list. It must be greater than 2736 * zero. 2737 * @param skippedWithSameCount A counter that will be incremented for each 2738 * map entry that is skipped with the same 2739 * count as a value that was not skipped. It 2740 * must not be {@code null} and must initially 2741 * be zero. 2742 * @param skippedWithLowerCount A counter that will be incremented for each 2743 * map entry that is skipped with a lower count 2744 * as the last value that was not skipped. It 2745 * must not be {@code null} and must initially 2746 * be zero. 2747 * 2748 * @return A list of the most frequently-occurring elements in the provided 2749 * map. 2750 */ 2751 @NotNull() 2752 private static <K> List<ObjectPair<K,Long>> getMostCommonElements( 2753 @NotNull final Map<K,AtomicLong> countMap, 2754 @NotNull final List<ObjectPair<K,Long>> mostCommonElementList, 2755 final int maxListSize, 2756 @NotNull final AtomicLong skippedWithSameCount, 2757 @NotNull final AtomicLong skippedWithLowerCount) 2758 { 2759 final TreeMap<Long,List<K>> reverseMap = 2760 new TreeMap<>(new ReverseComparator<Long>()); 2761 for (final Map.Entry<K,AtomicLong> e : countMap.entrySet()) 2762 { 2763 final Long count = e.getValue().get(); 2764 List<K> list = reverseMap.get(count); 2765 if (list == null) 2766 { 2767 list = new ArrayList<>(); 2768 reverseMap.put(count, list); 2769 } 2770 list.add(e.getKey()); 2771 } 2772 2773 for (final Map.Entry<Long,List<K>> e : reverseMap.entrySet()) 2774 { 2775 final Long l = e.getKey(); 2776 int numNotSkipped = 0; 2777 for (final K k : e.getValue()) 2778 { 2779 if (mostCommonElementList.size() >= maxListSize) 2780 { 2781 if (numNotSkipped > 0) 2782 { 2783 skippedWithSameCount.incrementAndGet(); 2784 } 2785 else 2786 { 2787 skippedWithLowerCount.incrementAndGet(); 2788 } 2789 } 2790 else 2791 { 2792 numNotSkipped++; 2793 mostCommonElementList.add(new ObjectPair<>(k, l)); 2794 } 2795 } 2796 } 2797 2798 return mostCommonElementList; 2799 } 2800 2801 2802 2803 /** 2804 * Updates the count of alternate authorization identities for the provided 2805 * DN. 2806 * 2807 * @param authzDN The DN of the alternate authorization identity that was 2808 * used. It may be {@code null} if no alternate 2809 * authorization identity was used. 2810 */ 2811 private void updateAuthzCount(@Nullable final String authzDN) 2812 { 2813 if (authzDN == null) 2814 { 2815 return; 2816 } 2817 2818 final String dnString = getDNString(authzDN); 2819 2820 AtomicLong l = authzDNs.get(dnString); 2821 if (l == null) 2822 { 2823 l = new AtomicLong(0L); 2824 authzDNs.put(dnString, l); 2825 } 2826 } 2827 2828 2829 2830 /** 2831 * Retrieves a string representation of the provided DN. It may either be 2832 * anonymized, using question marks in place of specific attribute values, or 2833 * it may be the actual string representation of the given DN. 2834 * 2835 * @param dn The DN for which to retrieve the string representation. 2836 * 2837 * @return A string representation of the provided DN, or {@code null} if the 2838 * given DN was {@code null}. 2839 */ 2840 @Nullable() 2841 private String getDNString(@Nullable final String dn) 2842 { 2843 if (dn == null) 2844 { 2845 return null; 2846 } 2847 2848 final DN parsedDN; 2849 try 2850 { 2851 parsedDN = new DN(dn); 2852 } 2853 catch (final Exception e) 2854 { 2855 Debug.debugException(e); 2856 return dn.toLowerCase(); 2857 } 2858 2859 if (parsedDN.isNullDN()) 2860 { 2861 return "{Null DN}"; 2862 } 2863 2864 if (doNotAnonymize.isPresent()) 2865 { 2866 return parsedDN.toNormalizedString(); 2867 } 2868 2869 final StringBuilder buffer = new StringBuilder(); 2870 final RDN[] rdns = parsedDN.getRDNs(); 2871 for (int i=0; i < rdns.length; i++) 2872 { 2873 if (i > 0) 2874 { 2875 buffer.append(','); 2876 } 2877 2878 final RDN rdn = rdns[i]; 2879 final String[] attributeNames = rdn.getAttributeNames(); 2880 for (int j=0; j < attributeNames.length; j++) 2881 { 2882 if (j > 0) 2883 { 2884 buffer.append('+'); 2885 } 2886 buffer.append(attributeNames[j].toLowerCase()); 2887 buffer.append("=?"); 2888 } 2889 } 2890 2891 return buffer.toString(); 2892 } 2893 2894 2895 2896 /** 2897 * Retrieves a prepared string representation of the provided search filter. 2898 * It may potentially be de-anonymized to include specific values. 2899 * 2900 * @param filterString The string representation of the filter to prepare. 2901 * It may be {@code null} if the log message does not 2902 * have a filter. 2903 * 2904 * @return A string representation of the provided filter (which may or may 2905 * not be anonymized), or {@code null} if the provided filter is 2906 * {@code null} or cannot be prepared. 2907 */ 2908 @Nullable() 2909 private String prepareFilter(@Nullable final String filterString) 2910 { 2911 if (filterString == null) 2912 { 2913 return null; 2914 } 2915 2916 if (doNotAnonymize.isPresent()) 2917 { 2918 return filterString.toLowerCase(); 2919 } 2920 2921 try 2922 { 2923 return new GenericFilter(Filter.create(filterString)).toString(). 2924 toLowerCase(); 2925 } 2926 catch (final Exception e) 2927 { 2928 Debug.debugException(e); 2929 return null; 2930 } 2931 } 2932 2933 2934 2935 /** 2936 * Writes a breakdown of the processing times for a specified type of 2937 * operation. 2938 * 2939 * @param t The name of the operation type. 2940 * @param n The total number of operations of the specified type that were 2941 * processed by the server. 2942 * @param m The map of operation counts by processing time bucket. 2943 */ 2944 private void printProcessingTimeHistogram(@NotNull final String t, 2945 final long n, 2946 @NotNull final LinkedHashMap<Long,AtomicLong> m) 2947 { 2948 printHistogram("Count of " + t + " operations by processing time:", n, m); 2949 } 2950 2951 2952 2953 /** 2954 * Writes a breakdown of the processing times for a specified type of 2955 * operation. 2956 * 2957 * @param h The header to display at the beginning of the histogram. 2958 * @param n The total number of operations that were processed by the 2959 * server. 2960 * @param m The map of operation counts by processing time bucket. 2961 */ 2962 private void printHistogram(@NotNull final String h, 2963 final long n, 2964 @NotNull final LinkedHashMap<Long,AtomicLong> m) 2965 { 2966 if (n <= 0) 2967 { 2968 return; 2969 } 2970 2971 out(); 2972 out(h); 2973 2974 long lowerBound = 0; 2975 long accumulatedCount = 0; 2976 final Iterator<Map.Entry<Long,AtomicLong>> i = m.entrySet().iterator(); 2977 while (i.hasNext()) 2978 { 2979 final Map.Entry<Long,AtomicLong> e = i.next(); 2980 final long upperBound = e.getKey(); 2981 final long count = e.getValue().get(); 2982 final double categoryPercent = 100.0 * count / n; 2983 2984 accumulatedCount += count; 2985 final double accumulatedPercent = 100.0 * accumulatedCount / n; 2986 2987 if (i.hasNext()) 2988 { 2989 final String lowerBoundString; 2990 if (lowerBound == 0L) 2991 { 2992 lowerBoundString = "0 milliseconds"; 2993 } 2994 else 2995 { 2996 final long lowerBoundNanos = lowerBound * 1_000_000L; 2997 lowerBoundString = DurationArgument.nanosToDuration(lowerBoundNanos); 2998 } 2999 3000 final long upperBoundNanos = upperBound * 1_000_000L; 3001 final String upperBoundString = 3002 DurationArgument.nanosToDuration(upperBoundNanos); 3003 3004 3005 out("Between ", lowerBoundString, " and ", upperBoundString, ": ", 3006 count, " (", decimalFormat.format(categoryPercent), "%, ", 3007 decimalFormat.format(accumulatedPercent), "% accumulated)"); 3008 lowerBound = upperBound; 3009 } 3010 else 3011 { 3012 final long lowerBoundNanos = lowerBound * 1_000_000L; 3013 final String lowerBoundString = 3014 DurationArgument.nanosToDuration(lowerBoundNanos); 3015 3016 out("Greater than ", lowerBoundString, ": ", count, " (", 3017 decimalFormat.format(categoryPercent), "%, ", 3018 decimalFormat.format(accumulatedPercent), "% accumulated)"); 3019 } 3020 } 3021 } 3022 3023 3024 3025 /** 3026 * Optionally prints information about the number and percent of operations of 3027 * the specified type that involved access to uncached data. 3028 * 3029 * @param operationType The type of operation. 3030 * @param numUncached The number of operations of the specified type that 3031 * involved access to uncached data. 3032 * @param numTotal The total number of operations of the specified 3033 * type. 3034 */ 3035 private void printUncached(@NotNull final String operationType, 3036 final long numUncached, 3037 final long numTotal) 3038 { 3039 if (numUncached == 0) 3040 { 3041 return; 3042 } 3043 3044 out(operationType, ": ", numUncached, " (", 3045 decimalFormat.format(100.0 * numUncached / numTotal), "%)"); 3046 } 3047 3048 3049 3050 /** 3051 * Prints data from the provided map of counts. 3052 * 3053 * @param countMap The map containing the data to print. 3054 * @param heading The heading to display before printing the contents 3055 * of the map. 3056 * @param singularItem The name to use for a single item represented by the 3057 * key of the given map. 3058 * @param pluralItem The name to use for zero or multiple items 3059 * represented by the key of the given map. 3060 */ 3061 private void printCounts(@Nullable final Map<String,AtomicLong> countMap, 3062 @NotNull final String heading, 3063 @NotNull final String singularItem, 3064 @NotNull final String pluralItem) 3065 { 3066 if ((countMap == null) || countMap.isEmpty()) 3067 { 3068 return; 3069 } 3070 3071 long totalCount = 0L; 3072 for (final AtomicLong l : countMap.values()) 3073 { 3074 totalCount += l.get(); 3075 } 3076 3077 out(); 3078 out(heading); 3079 3080 int displayCount = reportCount.getValue(); 3081 if (displayCount <= 0L) 3082 { 3083 displayCount = Integer.MAX_VALUE; 3084 } 3085 3086 final List<ObjectPair<String,Long>> countList = new ArrayList<>(); 3087 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 3088 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 3089 getMostCommonElements(countMap, countList, displayCount, 3090 skippedWithSameCount, skippedWithLowerCount); 3091 3092 long count = -1L; 3093 for (final ObjectPair<String,Long> p : countList) 3094 { 3095 count = p.getSecond(); 3096 3097 if (totalCount > 0L) 3098 { 3099 final double percent = 100.0 * count / totalCount; 3100 out(p.getFirst(), ": ", count, " (", decimalFormat.format(percent), 3101 ")"); 3102 } 3103 else 3104 { 3105 out(p.getFirst(), ": ", count); 3106 } 3107 } 3108 3109 if (skippedWithSameCount.get() > 0L) 3110 { 3111 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 3112 getSingularOrPlural(skippedWithSameCount.get(), singularItem, 3113 pluralItem) + 3114 " with a count of " + count + " }"); 3115 } 3116 3117 if (skippedWithLowerCount.get() > 0L) 3118 { 3119 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 3120 getSingularOrPlural(skippedWithLowerCount.get(), singularItem, 3121 pluralItem) + 3122 " with a count that is less than " + count + " }"); 3123 } 3124 } 3125 3126 3127 3128 /** 3129 * Prints data from the provided map of counts. 3130 * 3131 * @param countMap The map containing the data to print. 3132 * @param operationType The type of operation represented by the keys of 3133 * the map. 3134 */ 3135 private void printResultCodeCounts( 3136 @Nullable final Map<ResultCode,AtomicLong> countMap, 3137 @NotNull final String operationType) 3138 { 3139 if ((countMap == null) || countMap.isEmpty()) 3140 { 3141 return; 3142 } 3143 3144 long totalCount = 0L; 3145 for (final AtomicLong l : countMap.values()) 3146 { 3147 totalCount += l.get(); 3148 } 3149 3150 out(); 3151 out("Most common " + operationType + " operation result codes:"); 3152 3153 int displayCount = reportCount.getValue(); 3154 if (displayCount <= 0L) 3155 { 3156 displayCount = Integer.MAX_VALUE; 3157 } 3158 3159 final List<ObjectPair<ResultCode,Long>> resultCodeList = new ArrayList<>(); 3160 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 3161 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 3162 getMostCommonElements(countMap, resultCodeList, displayCount, 3163 skippedWithSameCount, skippedWithLowerCount); 3164 3165 long count = -1L; 3166 for (final ObjectPair<ResultCode,Long> p : resultCodeList) 3167 { 3168 count = p.getSecond(); 3169 3170 if (totalCount > 0L) 3171 { 3172 final double percent = 100.0 * count / totalCount; 3173 out(p.getFirst().getName(), " (", p.getFirst().intValue(), "): ", 3174 count, " (", decimalFormat.format(percent), ")"); 3175 } 3176 else 3177 { 3178 out(p.getFirst(), ": ", count); 3179 } 3180 } 3181 3182 if (skippedWithSameCount.get() > 0L) 3183 { 3184 out("{ Skipped " + skippedWithSameCount.get() + " additional result " + 3185 getSingularOrPlural(skippedWithSameCount.get(), "code", "codes") + 3186 " with a count of " + count + " }"); 3187 } 3188 3189 if (skippedWithLowerCount.get() > 0L) 3190 { 3191 out("{ Skipped " + skippedWithLowerCount.get() + " additional result " + 3192 getSingularOrPlural(skippedWithLowerCount.get(), "code", "codes") + 3193 " with a count that is less than " + count + " }"); 3194 } 3195 } 3196 3197 3198 3199 /** 3200 * Retrieves the appropriate singular or plural form based on the given 3201 * value. 3202 * 3203 * @param count The count that will be used to determine whether to 3204 * retrieve the singular or plural form. 3205 * @param singular The singular form for the value to return. 3206 * @param plural The plural form for the value to return. 3207 * 3208 * @return The singular form if the count is 1, or the plural form if the 3209 * count is any other value. 3210 */ 3211 @NotNull() 3212 private String getSingularOrPlural(final long count, 3213 @NotNull final String singular, 3214 @NotNull final String plural) 3215 { 3216 if (count == 1L) 3217 { 3218 return singular; 3219 } 3220 else 3221 { 3222 return plural; 3223 } 3224 } 3225}