001/* 002 * Copyright 2009-2023 Ping Identity Corporation 003 * All Rights Reserved. 004 */ 005/* 006 * Copyright 2009-2023 Ping Identity Corporation 007 * 008 * Licensed under the Apache License, Version 2.0 (the "License"); 009 * you may not use this file except in compliance with the License. 010 * You may obtain a copy of the License at 011 * 012 * http://www.apache.org/licenses/LICENSE-2.0 013 * 014 * Unless required by applicable law or agreed to in writing, software 015 * distributed under the License is distributed on an "AS IS" BASIS, 016 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 017 * See the License for the specific language governing permissions and 018 * limitations under the License. 019 */ 020/* 021 * Copyright (C) 2009-2023 Ping Identity Corporation 022 * 023 * This program is free software; you can redistribute it and/or modify 024 * it under the terms of the GNU General Public License (GPLv2 only) 025 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only) 026 * as published by the Free Software Foundation. 027 * 028 * This program is distributed in the hope that it will be useful, 029 * but WITHOUT ANY WARRANTY; without even the implied warranty of 030 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 031 * GNU General Public License for more details. 032 * 033 * You should have received a copy of the GNU General Public License 034 * along with this program; if not, see <http://www.gnu.org/licenses>. 035 */ 036package com.unboundid.ldap.sdk.unboundidds.examples; 037 038 039 040import java.io.File; 041import java.io.FileInputStream; 042import java.io.InputStream; 043import java.io.IOException; 044import java.io.OutputStream; 045import java.io.Serializable; 046import java.text.DecimalFormat; 047import java.util.ArrayList; 048import java.util.HashMap; 049import java.util.HashSet; 050import java.util.Iterator; 051import java.util.LinkedHashMap; 052import java.util.LinkedHashSet; 053import java.util.List; 054import java.util.Map; 055import java.util.TreeMap; 056import java.util.concurrent.atomic.AtomicLong; 057import java.util.zip.GZIPInputStream; 058import javax.crypto.BadPaddingException; 059 060import com.unboundid.ldap.sdk.DN; 061import com.unboundid.ldap.sdk.Filter; 062import com.unboundid.ldap.sdk.LDAPException; 063import com.unboundid.ldap.sdk.RDN; 064import com.unboundid.ldap.sdk.ResultCode; 065import com.unboundid.ldap.sdk.SearchScope; 066import com.unboundid.ldap.sdk.Version; 067import com.unboundid.ldap.sdk.unboundidds.logs.LogException; 068import com.unboundid.ldap.sdk.unboundidds.logs.v2. 069 AbandonRequestAccessLogMessage; 070import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogMessage; 071import com.unboundid.ldap.sdk.unboundidds.logs.v2.AccessLogReader; 072import com.unboundid.ldap.sdk.unboundidds.logs.v2.AddResultAccessLogMessage; 073import com.unboundid.ldap.sdk.unboundidds.logs.v2.BindResultAccessLogMessage; 074import com.unboundid.ldap.sdk.unboundidds.logs.v2.CompareResultAccessLogMessage; 075import com.unboundid.ldap.sdk.unboundidds.logs.v2.ConnectAccessLogMessage; 076import com.unboundid.ldap.sdk.unboundidds.logs.v2.DeleteResultAccessLogMessage; 077import com.unboundid.ldap.sdk.unboundidds.logs.v2.DisconnectAccessLogMessage; 078import com.unboundid.ldap.sdk.unboundidds.logs.v2. 079 ExtendedRequestAccessLogMessage; 080import com.unboundid.ldap.sdk.unboundidds.logs.v2. 081 ExtendedResultAccessLogMessage; 082import com.unboundid.ldap.sdk.unboundidds.logs.v2. 083 ModifyDNResultAccessLogMessage; 084import com.unboundid.ldap.sdk.unboundidds.logs.v2.ModifyResultAccessLogMessage; 085import com.unboundid.ldap.sdk.unboundidds.logs.v2. 086 OperationRequestAccessLogMessage; 087import com.unboundid.ldap.sdk.unboundidds.logs.v2. 088 OperationResultAccessLogMessage; 089import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchRequestAccessLogMessage; 090import com.unboundid.ldap.sdk.unboundidds.logs.v2.SearchResultAccessLogMessage; 091import com.unboundid.ldap.sdk.unboundidds.logs.v2. 092 SecurityNegotiationAccessLogMessage; 093import com.unboundid.ldap.sdk.unboundidds.logs.v2.UnbindRequestAccessLogMessage; 094import com.unboundid.ldap.sdk.unboundidds.logs.v2.json.JSONAccessLogReader; 095import com.unboundid.ldap.sdk.unboundidds.logs.v2.text. 096 TextFormattedAccessLogReader; 097import com.unboundid.ldap.sdk.unboundidds.tools.ToolUtils; 098import com.unboundid.util.CommandLineTool; 099import com.unboundid.util.Debug; 100import com.unboundid.util.NotMutable; 101import com.unboundid.util.NotNull; 102import com.unboundid.util.Nullable; 103import com.unboundid.util.OIDRegistry; 104import com.unboundid.util.OIDRegistryItem; 105import com.unboundid.util.ObjectPair; 106import com.unboundid.util.ReverseComparator; 107import com.unboundid.util.StaticUtils; 108import com.unboundid.util.ThreadSafety; 109import com.unboundid.util.ThreadSafetyLevel; 110import com.unboundid.util.args.ArgumentException; 111import com.unboundid.util.args.ArgumentParser; 112import com.unboundid.util.args.BooleanArgument; 113import com.unboundid.util.args.DurationArgument; 114import com.unboundid.util.args.FileArgument; 115import com.unboundid.util.args.IntegerArgument; 116 117 118 119/** 120 * This class provides a tool that may be used to read and summarize the 121 * contents of one or more access log files from Ping Identity, UnboundID and 122 * Nokia/Alcatel-Lucent 8661 server products. 123 * <BR> 124 * <BLOCKQUOTE> 125 * <B>NOTE:</B> This class, and other classes within the 126 * {@code com.unboundid.ldap.sdk.unboundidds} package structure, are only 127 * supported for use against Ping Identity, UnboundID, and 128 * Nokia/Alcatel-Lucent 8661 server products. These classes provide support 129 * for proprietary functionality or for external specifications that are not 130 * considered stable or mature enough to be guaranteed to work in an 131 * interoperable way with other types of LDAP servers. 132 * </BLOCKQUOTE> 133 * Information that will be reported includes: 134 * <UL> 135 * <LI>The total length of time covered by the log files.</LI> 136 * <LI>The number of connections established and disconnected, the addresses 137 * of the most commonly-connecting clients, and the average rate of 138 * connects and disconnects per second.</LI> 139 * <LI>The number of operations processed, overall and by operation type, 140 * and the average rate of operations per second.</LI> 141 * <LI>The average duration for operations processed, overall and by operation 142 * type.</LI> 143 * <LI>A breakdown of operation processing times into a number of predefined 144 * categories, ranging from less than one millisecond to over one 145 * minute.</LI> 146 * <LI>A breakdown of the most common result codes for each type of operation 147 * and their relative frequencies.</LI> 148 * <LI>The most common types of extended requests processed and their 149 * relative frequencies.</LI> 150 * <LI>The number of unindexed search operations processed and the most common 151 * types of filters used in unindexed searches.</LI> 152 * <LI>A breakdown of the relative frequencies for each type of search 153 * scope.</LI> 154 * <LI>The most common types of search filters used for search 155 * operations and their relative frequencies.</LI> 156 * </UL> 157 * It is designed to work with access log files using either the default log 158 * format with separate request and response messages, as well as log files 159 * in which the request and response details have been combined on the same 160 * line. The log files to be processed should be provided as command-line 161 * arguments. 162 * <BR><BR> 163 * The APIs demonstrated by this example include: 164 * <UL> 165 * <LI>Access log parsing (from the 166 * {@code com.unboundid.ldap.sdk.unboundidds.logs} package)</LI> 167 * <LI>Argument parsing (from the {@code com.unboundid.util.args} 168 * package)</LI> 169 * </UL> 170 */ 171@NotMutable() 172@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE) 173public final class SummarizeAccessLog 174 extends CommandLineTool 175 implements Serializable 176{ 177 /** 178 * The column at which long lines should be wrapped. 179 */ 180 private static final int WRAP_COLUMN = StaticUtils.TERMINAL_WIDTH_COLUMNS - 1; 181 182 183 184 /** 185 * The serial version UID for this serializable class. 186 */ 187 private static final long serialVersionUID = 7189168366509887130L; 188 189 190 191 // Variables used for accessing argument information. 192 @Nullable private ArgumentParser argumentParser; 193 194 // An argument that may be used to indicate that the summarized output should 195 // not be anonymized, and should include attribute values. 196 @Nullable private BooleanArgument doNotAnonymize; 197 198 // An argument that may be used to indicate that the log files are compressed. 199 @Nullable private BooleanArgument isCompressed; 200 201 // An argument that may be used to indicate that the log content is 202 // JSON-formatted rather than text-formatted. 203 @Nullable private BooleanArgument json; 204 205 // An argument used to specify the encryption passphrase. 206 @Nullable private FileArgument encryptionPassphraseFile; 207 208 // An argument used to specify the maximum number of values to report for each 209 // item. 210 @Nullable private IntegerArgument reportCount; 211 212 // The decimal format that will be used for this class. 213 @NotNull private final DecimalFormat decimalFormat; 214 215 // The total duration for log content, in milliseconds. 216 private long logDurationMillis; 217 218 // The total processing time for each type of operation. 219 private double addProcessingDuration; 220 private double bindProcessingDuration; 221 private double compareProcessingDuration; 222 private double deleteProcessingDuration; 223 private double extendedProcessingDuration; 224 private double modifyProcessingDuration; 225 private double modifyDNProcessingDuration; 226 private double searchProcessingDuration; 227 228 // A variable used for tracking total work queue wait time. 229 private long totalWorkQueueWaitTime; 230 231 // A variable used for counting the number of messages of each type. 232 private long numAbandons; 233 private long numAdds; 234 private long numBinds; 235 private long numCompares; 236 private long numConnects; 237 private long numDeletes; 238 private long numDisconnects; 239 private long numExtended; 240 private long numModifies; 241 private long numModifyDNs; 242 private long numSearches; 243 private long numUnbinds; 244 245 // The number of operations of each type that accessed uncached data. 246 private long numUncachedAdds; 247 private long numUncachedBinds; 248 private long numUncachedCompares; 249 private long numUncachedDeletes; 250 private long numUncachedExtended; 251 private long numUncachedModifies; 252 private long numUncachedModifyDNs; 253 private long numUncachedSearches; 254 255 // The number of unindexed searches processed within the server. 256 private long numUnindexedAttempts; 257 private long numUnindexedFailed; 258 private long numUnindexedSuccessful; 259 260 // The number of request and response controls used. 261 private long numRequestControls; 262 private long numResponseControls; 263 264 // Variables used for maintaining counts for common types of information. 265 @NotNull private final HashMap<Long,AtomicLong> searchEntryCounts; 266 @NotNull private final HashMap<Long,String> ipAddressesByConnectionID; 267 @NotNull private final HashMap<ResultCode,AtomicLong> addResultCodes; 268 @NotNull private final HashMap<ResultCode,AtomicLong> bindResultCodes; 269 @NotNull private final HashMap<ResultCode,AtomicLong> compareResultCodes; 270 @NotNull private final HashMap<ResultCode,AtomicLong> deleteResultCodes; 271 @NotNull private final HashMap<ResultCode,AtomicLong> extendedResultCodes; 272 @NotNull private final HashMap<ResultCode,AtomicLong> modifyResultCodes; 273 @NotNull private final HashMap<ResultCode,AtomicLong> modifyDNResultCodes; 274 @NotNull private final HashMap<ResultCode,AtomicLong> searchResultCodes; 275 @NotNull private final HashMap<SearchScope,AtomicLong> searchScopes; 276 @NotNull private final HashMap<String,AtomicLong> authenticationTypes; 277 @NotNull private final HashMap<String,AtomicLong> authzDNs; 278 @NotNull private final HashMap<String,AtomicLong> bindFailuresByDN; 279 @NotNull private final HashMap<String,AtomicLong> bindFailuresByIPAddress; 280 @NotNull private final HashMap<String,AtomicLong> consecutiveFailedBindsByDN; 281 @NotNull private final HashMap<String,AtomicLong> outstandingFailedBindDNs; 282 @NotNull private final HashMap<String,AtomicLong> successfulBindDNs; 283 @NotNull private final HashMap<String,AtomicLong> clientAddresses; 284 @NotNull private final HashMap<String,AtomicLong> clientConnectionPolicies; 285 @NotNull private final HashMap<String,AtomicLong> disconnectReasons; 286 @NotNull private final HashMap<String,AtomicLong> extendedOperations; 287 @NotNull private final HashMap<String,AtomicLong> filterComponentCounts; 288 @NotNull private final HashMap<String,AtomicLong> filterTypes; 289 @NotNull private final HashMap<String,AtomicLong> mostExpensiveFilters; 290 @NotNull private final HashMap<String,AtomicLong> multiEntryFilters; 291 @NotNull private final HashMap<String,AtomicLong> noEntryFilters; 292 @NotNull private final HashMap<String,AtomicLong> oneEntryFilters; 293 @NotNull private final HashMap<String,AtomicLong> preAuthzPrivilegesUsed; 294 @NotNull private final HashMap<String,AtomicLong> privilegesMissing; 295 @NotNull private final HashMap<String,AtomicLong> privilegesUsed; 296 @NotNull private final HashMap<String,AtomicLong> requestControlOIDs; 297 @NotNull private final HashMap<String,AtomicLong> responseControlOIDs; 298 @NotNull private final HashMap<String,AtomicLong> searchBaseDNs; 299 @NotNull private final HashMap<String,AtomicLong> tlsCipherSuites; 300 @NotNull private final HashMap<String,AtomicLong> tlsProtocols; 301 @NotNull private final HashMap<String,AtomicLong> unindexedFilters; 302 @NotNull private final HashMap<String,String> extendedOperationOIDsToNames; 303 @NotNull private final HashSet<String> processedRequests; 304 @NotNull private final LinkedHashMap<Long,AtomicLong> addProcessingTimes; 305 @NotNull private final LinkedHashMap<Long,AtomicLong> bindProcessingTimes; 306 @NotNull private final LinkedHashMap<Long,AtomicLong> compareProcessingTimes; 307 @NotNull private final LinkedHashMap<Long,AtomicLong> deleteProcessingTimes; 308 @NotNull private final LinkedHashMap<Long,AtomicLong> extendedProcessingTimes; 309 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyProcessingTimes; 310 @NotNull private final LinkedHashMap<Long,AtomicLong> modifyDNProcessingTimes; 311 @NotNull private final LinkedHashMap<Long,AtomicLong> searchProcessingTimes; 312 @NotNull private final LinkedHashMap<Long,AtomicLong> workQueueWaitTimes; 313 @NotNull private final LinkedHashSet<Filter> 314 filtersRepresentingPotentialInjectionAttempt; 315 316 317 318 /** 319 * Parse the provided command line arguments and perform the appropriate 320 * processing. 321 * 322 * @param args The command line arguments provided to this program. 323 */ 324 public static void main(@NotNull final String[] args) 325 { 326 final ResultCode resultCode = main(args, System.out, System.err); 327 if (resultCode != ResultCode.SUCCESS) 328 { 329 System.exit(resultCode.intValue()); 330 } 331 } 332 333 334 335 /** 336 * Parse the provided command line arguments and perform the appropriate 337 * processing. 338 * 339 * @param args The command line arguments provided to this program. 340 * @param outStream The output stream to which standard out should be 341 * written. It may be {@code null} if output should be 342 * suppressed. 343 * @param errStream The output stream to which standard error should be 344 * written. It may be {@code null} if error messages 345 * should be suppressed. 346 * 347 * @return A result code indicating whether the processing was successful. 348 */ 349 @NotNull() 350 public static ResultCode main(@NotNull final String[] args, 351 @Nullable final OutputStream outStream, 352 @Nullable final OutputStream errStream) 353 { 354 final SummarizeAccessLog summarizer = 355 new SummarizeAccessLog(outStream, errStream); 356 return summarizer.runTool(args); 357 } 358 359 360 361 /** 362 * Creates a new instance of this tool. 363 * 364 * @param outStream The output stream to which standard out should be 365 * written. It may be {@code null} if output should be 366 * suppressed. 367 * @param errStream The output stream to which standard error should be 368 * written. It may be {@code null} if error messages 369 * should be suppressed. 370 */ 371 public SummarizeAccessLog(@Nullable final OutputStream outStream, 372 @Nullable final OutputStream errStream) 373 { 374 super(outStream, errStream); 375 376 argumentParser = null; 377 doNotAnonymize = null; 378 isCompressed = null; 379 json = null; 380 encryptionPassphraseFile = null; 381 reportCount = null; 382 383 decimalFormat = new DecimalFormat("0.000"); 384 385 logDurationMillis = 0L; 386 387 addProcessingDuration = 0.0; 388 bindProcessingDuration = 0.0; 389 compareProcessingDuration = 0.0; 390 deleteProcessingDuration = 0.0; 391 extendedProcessingDuration = 0.0; 392 modifyProcessingDuration = 0.0; 393 modifyDNProcessingDuration = 0.0; 394 searchProcessingDuration = 0.0; 395 396 totalWorkQueueWaitTime = 0L; 397 398 numAbandons = 0L; 399 numAdds = 0L; 400 numBinds = 0L; 401 numCompares = 0L; 402 numConnects = 0L; 403 numDeletes = 0L; 404 numDisconnects = 0L; 405 numExtended = 0L; 406 numModifies = 0L; 407 numModifyDNs = 0L; 408 numSearches = 0L; 409 numUnbinds = 0L; 410 411 numUncachedAdds = 0L; 412 numUncachedBinds = 0L; 413 numUncachedCompares = 0L; 414 numUncachedDeletes = 0L; 415 numUncachedExtended = 0L; 416 numUncachedModifies = 0L; 417 numUncachedModifyDNs = 0L; 418 numUncachedSearches = 0L; 419 420 numUnindexedAttempts = 0L; 421 numUnindexedFailed = 0L; 422 numUnindexedSuccessful = 0L; 423 424 numRequestControls = 0L; 425 numResponseControls = 0L; 426 427 searchEntryCounts = new HashMap<>(StaticUtils.computeMapCapacity(10)); 428 ipAddressesByConnectionID = 429 new HashMap<>(StaticUtils.computeMapCapacity(100)); 430 addResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 431 bindResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 432 compareResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 433 deleteResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 434 extendedResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 435 modifyResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 436 modifyDNResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 437 searchResultCodes = new HashMap<>(StaticUtils.computeMapCapacity(10)); 438 searchScopes = new HashMap<>(StaticUtils.computeMapCapacity(4)); 439 authenticationTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 440 authzDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 441 bindFailuresByDN = new HashMap<>(StaticUtils.computeMapCapacity(100)); 442 bindFailuresByIPAddress = 443 new HashMap<>(StaticUtils.computeMapCapacity(100)); 444 outstandingFailedBindDNs = 445 new HashMap<>(StaticUtils.computeMapCapacity(100)); 446 successfulBindDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 447 clientAddresses = new HashMap<>(StaticUtils.computeMapCapacity(100)); 448 clientConnectionPolicies = 449 new HashMap<>(StaticUtils.computeMapCapacity(100)); 450 disconnectReasons = new HashMap<>(StaticUtils.computeMapCapacity(100)); 451 extendedOperations = new HashMap<>(StaticUtils.computeMapCapacity(10)); 452 filterComponentCounts = new HashMap<>(StaticUtils.computeMapCapacity(10)); 453 filterTypes = new HashMap<>(StaticUtils.computeMapCapacity(100)); 454 mostExpensiveFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 455 multiEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 456 noEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 457 oneEntryFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 458 preAuthzPrivilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100)); 459 privilegesMissing = new HashMap<>(StaticUtils.computeMapCapacity(100)); 460 privilegesUsed = new HashMap<>(StaticUtils.computeMapCapacity(100)); 461 requestControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 462 responseControlOIDs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 463 searchBaseDNs = new HashMap<>(StaticUtils.computeMapCapacity(100)); 464 tlsCipherSuites = new HashMap<>(StaticUtils.computeMapCapacity(100)); 465 tlsProtocols = new HashMap<>(StaticUtils.computeMapCapacity(100)); 466 unindexedFilters = new HashMap<>(StaticUtils.computeMapCapacity(100)); 467 consecutiveFailedBindsByDN = 468 new HashMap<>(StaticUtils.computeMapCapacity(100)); 469 extendedOperationOIDsToNames = 470 new HashMap<>(StaticUtils.computeMapCapacity(100)); 471 processedRequests = new HashSet<>(StaticUtils.computeMapCapacity(100)); 472 addProcessingTimes = 473 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 474 bindProcessingTimes = 475 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 476 compareProcessingTimes = 477 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 478 deleteProcessingTimes = 479 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 480 extendedProcessingTimes = 481 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 482 modifyProcessingTimes = 483 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 484 modifyDNProcessingTimes = 485 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 486 searchProcessingTimes = 487 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 488 workQueueWaitTimes = 489 new LinkedHashMap<>(StaticUtils.computeMapCapacity(11)); 490 filtersRepresentingPotentialInjectionAttempt = 491 new LinkedHashSet<>(StaticUtils.computeMapCapacity(10)); 492 493 populateProcessingTimeMap(addProcessingTimes); 494 populateProcessingTimeMap(bindProcessingTimes); 495 populateProcessingTimeMap(compareProcessingTimes); 496 populateProcessingTimeMap(deleteProcessingTimes); 497 populateProcessingTimeMap(extendedProcessingTimes); 498 populateProcessingTimeMap(modifyProcessingTimes); 499 populateProcessingTimeMap(modifyDNProcessingTimes); 500 populateProcessingTimeMap(searchProcessingTimes); 501 populateProcessingTimeMap(workQueueWaitTimes); 502 } 503 504 505 506 /** 507 * Retrieves the name for this tool. 508 * 509 * @return The name for this tool. 510 */ 511 @Override() 512 @NotNull() 513 public String getToolName() 514 { 515 return "summarize-access-log"; 516 } 517 518 519 520 /** 521 * Retrieves the description for this tool. 522 * 523 * @return The description for this tool. 524 */ 525 @Override() 526 @NotNull() 527 public String getToolDescription() 528 { 529 return "Examine one or more access log files from Ping Identity, " + 530 "UnboundID, or Nokia/Alcatel-Lucent 8661 server products to display " + 531 "a number of metrics about operations processed within the server."; 532 } 533 534 535 536 /** 537 * Retrieves the version string for this tool. 538 * 539 * @return The version string for this tool. 540 */ 541 @Override() 542 @NotNull() 543 public String getToolVersion() 544 { 545 return Version.NUMERIC_VERSION_STRING; 546 } 547 548 549 550 /** 551 * Retrieves the minimum number of unnamed trailing arguments that are 552 * required. 553 * 554 * @return One, to indicate that at least one trailing argument (representing 555 * the path to an access log file) must be provided. 556 */ 557 @Override() 558 public int getMinTrailingArguments() 559 { 560 return 1; 561 } 562 563 564 565 /** 566 * Retrieves the maximum number of unnamed trailing arguments that may be 567 * provided for this tool. 568 * 569 * @return The maximum number of unnamed trailing arguments that may be 570 * provided for this tool. 571 */ 572 @Override() 573 public int getMaxTrailingArguments() 574 { 575 return -1; 576 } 577 578 579 580 /** 581 * Retrieves a placeholder string that should be used for trailing arguments 582 * in the usage information for this tool. 583 * 584 * @return A placeholder string that should be used for trailing arguments in 585 * the usage information for this tool. 586 */ 587 @Override() 588 @NotNull() 589 public String getTrailingArgumentsPlaceholder() 590 { 591 return "{path}"; 592 } 593 594 595 596 /** 597 * Indicates whether this tool should provide support for an interactive mode, 598 * in which the tool offers a mode in which the arguments can be provided in 599 * a text-driven menu rather than requiring them to be given on the command 600 * line. If interactive mode is supported, it may be invoked using the 601 * "--interactive" argument. Alternately, if interactive mode is supported 602 * and {@link #defaultsToInteractiveMode()} returns {@code true}, then 603 * interactive mode may be invoked by simply launching the tool without any 604 * arguments. 605 * 606 * @return {@code true} if this tool supports interactive mode, or 607 * {@code false} if not. 608 */ 609 @Override() 610 public boolean supportsInteractiveMode() 611 { 612 return true; 613 } 614 615 616 617 /** 618 * Indicates whether this tool defaults to launching in interactive mode if 619 * the tool is invoked without any command-line arguments. This will only be 620 * used if {@link #supportsInteractiveMode()} returns {@code true}. 621 * 622 * @return {@code true} if this tool defaults to using interactive mode if 623 * launched without any command-line arguments, or {@code false} if 624 * not. 625 */ 626 @Override() 627 public boolean defaultsToInteractiveMode() 628 { 629 return true; 630 } 631 632 633 634 /** 635 * Indicates whether this tool should provide arguments for redirecting output 636 * to a file. If this method returns {@code true}, then the tool will offer 637 * an "--outputFile" argument that will specify the path to a file to which 638 * all standard output and standard error content will be written, and it will 639 * also offer a "--teeToStandardOut" argument that can only be used if the 640 * "--outputFile" argument is present and will cause all output to be written 641 * to both the specified output file and to standard output. 642 * 643 * @return {@code true} if this tool should provide arguments for redirecting 644 * output to a file, or {@code false} if not. 645 */ 646 @Override() 647 protected boolean supportsOutputFile() 648 { 649 return true; 650 } 651 652 653 654 /** 655 * Indicates whether this tool supports the use of a properties file for 656 * specifying default values for arguments that aren't specified on the 657 * command line. 658 * 659 * @return {@code true} if this tool supports the use of a properties file 660 * for specifying default values for arguments that aren't specified 661 * on the command line, or {@code false} if not. 662 */ 663 @Override() 664 public boolean supportsPropertiesFile() 665 { 666 return true; 667 } 668 669 670 671 /** 672 * Adds the command-line arguments supported for use with this tool to the 673 * provided argument parser. The tool may need to retain references to the 674 * arguments (and/or the argument parser, if trailing arguments are allowed) 675 * to it in order to obtain their values for use in later processing. 676 * 677 * @param parser The argument parser to which the arguments are to be added. 678 * 679 * @throws ArgumentException If a problem occurs while adding any of the 680 * tool-specific arguments to the provided 681 * argument parser. 682 */ 683 @Override() 684 public void addToolArguments(@NotNull final ArgumentParser parser) 685 throws ArgumentException 686 { 687 // We need to save a reference to the argument parser so that we can get 688 // the trailing arguments later. 689 argumentParser = parser; 690 691 // Add an argument that makes it possible to read a JSON-formatted access 692 // log file. 693 String description = "Indicates that the log file contains " + 694 "JSON-formatted log messages rather than text-formatted messages."; 695 json = new BooleanArgument(null, "json", description); 696 parser.addArgument(json); 697 698 699 // Add an argument that makes it possible to read a compressed log file. 700 // Note that this argument is no longer needed for dealing with compressed 701 // files, since the tool will automatically detect whether a file is 702 // compressed. However, the argument is still provided for the purpose of 703 // backward compatibility. 704 description = "Indicates that the log file is compressed."; 705 isCompressed = new BooleanArgument('c', "isCompressed", description); 706 isCompressed.addLongIdentifier("is-compressed", true); 707 isCompressed.addLongIdentifier("compressed", true); 708 isCompressed.setHidden(true); 709 parser.addArgument(isCompressed); 710 711 712 // Add an argument that indicates that the tool should read the encryption 713 // passphrase from a file. 714 description = "Indicates that the log file is encrypted and that the " + 715 "encryption passphrase is contained in the specified file. If " + 716 "the log data is encrypted and this argument is not provided, then " + 717 "the tool will interactively prompt for the encryption passphrase."; 718 encryptionPassphraseFile = new FileArgument(null, 719 "encryptionPassphraseFile", false, 1, null, description, true, true, 720 true, false); 721 encryptionPassphraseFile.addLongIdentifier("encryption-passphrase-file", 722 true); 723 encryptionPassphraseFile.addLongIdentifier("encryptionPasswordFile", true); 724 encryptionPassphraseFile.addLongIdentifier("encryption-password-file", 725 true); 726 parser.addArgument(encryptionPassphraseFile); 727 728 729 // Add an argument that indicates the number of values to display for each 730 // item being summarized. 731 description = "The number of values to display for each item being " + 732 "summarized. A value of zero indicates that all items should be " + 733 "displayed. If this is not provided, a default value of 20 will " + 734 "be used."; 735 reportCount = new IntegerArgument(null, "reportCount", false, 0, null, 736 description, 0, Integer.MAX_VALUE, 20); 737 reportCount.addLongIdentifier("report-count", true); 738 reportCount.addLongIdentifier("maximumCount", true); 739 reportCount.addLongIdentifier("maximum-count", true); 740 reportCount.addLongIdentifier("maxCount", true); 741 reportCount.addLongIdentifier("max-count", true); 742 reportCount.addLongIdentifier("count", true); 743 parser.addArgument(reportCount); 744 745 746 // Add an argument that indicates that the output should not be anonymized. 747 description = "Do not anonymize the output, but include actual attribute " + 748 "values in filters and DNs. This will also have the effect of " + 749 "de-generifying those values, so output including the most common " + 750 "filters and DNs in some category will be specific instances of " + 751 "those filters and DNs instead of generic patterns."; 752 doNotAnonymize = new BooleanArgument(null, "doNotAnonymize", 1, 753 description); 754 doNotAnonymize.addLongIdentifier("do-not-anonymize", true); 755 doNotAnonymize.addLongIdentifier("deAnonymize", true); 756 doNotAnonymize.addLongIdentifier("de-anonymize", true); 757 parser.addArgument(doNotAnonymize); 758 } 759 760 761 762 /** 763 * Performs any necessary processing that should be done to ensure that the 764 * provided set of command-line arguments were valid. This method will be 765 * called after the basic argument parsing has been performed and immediately 766 * before the {@link #doToolProcessing} method is invoked. 767 * 768 * @throws ArgumentException If there was a problem with the command-line 769 * arguments provided to this program. 770 */ 771 @Override() 772 public void doExtendedArgumentValidation() 773 throws ArgumentException 774 { 775 // Make sure that at least one access log file path was provided. 776 final List<String> trailingArguments = 777 argumentParser.getTrailingArguments(); 778 if ((trailingArguments == null) || trailingArguments.isEmpty()) 779 { 780 throw new ArgumentException("No access log file paths were provided."); 781 } 782 } 783 784 785 786 /** 787 * Performs the core set of processing for this tool. 788 * 789 * @return A result code that indicates whether the processing completed 790 * successfully. 791 */ 792 @Override() 793 @NotNull() 794 public ResultCode doToolProcessing() 795 { 796 int displayCount = reportCount.getValue(); 797 if (displayCount <= 0) 798 { 799 displayCount = Integer.MAX_VALUE; 800 } 801 802 String encryptionPassphrase = null; 803 if (encryptionPassphraseFile.isPresent()) 804 { 805 try 806 { 807 encryptionPassphrase = ToolUtils.readEncryptionPassphraseFromFile( 808 encryptionPassphraseFile.getValue()); 809 } 810 catch (final LDAPException e) 811 { 812 Debug.debugException(e); 813 err(e.getMessage()); 814 return e.getResultCode(); 815 } 816 } 817 818 819 long logLines = 0L; 820 for (final String path : argumentParser.getTrailingArguments()) 821 { 822 final File f = new File(path); 823 out("Examining access log ", f.getAbsolutePath()); 824 AccessLogReader reader = null; 825 InputStream inputStream = null; 826 try 827 { 828 inputStream = new FileInputStream(f); 829 830 final ObjectPair<InputStream,String> p = 831 ToolUtils.getPossiblyPassphraseEncryptedInputStream(inputStream, 832 encryptionPassphrase, 833 (! encryptionPassphraseFile.isPresent()), 834 "Log file '" + path + "' is encrypted. Please enter the " + 835 "encryption passphrase:", 836 "ERROR: The provided passphrase was incorrect.", 837 getOut(), getErr()); 838 inputStream = p.getFirst(); 839 if ((p.getSecond() != null) && (encryptionPassphrase == null)) 840 { 841 encryptionPassphrase = p.getSecond(); 842 } 843 844 if (isCompressed.isPresent()) 845 { 846 inputStream = new GZIPInputStream(inputStream); 847 } 848 else 849 { 850 inputStream = 851 ToolUtils.getPossiblyGZIPCompressedInputStream(inputStream); 852 } 853 854 if (json.isPresent()) 855 { 856 reader = new JSONAccessLogReader(inputStream); 857 } 858 else 859 { 860 reader = new TextFormattedAccessLogReader(inputStream); 861 } 862 } 863 catch (final Exception e) 864 { 865 Debug.debugException(e); 866 err("Unable to open access log file ", f.getAbsolutePath(), ": ", 867 StaticUtils.getExceptionMessage(e)); 868 return ResultCode.LOCAL_ERROR; 869 } 870 finally 871 { 872 if ((reader == null) && (inputStream != null)) 873 { 874 try 875 { 876 inputStream.close(); 877 } 878 catch (final Exception e) 879 { 880 Debug.debugException(e); 881 } 882 } 883 } 884 885 long startTime = 0L; 886 long stopTime = 0L; 887 888 while (true) 889 { 890 final AccessLogMessage msg; 891 try 892 { 893 msg = reader.readMessage(); 894 } 895 catch (final IOException ioe) 896 { 897 Debug.debugException(ioe); 898 err("Error reading from access log file ", f.getAbsolutePath(), 899 ": ", StaticUtils.getExceptionMessage(ioe)); 900 901 if ((ioe.getCause() != null) && 902 (ioe.getCause() instanceof BadPaddingException)) 903 { 904 err("This error is likely because the log is encrypted and the " + 905 "server still has the log file open. It is recommended " + 906 "that you only try to examine encrypted logs after they " + 907 "have been rotated. You can use the rotate-log tool to " + 908 "force a rotation at any time. Attempting to proceed with " + 909 "just the data that was successfully read."); 910 break; 911 } 912 else 913 { 914 return ResultCode.LOCAL_ERROR; 915 } 916 } 917 catch (final LogException le) 918 { 919 Debug.debugException(le); 920 err("Encountered an error while attempting to parse a line in" + 921 "access log file ", f.getAbsolutePath(), ": ", 922 StaticUtils.getExceptionMessage(le)); 923 continue; 924 } 925 926 if (msg == null) 927 { 928 break; 929 } 930 931 logLines++; 932 stopTime = msg.getTimestamp().getTime(); 933 if (startTime == 0L) 934 { 935 startTime = stopTime; 936 } 937 938 switch (msg.getMessageType()) 939 { 940 case CONNECT: 941 processConnect((ConnectAccessLogMessage) msg); 942 break; 943 case SECURITY_NEGOTIATION: 944 processSecurityNegotiation( 945 (SecurityNegotiationAccessLogMessage) msg); 946 break; 947 case DISCONNECT: 948 processDisconnect((DisconnectAccessLogMessage) msg); 949 break; 950 case REQUEST: 951 switch (((OperationRequestAccessLogMessage) msg).getOperationType()) 952 { 953 case ABANDON: 954 processAbandonRequest((AbandonRequestAccessLogMessage) msg); 955 break; 956 case EXTENDED: 957 processExtendedRequest((ExtendedRequestAccessLogMessage) msg); 958 break; 959 case SEARCH: 960 processSearchRequest((SearchRequestAccessLogMessage) msg); 961 break; 962 case UNBIND: 963 processUnbindRequest((UnbindRequestAccessLogMessage) msg); 964 break; 965 } 966 break; 967 case RESULT: 968 switch (((OperationRequestAccessLogMessage) msg).getOperationType()) 969 { 970 case ADD: 971 processAddResult((AddResultAccessLogMessage) msg); 972 break; 973 case BIND: 974 processBindResult((BindResultAccessLogMessage) msg); 975 break; 976 case COMPARE: 977 processCompareResult((CompareResultAccessLogMessage) msg); 978 break; 979 case DELETE: 980 processDeleteResult((DeleteResultAccessLogMessage) msg); 981 break; 982 case EXTENDED: 983 processExtendedResult((ExtendedResultAccessLogMessage) msg); 984 break; 985 case MODIFY: 986 processModifyResult((ModifyResultAccessLogMessage) msg); 987 break; 988 case MODDN: 989 processModifyDNResult((ModifyDNResultAccessLogMessage) msg); 990 break; 991 case SEARCH: 992 processSearchResult((SearchResultAccessLogMessage) msg); 993 break; 994 } 995 break; 996 997 case ASSURANCE_COMPLETE: 998 case CLIENT_CERTIFICATE: 999 case ENTRY_REBALANCING_REQUEST: 1000 case ENTRY_REBALANCING_RESULT: 1001 case FORWARD: 1002 case FORWARD_FAILED: 1003 case ENTRY: 1004 case REFERENCE: 1005 default: 1006 // Nothing needs to be done for these message types. 1007 } 1008 } 1009 1010 try 1011 { 1012 reader.close(); 1013 } 1014 catch (final Exception e) 1015 { 1016 Debug.debugException(e); 1017 } 1018 logDurationMillis += (stopTime - startTime); 1019 1020 1021 // If there are any outstanding authentication failures, then update the 1022 // set of consecutive failures as appropriate. 1023 for (final Map.Entry<String,AtomicLong> e : 1024 outstandingFailedBindDNs.entrySet()) 1025 { 1026 final String dn = e.getKey(); 1027 final AtomicLong outstandingFailureCount = e.getValue(); 1028 final AtomicLong consecutiveFailures = 1029 consecutiveFailedBindsByDN.get(dn); 1030 if ((consecutiveFailures == null) || 1031 (outstandingFailureCount.get() > consecutiveFailures.get())) 1032 { 1033 consecutiveFailedBindsByDN.put(dn, outstandingFailureCount); 1034 } 1035 } 1036 outstandingFailedBindDNs.clear(); 1037 } 1038 1039 1040 final int numFiles = argumentParser.getTrailingArguments().size(); 1041 out(); 1042 out("Examined ", logLines, " lines in ", numFiles, 1043 ((numFiles == 1) ? " file" : " files"), 1044 " covering a total duration of ", 1045 StaticUtils.millisToHumanReadableDuration(logDurationMillis)); 1046 if (logLines == 0) 1047 { 1048 return ResultCode.SUCCESS; 1049 } 1050 1051 out(); 1052 1053 final double logDurationSeconds = logDurationMillis / 1_000.0; 1054 final double connectsPerSecond = numConnects / logDurationSeconds; 1055 final double disconnectsPerSecond = numDisconnects / logDurationSeconds; 1056 1057 out("Total connections established: ", numConnects, " (", 1058 decimalFormat.format(connectsPerSecond), "/second)"); 1059 out("Total disconnects: ", numDisconnects, " (", 1060 decimalFormat.format(disconnectsPerSecond), "/second)"); 1061 1062 printCounts(clientAddresses, "Most common client addresses:", "address", 1063 "addresses"); 1064 1065 printCounts(clientConnectionPolicies, 1066 "Most common client connection policies:", "policy", "policies"); 1067 1068 printCounts(tlsProtocols, "Most common TLS protocol versions:", "version", 1069 "versions"); 1070 1071 printCounts(tlsCipherSuites, "Most common TLS cipher suites:", 1072 "cipher suite", "cipher suites"); 1073 1074 printCounts(disconnectReasons, "Most common disconnect reasons:", "reason", 1075 "reasons"); 1076 1077 final long totalOps = numAbandons + numAdds + numBinds + numCompares + 1078 numDeletes + numExtended + numModifies + numModifyDNs + numSearches + 1079 numUnbinds; 1080 final long totalResults = totalOps - numAbandons - numUnbinds; 1081 1082 if (totalOps > 0) 1083 { 1084 final double percentAbandon = 100.0 * numAbandons / totalOps; 1085 final double percentAdd = 100.0 * numAdds / totalOps; 1086 final double percentBind = 100.0 * numBinds / totalOps; 1087 final double percentCompare = 100.0 * numCompares / totalOps; 1088 final double percentDelete = 100.0 * numDeletes / totalOps; 1089 final double percentExtended = 100.0 * numExtended / totalOps; 1090 final double percentModify = 100.0 * numModifies / totalOps; 1091 final double percentModifyDN = 100.0 * numModifyDNs / totalOps; 1092 final double percentSearch = 100.0 * numSearches / totalOps; 1093 final double percentUnbind = 100.0 * numUnbinds / totalOps; 1094 1095 final double abandonsPerSecond = numAbandons / logDurationSeconds; 1096 final double addsPerSecond = numAdds / logDurationSeconds; 1097 final double bindsPerSecond = numBinds / logDurationSeconds; 1098 final double comparesPerSecond = numCompares / logDurationSeconds; 1099 final double deletesPerSecond = numDeletes / logDurationSeconds; 1100 final double extendedPerSecond = numExtended / logDurationSeconds; 1101 final double modifiesPerSecond = numModifies / logDurationSeconds; 1102 final double modifyDNsPerSecond = numModifyDNs / logDurationSeconds; 1103 final double searchesPerSecond = numSearches / logDurationSeconds; 1104 final double unbindsPerSecond = numUnbinds / logDurationSeconds; 1105 1106 out(); 1107 out("Total operations examined: ", totalOps); 1108 out("Abandon operations examined: ", numAbandons, " (", 1109 decimalFormat.format(percentAbandon), "%, ", 1110 decimalFormat.format(abandonsPerSecond), "/second)"); 1111 out("Add operations examined: ", numAdds, " (", 1112 decimalFormat.format(percentAdd), "%, ", 1113 decimalFormat.format(addsPerSecond), "/second)"); 1114 out("Bind operations examined: ", numBinds, " (", 1115 decimalFormat.format(percentBind), "%, ", 1116 decimalFormat.format(bindsPerSecond), "/second)"); 1117 out("Compare operations examined: ", numCompares, " (", 1118 decimalFormat.format(percentCompare), "%, ", 1119 decimalFormat.format(comparesPerSecond), "/second)"); 1120 out("Delete operations examined: ", numDeletes, " (", 1121 decimalFormat.format(percentDelete), "%, ", 1122 decimalFormat.format(deletesPerSecond), "/second)"); 1123 out("Extended operations examined: ", numExtended, " (", 1124 decimalFormat.format(percentExtended), "%, ", 1125 decimalFormat.format(extendedPerSecond), "/second)"); 1126 out("Modify operations examined: ", numModifies, " (", 1127 decimalFormat.format(percentModify), "%, ", 1128 decimalFormat.format(modifiesPerSecond), "/second)"); 1129 out("Modify DN operations examined: ", numModifyDNs, " (", 1130 decimalFormat.format(percentModifyDN), "%, ", 1131 decimalFormat.format(modifyDNsPerSecond), "/second)"); 1132 out("Search operations examined: ", numSearches, " (", 1133 decimalFormat.format(percentSearch), "%, ", 1134 decimalFormat.format(searchesPerSecond), "/second)"); 1135 out("Unbind operations examined: ", numUnbinds, " (", 1136 decimalFormat.format(percentUnbind), "%, ", 1137 decimalFormat.format(unbindsPerSecond), "/second)"); 1138 1139 final double totalProcessingDuration = addProcessingDuration + 1140 bindProcessingDuration + compareProcessingDuration + 1141 deleteProcessingDuration + extendedProcessingDuration + 1142 modifyProcessingDuration + modifyDNProcessingDuration + 1143 searchProcessingDuration; 1144 1145 out(); 1146 out("Average operation processing duration: ", 1147 decimalFormat.format(totalProcessingDuration / totalOps), "ms"); 1148 1149 if (numAdds > 0) 1150 { 1151 out("Average add operation processing duration: ", 1152 decimalFormat.format(addProcessingDuration / numAdds), "ms"); 1153 } 1154 1155 if (numBinds > 0) 1156 { 1157 out("Average bind operation processing duration: ", 1158 decimalFormat.format(bindProcessingDuration / numBinds), "ms"); 1159 } 1160 1161 if (numCompares > 0) 1162 { 1163 out("Average compare operation processing duration: ", 1164 decimalFormat.format(compareProcessingDuration / numCompares), 1165 "ms"); 1166 } 1167 1168 if (numDeletes > 0) 1169 { 1170 out("Average delete operation processing duration: ", 1171 decimalFormat.format(deleteProcessingDuration / numDeletes), "ms"); 1172 } 1173 1174 if (numExtended > 0) 1175 { 1176 out("Average extended operation processing duration: ", 1177 decimalFormat.format(extendedProcessingDuration / numExtended), 1178 "ms"); 1179 } 1180 1181 if (numModifies > 0) 1182 { 1183 out("Average modify operation processing duration: ", 1184 decimalFormat.format(modifyProcessingDuration / numModifies), "ms"); 1185 } 1186 1187 if (numModifyDNs > 0) 1188 { 1189 out("Average modify DN operation processing duration: ", 1190 decimalFormat.format(modifyDNProcessingDuration / numModifyDNs), 1191 "ms"); 1192 } 1193 1194 if (numSearches > 0) 1195 { 1196 out("Average search operation processing duration: ", 1197 decimalFormat.format(searchProcessingDuration / numSearches), "ms"); 1198 } 1199 1200 printProcessingTimeHistogram("add", numAdds, addProcessingTimes); 1201 printProcessingTimeHistogram("bind", numBinds, bindProcessingTimes); 1202 printProcessingTimeHistogram("compare", numCompares, 1203 compareProcessingTimes); 1204 printProcessingTimeHistogram("delete", numDeletes, deleteProcessingTimes); 1205 printProcessingTimeHistogram("extended", numExtended, 1206 extendedProcessingTimes); 1207 printProcessingTimeHistogram("modify", numModifies, 1208 modifyProcessingTimes); 1209 printProcessingTimeHistogram("modify DN", numModifyDNs, 1210 modifyDNProcessingTimes); 1211 printProcessingTimeHistogram("search", numSearches, 1212 searchProcessingTimes); 1213 1214 if (totalWorkQueueWaitTime > 0L) 1215 { 1216 out(); 1217 out("Average work queue wait time: ", 1218 decimalFormat.format(totalWorkQueueWaitTime / totalResults), "ms"); 1219 printHistogram("Count of operations by work queue wait time:", 1220 totalResults, workQueueWaitTimes); 1221 } 1222 1223 printResultCodeCounts(addResultCodes, "add"); 1224 printResultCodeCounts(bindResultCodes, "bind"); 1225 printResultCodeCounts(compareResultCodes, "compare"); 1226 printResultCodeCounts(deleteResultCodes, "delete"); 1227 printResultCodeCounts(extendedResultCodes, "extended"); 1228 printResultCodeCounts(modifyResultCodes, "modify"); 1229 printResultCodeCounts(modifyDNResultCodes, "modify DN"); 1230 printResultCodeCounts(searchResultCodes, "search"); 1231 1232 printCounts(preAuthzPrivilegesUsed, 1233 "Most common pre-authorization privileges used:", "privilege", 1234 "privileges"); 1235 printCounts(privilegesUsed, "Most common privileges used:", "privilege", 1236 "privileges"); 1237 printCounts(privilegesMissing, "Most common missing privileges:", 1238 "privilege", "privileges"); 1239 1240 printCounts(successfulBindDNs, 1241 "Most common bind DNs used in successful authentication attempts:", 1242 "DN", "DNs"); 1243 printCounts(bindFailuresByDN, 1244 "Most common bind DNs used in failed authentication attempts:", 1245 "DN", "DNs"); 1246 printCounts(bindFailuresByIPAddress, 1247 "Most common IP addresses used in failed authentication attempts:", 1248 "IP", "IPs"); 1249 if (doNotAnonymize.isPresent()) 1250 { 1251 printCounts(consecutiveFailedBindsByDN, 1252 "Bind DNs with the most consecutive authentication failures:", 1253 "DN", "DNs"); 1254 } 1255 printCounts(authenticationTypes, "Most common authentication types:", 1256 "authentication type", "authentication types"); 1257 1258 long numResultsWithAuthzID = 0L; 1259 for (final AtomicLong l : authzDNs.values()) 1260 { 1261 numResultsWithAuthzID += l.get(); 1262 } 1263 1264 out(); 1265 final double percentWithAuthzID = 1266 100.0 * numResultsWithAuthzID / totalOps; 1267 out("Number of operations with an alternate authorization identity: ", 1268 numResultsWithAuthzID, " (", 1269 decimalFormat.format(percentWithAuthzID), "%)"); 1270 1271 printCounts(authzDNs, "Most common alternate authorization identity DNs:", 1272 "DN", "DNs"); 1273 1274 if (! requestControlOIDs.isEmpty()) 1275 { 1276 final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>(); 1277 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1278 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1279 getMostCommonElements(requestControlOIDs, controlCounts, displayCount, 1280 skippedWithSameCount, skippedWithLowerCount); 1281 1282 out(); 1283 out("Most common request control types:"); 1284 1285 long count = -1L; 1286 for (final ObjectPair<String,Long> p : controlCounts) 1287 { 1288 count = p.getSecond(); 1289 final double percent = 100.0 * count / numRequestControls; 1290 1291 final String oid = p.getFirst(); 1292 final OIDRegistryItem item = OIDRegistry.getDefault().get(oid); 1293 if (item == null) 1294 { 1295 out(p.getFirst(), ": ", p.getSecond(), " (", 1296 decimalFormat.format(percent), "%)"); 1297 } 1298 else 1299 { 1300 out(p.getFirst(), " (", item.getName(), "): ", p.getSecond(), " (", 1301 decimalFormat.format(percent), "%)"); 1302 } 1303 } 1304 1305 if (skippedWithSameCount.get() > 0L) 1306 { 1307 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1308 getSingularOrPlural(skippedWithSameCount.get(), "control", 1309 "controls") + 1310 " with a count of " + count + " }"); 1311 } 1312 1313 if (skippedWithLowerCount.get() > 0L) 1314 { 1315 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1316 getSingularOrPlural(skippedWithLowerCount.get(), "control", 1317 "controls") + 1318 " with a count that is less than " + count + " }"); 1319 } 1320 } 1321 1322 if (! responseControlOIDs.isEmpty()) 1323 { 1324 final List<ObjectPair<String,Long>> controlCounts = new ArrayList<>(); 1325 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1326 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1327 getMostCommonElements(responseControlOIDs, controlCounts, displayCount, 1328 skippedWithSameCount, skippedWithLowerCount); 1329 1330 out(); 1331 out("Most common response control types:"); 1332 1333 long count = -1L; 1334 for (final ObjectPair<String,Long> p : controlCounts) 1335 { 1336 count = p.getSecond(); 1337 final double percent = 100.0 * count / numResponseControls; 1338 1339 final String oid = p.getFirst(); 1340 final OIDRegistryItem item = OIDRegistry.getDefault().get(oid); 1341 if (item == null) 1342 { 1343 out(p.getFirst(), ": ", p.getSecond(), " (", 1344 decimalFormat.format(percent), "%)"); 1345 } 1346 else 1347 { 1348 out(p.getFirst(), " (", item.getName(), "): ", p.getSecond(), " (", 1349 decimalFormat.format(percent), "%)"); 1350 } 1351 } 1352 1353 if (skippedWithSameCount.get() > 0L) 1354 { 1355 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1356 getSingularOrPlural(skippedWithSameCount.get(), "control", 1357 "controls") + 1358 " with a count of " + count + " }"); 1359 } 1360 1361 if (skippedWithLowerCount.get() > 0L) 1362 { 1363 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1364 getSingularOrPlural(skippedWithLowerCount.get(), "control", 1365 "controls") + 1366 " with a count that is less than " + count + " }"); 1367 } 1368 } 1369 1370 if (! extendedOperations.isEmpty()) 1371 { 1372 final List<ObjectPair<String,Long>> extOpCounts = new ArrayList<>(); 1373 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1374 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1375 getMostCommonElements(extendedOperations, extOpCounts, displayCount, 1376 skippedWithSameCount, skippedWithLowerCount); 1377 1378 out(); 1379 out("Most common extended operation types:"); 1380 1381 long count = -1L; 1382 for (final ObjectPair<String,Long> p : extOpCounts) 1383 { 1384 count = p.getSecond(); 1385 final double percent = 100.0 * count / numExtended; 1386 1387 final String oid = p.getFirst(); 1388 final String name = extendedOperationOIDsToNames.get(oid); 1389 if (name == null) 1390 { 1391 out(p.getFirst(), ": ", p.getSecond(), " (", 1392 decimalFormat.format(percent), "%)"); 1393 } 1394 else 1395 { 1396 out(p.getFirst(), " (", name, "): ", p.getSecond(), " (", 1397 decimalFormat.format(percent), "%)"); 1398 } 1399 } 1400 1401 if (skippedWithSameCount.get() > 0L) 1402 { 1403 out("{ Skipped " + skippedWithSameCount.get() + 1404 " additional extended " + 1405 getSingularOrPlural(skippedWithSameCount.get(), "operation", 1406 "operations") + 1407 " with a count of " + count + " }"); 1408 } 1409 1410 if (skippedWithLowerCount.get() > 0L) 1411 { 1412 out("{ Skipped " + skippedWithLowerCount.get() + 1413 " additional extended " + 1414 getSingularOrPlural(skippedWithLowerCount.get(), "operation", 1415 "operations") + 1416 " with a count that is less than " + count + " }"); 1417 } 1418 } 1419 1420 out(); 1421 out("Number of unindexed search attempts: ", numUnindexedAttempts); 1422 out("Number of successfully-completed unindexed searches: ", 1423 numUnindexedSuccessful); 1424 out("Number of failed unindexed searches: ", numUnindexedFailed); 1425 1426 printCounts(unindexedFilters, "Most common unindexed search filters:", 1427 "filter", "filters"); 1428 1429 if (! searchScopes.isEmpty()) 1430 { 1431 final List<ObjectPair<SearchScope,Long>> scopeCounts = 1432 new ArrayList<>(); 1433 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1434 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1435 getMostCommonElements(searchScopes, scopeCounts, displayCount, 1436 skippedWithSameCount, skippedWithLowerCount); 1437 1438 out(); 1439 out("Most common search scopes:"); 1440 1441 long count = -1L; 1442 for (final ObjectPair<SearchScope,Long> p : scopeCounts) 1443 { 1444 count = p.getSecond(); 1445 final double percent = 100.0 * count / numSearches; 1446 out(p.getFirst().getName().toLowerCase(), " (", 1447 p.getFirst().intValue(), "): ", p.getSecond(), " (", 1448 decimalFormat.format(percent), "%)"); 1449 } 1450 1451 if (skippedWithSameCount.get() > 0L) 1452 { 1453 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1454 getSingularOrPlural(skippedWithSameCount.get(), "scope", 1455 "scopes") + 1456 " with a count of " + count + " }"); 1457 } 1458 1459 if (skippedWithLowerCount.get() > 0L) 1460 { 1461 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1462 getSingularOrPlural(skippedWithLowerCount.get(), "scope", 1463 "scopes") + 1464 " with a count that is less than " + count + " }"); 1465 } 1466 } 1467 1468 if (! searchEntryCounts.isEmpty()) 1469 { 1470 final List<ObjectPair<Long,Long>> entryCounts = new ArrayList<>(); 1471 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1472 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1473 getMostCommonElements(searchEntryCounts, entryCounts, displayCount, 1474 skippedWithSameCount, skippedWithLowerCount); 1475 1476 out(); 1477 out("Most common search entry counts:"); 1478 1479 long count = -1L; 1480 for (final ObjectPair<Long,Long> p : entryCounts) 1481 { 1482 count = p.getSecond(); 1483 final double percent = 100.0 * count / numSearches; 1484 out(p.getFirst(), " matching ", 1485 getSingularOrPlural(p.getFirst(), "entry", "entries"), 1486 ": ", p.getSecond(), " (", decimalFormat.format(percent), "%)"); 1487 } 1488 1489 if (skippedWithSameCount.get() > 0L) 1490 { 1491 out("{ Skipped " + skippedWithSameCount.get() + " additional entry " + 1492 getSingularOrPlural(skippedWithSameCount.get(), "count", 1493 "counts") + 1494 " with a count of " + count + " }"); 1495 } 1496 1497 if (skippedWithLowerCount.get() > 0L) 1498 { 1499 out("{ Skipped " + skippedWithLowerCount.get() + 1500 " additional entry " + 1501 getSingularOrPlural(skippedWithLowerCount.get(), "count", 1502 "counts") + 1503 " with a count that is less than " + count + " }"); 1504 } 1505 } 1506 1507 printCounts(searchBaseDNs, 1508 "Most common base DNs for searches with a non-base scope:", 1509 "base DN", "base DNs"); 1510 1511 printCounts(filterTypes, 1512 "Most common filters for searches with a non-base scope:", 1513 "filter", "filters"); 1514 1515 printCounts(filterComponentCounts, 1516 "Most common search filter component counts:", "filter", 1517 "filters"); 1518 1519 if (doNotAnonymize.isPresent() && 1520 (! filtersRepresentingPotentialInjectionAttempt.isEmpty())) 1521 { 1522 out(); 1523 wrapOut(0, WRAP_COLUMN, 1524 "Search filters that may indicate an unsuccessful injection " + 1525 "attempt. These include filters with an assertion value " + 1526 "that contains one or more of the following: parentheses, " + 1527 "ampersands, pipes, single quotes, double quotes, or the " + 1528 "words 'select' and 'from':"); 1529 for (final Filter f : filtersRepresentingPotentialInjectionAttempt) 1530 { 1531 out("* " + f.toString()); 1532 } 1533 } 1534 1535 if (numSearches > 0L) 1536 { 1537 long numSearchesMatchingNoEntries = 0L; 1538 for (final AtomicLong l : noEntryFilters.values()) 1539 { 1540 numSearchesMatchingNoEntries += l.get(); 1541 } 1542 1543 out(); 1544 final double noEntryPercent = 1545 100.0 * numSearchesMatchingNoEntries / numSearches; 1546 out("Number of searches matching no entries: ", 1547 numSearchesMatchingNoEntries, " (", 1548 decimalFormat.format(noEntryPercent), "%)"); 1549 1550 printCounts(noEntryFilters, 1551 "Most common filters for searches matching no entries:", 1552 "filter", "filters"); 1553 1554 1555 long numSearchesMatchingOneEntry = 0L; 1556 for (final AtomicLong l : oneEntryFilters.values()) 1557 { 1558 numSearchesMatchingOneEntry += l.get(); 1559 } 1560 1561 out(); 1562 final double oneEntryPercent = 1563 100.0 * numSearchesMatchingOneEntry / numSearches; 1564 out("Number of searches matching one entry: ", 1565 numSearchesMatchingOneEntry, " (", 1566 decimalFormat.format(oneEntryPercent), "%)"); 1567 1568 printCounts(oneEntryFilters, 1569 "Most common filters for searches matching one entry:", 1570 "filter", "filters"); 1571 1572 1573 long numSearchesMatchingMultipleEntries = 0L; 1574 for (final AtomicLong l : multiEntryFilters.values()) 1575 { 1576 numSearchesMatchingMultipleEntries += l.get(); 1577 } 1578 1579 out(); 1580 final double multiEntryPercent = 1581 100.0 * numSearchesMatchingMultipleEntries / numSearches; 1582 out("Number of searches matching multiple entries: ", 1583 numSearchesMatchingMultipleEntries, " (", 1584 decimalFormat.format(multiEntryPercent), "%)"); 1585 1586 printCounts(multiEntryFilters, 1587 "Most common filters for searches matching multiple entries:", 1588 "filter", "filters"); 1589 } 1590 } 1591 1592 if (! mostExpensiveFilters.isEmpty()) 1593 { 1594 final List<ObjectPair<String,Long>> filterDurations = new ArrayList<>(); 1595 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 1596 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 1597 getMostCommonElements(mostExpensiveFilters, filterDurations, 1598 displayCount, skippedWithSameCount, skippedWithLowerCount); 1599 1600 out(); 1601 out("Filters for searches with the longest processing times:"); 1602 1603 String durationStr = ""; 1604 for (final ObjectPair<String,Long> p : filterDurations) 1605 { 1606 final long durationMicros = p.getSecond(); 1607 final double durationMillis = durationMicros / 1_000.0; 1608 durationStr = decimalFormat.format(durationMillis) + " ms"; 1609 out(p.getFirst(), ": ", durationStr); 1610 } 1611 1612 if (skippedWithSameCount.get() > 0L) 1613 { 1614 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 1615 getSingularOrPlural(skippedWithSameCount.get(), "filter", 1616 "filters") + 1617 " with a duration of " + durationStr + " }"); 1618 } 1619 1620 if (skippedWithLowerCount.get() > 0L) 1621 { 1622 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 1623 getSingularOrPlural(skippedWithLowerCount.get(), "filter", 1624 "filters") + 1625 " with a duration that is less than " + durationStr + " }"); 1626 } 1627 } 1628 1629 final long totalUncached = numUncachedAdds + numUncachedBinds + 1630 numUncachedCompares + numUncachedDeletes + numUncachedExtended + 1631 numUncachedModifies + numUncachedModifyDNs + numUncachedSearches; 1632 if (totalUncached > 0L) 1633 { 1634 out(); 1635 out("Operations accessing uncached data:"); 1636 printUncached("Add", numUncachedAdds, numAdds); 1637 printUncached("Bind", numUncachedBinds, numBinds); 1638 printUncached("Compare", numUncachedCompares, numCompares); 1639 printUncached("Delete", numUncachedDeletes, numDeletes); 1640 printUncached("Extended", numUncachedExtended, numExtended); 1641 printUncached("Modify", numUncachedModifies, numModifies); 1642 printUncached("Modify DN", numUncachedModifyDNs, numModifyDNs); 1643 printUncached("Search", numUncachedSearches, numSearches); 1644 } 1645 1646 1647 return ResultCode.SUCCESS; 1648 } 1649 1650 1651 1652 /** 1653 * Retrieves a set of information that may be used to generate example usage 1654 * information. Each element in the returned map should consist of a map 1655 * between an example set of arguments and a string that describes the 1656 * behavior of the tool when invoked with that set of arguments. 1657 * 1658 * @return A set of information that may be used to generate example usage 1659 * information. It may be {@code null} or empty if no example usage 1660 * information is available. 1661 */ 1662 @Override() 1663 @NotNull() 1664 public LinkedHashMap<String[],String> getExampleUsages() 1665 { 1666 final LinkedHashMap<String[],String> examples = 1667 new LinkedHashMap<>(StaticUtils.computeMapCapacity(1)); 1668 1669 final String[] args = 1670 { 1671 "/ds/logs/access" 1672 }; 1673 final String description = 1674 "Analyze the contents of the /ds/logs/access access log file."; 1675 examples.put(args, description); 1676 1677 return examples; 1678 } 1679 1680 1681 1682 /** 1683 * Populates the provided processing time map with an initial set of values. 1684 * 1685 * @param m The processing time map to be populated. 1686 */ 1687 private static void populateProcessingTimeMap( 1688 @NotNull final HashMap<Long,AtomicLong> m) 1689 { 1690 m.put(1L, new AtomicLong(0L)); 1691 m.put(2L, new AtomicLong(0L)); 1692 m.put(3L, new AtomicLong(0L)); 1693 m.put(5L, new AtomicLong(0L)); 1694 m.put(10L, new AtomicLong(0L)); 1695 m.put(20L, new AtomicLong(0L)); 1696 m.put(30L, new AtomicLong(0L)); 1697 m.put(50L, new AtomicLong(0L)); 1698 m.put(100L, new AtomicLong(0L)); 1699 m.put(1_000L, new AtomicLong(0L)); 1700 m.put(2_000L, new AtomicLong(0L)); 1701 m.put(3_000L, new AtomicLong(0L)); 1702 m.put(5_000L, new AtomicLong(0L)); 1703 m.put(10_000L, new AtomicLong(0L)); 1704 m.put(20_000L, new AtomicLong(0L)); 1705 m.put(30_000L, new AtomicLong(0L)); 1706 m.put(60_000L, new AtomicLong(0L)); 1707 m.put(Long.MAX_VALUE, new AtomicLong(0L)); 1708 } 1709 1710 1711 1712 /** 1713 * Performs any necessary processing for a connect message. 1714 * 1715 * @param m The log message to be processed. 1716 */ 1717 private void processConnect(@NotNull final ConnectAccessLogMessage m) 1718 { 1719 numConnects++; 1720 1721 final String clientAddr = m.getSourceAddress(); 1722 if (clientAddr != null) 1723 { 1724 final Long connectionID = m.getConnectionID(); 1725 if (connectionID != null) 1726 { 1727 ipAddressesByConnectionID.put(connectionID, clientAddr); 1728 } 1729 1730 AtomicLong count = clientAddresses.get(clientAddr); 1731 if (count == null) 1732 { 1733 count = new AtomicLong(0L); 1734 clientAddresses.put(clientAddr, count); 1735 } 1736 count.incrementAndGet(); 1737 } 1738 1739 final String ccp = m.getClientConnectionPolicy(); 1740 if (ccp != null) 1741 { 1742 AtomicLong l = clientConnectionPolicies.get(ccp); 1743 if (l == null) 1744 { 1745 l = new AtomicLong(0L); 1746 clientConnectionPolicies.put(ccp, l); 1747 } 1748 l.incrementAndGet(); 1749 } 1750 } 1751 1752 1753 1754 /** 1755 * Performs any necessary processing for a security negotiation message. 1756 * 1757 * @param m The log message to be processed. 1758 */ 1759 private void processSecurityNegotiation( 1760 @NotNull final SecurityNegotiationAccessLogMessage m) 1761 { 1762 final String protocol = m.getProtocol(); 1763 if (protocol != null) 1764 { 1765 AtomicLong l = tlsProtocols.get(protocol); 1766 if (l == null) 1767 { 1768 l = new AtomicLong(0L); 1769 tlsProtocols.put(protocol, l); 1770 } 1771 l.incrementAndGet(); 1772 } 1773 1774 final String cipherSuite = m.getCipher(); 1775 if (cipherSuite != null) 1776 { 1777 AtomicLong l = tlsCipherSuites.get(cipherSuite); 1778 if (l == null) 1779 { 1780 l = new AtomicLong(0L); 1781 tlsCipherSuites.put(cipherSuite, l); 1782 } 1783 l.incrementAndGet(); 1784 } 1785 } 1786 1787 1788 1789 /** 1790 * Performs any necessary processing for a disconnect message. 1791 * 1792 * @param m The log message to be processed. 1793 */ 1794 private void processDisconnect(@NotNull final DisconnectAccessLogMessage m) 1795 { 1796 numDisconnects++; 1797 1798 final Long connectionID = m.getConnectionID(); 1799 if (connectionID != null) 1800 { 1801 ipAddressesByConnectionID.remove(connectionID); 1802 } 1803 1804 final String reason = m.getDisconnectReason(); 1805 if (reason != null) 1806 { 1807 AtomicLong l = disconnectReasons.get(reason); 1808 if (l == null) 1809 { 1810 l = new AtomicLong(0L); 1811 disconnectReasons.put(reason, l); 1812 } 1813 l.incrementAndGet(); 1814 } 1815 } 1816 1817 1818 1819 /** 1820 * Performs any necessary processing for an abandon request message. 1821 * 1822 * @param m The log message to be processed. 1823 */ 1824 private void processAbandonRequest( 1825 @NotNull final AbandonRequestAccessLogMessage m) 1826 { 1827 numAbandons++; 1828 } 1829 1830 1831 1832 /** 1833 * Performs any necessary processing for an extended request message. 1834 * 1835 * @param m The log message to be processed. 1836 */ 1837 private void processExtendedRequest( 1838 @NotNull final ExtendedRequestAccessLogMessage m) 1839 { 1840 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1841 processExtendedRequestInternal(m); 1842 } 1843 1844 1845 1846 /** 1847 * Performs the internal processing for an extended request message. 1848 * 1849 * @param m The log message to be processed. 1850 */ 1851 private void processExtendedRequestInternal( 1852 @NotNull final ExtendedRequestAccessLogMessage m) 1853 { 1854 final String oid = m.getRequestOID(); 1855 if (oid != null) 1856 { 1857 AtomicLong l = extendedOperations.get(oid); 1858 if (l == null) 1859 { 1860 l = new AtomicLong(0L); 1861 extendedOperations.put(oid, l); 1862 } 1863 l.incrementAndGet(); 1864 1865 final String requestType = m.getRequestType(); 1866 if ((requestType != null) && 1867 (! extendedOperationOIDsToNames.containsKey(oid))) 1868 { 1869 extendedOperationOIDsToNames.put(oid, requestType); 1870 } 1871 } 1872 } 1873 1874 1875 1876 /** 1877 * Performs any necessary processing for a search request message. 1878 * 1879 * @param m The log message to be processed. 1880 */ 1881 private void processSearchRequest( 1882 @NotNull final SearchRequestAccessLogMessage m) 1883 { 1884 processedRequests.add(m.getConnectionID() + "-" + m.getOperationID()); 1885 processSearchRequestInternal(m); 1886 } 1887 1888 1889 1890 /** 1891 * Performs any necessary processing for a search request message. 1892 * 1893 * @param m The log message to be processed. 1894 */ 1895 private void processSearchRequestInternal( 1896 @NotNull final SearchRequestAccessLogMessage m) 1897 { 1898 final SearchScope scope = m.getScope(); 1899 if (scope != null) 1900 { 1901 AtomicLong scopeCount = searchScopes.get(scope); 1902 if (scopeCount == null) 1903 { 1904 scopeCount = new AtomicLong(0L); 1905 searchScopes.put(scope, scopeCount); 1906 } 1907 scopeCount.incrementAndGet(); 1908 1909 if (! scope.equals(SearchScope.BASE)) 1910 { 1911 final String filterString = prepareFilter(m.getFilter()); 1912 if (filterString != null) 1913 { 1914 AtomicLong filterCount = filterTypes.get(filterString); 1915 if (filterCount == null) 1916 { 1917 filterCount = new AtomicLong(0L); 1918 filterTypes.put(filterString, filterCount); 1919 } 1920 filterCount.incrementAndGet(); 1921 1922 1923 final String baseDN = getDNString(m.getBaseDN()); 1924 if (baseDN != null) 1925 { 1926 AtomicLong baseDNCount = searchBaseDNs.get(baseDN); 1927 if (baseDNCount == null) 1928 { 1929 baseDNCount = new AtomicLong(0L); 1930 searchBaseDNs.put(baseDN, baseDNCount); 1931 } 1932 baseDNCount.incrementAndGet(); 1933 } 1934 } 1935 } 1936 } 1937 1938 final String filterString = m.getFilter(); 1939 if (filterString != null) 1940 { 1941 try 1942 { 1943 final Filter filter = Filter.create(filterString); 1944 if (mayRepresentInjectionAttempt(filter)) 1945 { 1946 filtersRepresentingPotentialInjectionAttempt.add(filter); 1947 } 1948 1949 1950 final int numComponents = countComponents(filter); 1951 final String label; 1952 if (numComponents == 1) 1953 { 1954 label = "1 component"; 1955 } 1956 else 1957 { 1958 label = numComponents + " components"; 1959 } 1960 1961 AtomicLong count = filterComponentCounts.get(label); 1962 if (count == null) 1963 { 1964 count = new AtomicLong(0L); 1965 filterComponentCounts.put(label, count); 1966 } 1967 1968 count.incrementAndGet(); 1969 } 1970 catch (final Exception e) 1971 { 1972 Debug.debugException(e); 1973 } 1974 } 1975 } 1976 1977 1978 1979 /** 1980 * Indicates whether the provided search filter may represent an injection 1981 * attempt. Filters that may represent injection attempts include: 1982 * <UL> 1983 * <LI>Filters with assertion values that contain parentheses, ampersands, 1984 * pipes, or single or double quotes.</LI> 1985 * <LI>Filters that contain the words "select" and "from".</LI> 1986 * </UL> 1987 * 1988 * @param filter The filter to examine. It must not be {@code null}. 1989 * 1990 * @return {@code true} if the provided filter may represent an injection 1991 * attempt, or {@code false} if not. 1992 */ 1993 static boolean mayRepresentInjectionAttempt(@NotNull final Filter filter) 1994 { 1995 switch (filter.getFilterType()) 1996 { 1997 case Filter.FILTER_TYPE_AND: 1998 case Filter.FILTER_TYPE_OR: 1999 for (final Filter f : filter.getComponents()) 2000 { 2001 if (mayRepresentInjectionAttempt(f)) 2002 { 2003 return true; 2004 } 2005 } 2006 return false; 2007 2008 case Filter.FILTER_TYPE_NOT: 2009 return mayRepresentInjectionAttempt(filter.getNOTComponent()); 2010 2011 case Filter.FILTER_TYPE_EQUALITY: 2012 case Filter.FILTER_TYPE_GREATER_OR_EQUAL: 2013 case Filter.FILTER_TYPE_LESS_OR_EQUAL: 2014 case Filter.FILTER_TYPE_APPROXIMATE_MATCH: 2015 case Filter.FILTER_TYPE_EXTENSIBLE_MATCH: 2016 return mayRepresentInjectionAttempt(filter.getAssertionValue()); 2017 2018 case Filter.FILTER_TYPE_SUBSTRING: 2019 final String[] subAnyStrings = filter.getSubAnyStrings(); 2020 if (subAnyStrings != null) 2021 { 2022 for (final String subAnyString : subAnyStrings) 2023 { 2024 if (mayRepresentInjectionAttempt(subAnyString)) 2025 { 2026 return true; 2027 } 2028 } 2029 } 2030 2031 return mayRepresentInjectionAttempt(filter.getSubInitialString()) || 2032 mayRepresentInjectionAttempt(filter.getSubFinalString()); 2033 2034 case Filter.FILTER_TYPE_PRESENCE: 2035 default: 2036 return false; 2037 } 2038 } 2039 2040 2041 2042 /** 2043 * Indicates whether the provided string (which should be a filter assertion 2044 * value or substring component) may represent an injection attempt. 2045 * 2046 * @param value The value for which to make the determination. It may 2047 * optionally be {@code null}. 2048 * 2049 * @return {@code true} if the provided value may represent an injection 2050 * attempt, or {@code false} if not. 2051 */ 2052 private static boolean mayRepresentInjectionAttempt( 2053 @Nullable final String value) 2054 { 2055 if (value == null) 2056 { 2057 return false; 2058 } 2059 2060 final String lowerValue = StaticUtils.toLowerCase(value); 2061 return (lowerValue.contains("(") || 2062 lowerValue.contains(")") || 2063 lowerValue.contains("&") || 2064 lowerValue.contains("|") || 2065 lowerValue.contains("\"") || 2066 lowerValue.contains("'") || 2067 ((lowerValue.contains("select") && lowerValue.contains("from")))); 2068 } 2069 2070 2071 2072 /** 2073 * Counts the number of components in the specified filter. Presence, 2074 * equality, substring, greater-or-equal, less-or-equal, approximate-match, 2075 * and extensible-match filters will all be considered a single component. 2076 * AND and OR filters will be one plus the aggregate component count for each 2077 * of the components they contain. NOT filters will be one plus the component 2078 * count for the filter it contains. 2079 * 2080 * @param filter The filter for which to count the number of components. It 2081 * must not be {@code null}. 2082 * 2083 * @return The number of components in the specified filter. 2084 */ 2085 static int countComponents(@NotNull final Filter filter) 2086 { 2087 switch (filter.getFilterType()) 2088 { 2089 case Filter.FILTER_TYPE_AND: 2090 case Filter.FILTER_TYPE_OR: 2091 int count = 1; 2092 for (final Filter f : filter.getComponents()) 2093 { 2094 count += countComponents(f); 2095 } 2096 return count; 2097 2098 case Filter.FILTER_TYPE_NOT: 2099 return 1 + countComponents(filter.getNOTComponent()); 2100 2101 case Filter.FILTER_TYPE_PRESENCE: 2102 case Filter.FILTER_TYPE_EQUALITY: 2103 case Filter.FILTER_TYPE_SUBSTRING: 2104 case Filter.FILTER_TYPE_GREATER_OR_EQUAL: 2105 case Filter.FILTER_TYPE_LESS_OR_EQUAL: 2106 case Filter.FILTER_TYPE_APPROXIMATE_MATCH: 2107 case Filter.FILTER_TYPE_EXTENSIBLE_MATCH: 2108 default: 2109 return 1; 2110 } 2111 } 2112 2113 2114 2115 /** 2116 * Performs any necessary processing for an unbind request message. 2117 * 2118 * @param m The log message to be processed. 2119 */ 2120 private void processUnbindRequest( 2121 @NotNull final UnbindRequestAccessLogMessage m) 2122 { 2123 numUnbinds++; 2124 } 2125 2126 2127 2128 /** 2129 * Performs any necessary processing for an add result message. 2130 * 2131 * @param m The log message to be processed. 2132 */ 2133 private void processAddResult(@NotNull final AddResultAccessLogMessage m) 2134 { 2135 numAdds++; 2136 2137 updateCommonResult(m); 2138 2139 updateResultCodeCount(m.getResultCode(), addResultCodes); 2140 addProcessingDuration += 2141 doubleValue(m.getProcessingTimeMillis(), addProcessingTimes); 2142 2143 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2144 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2145 { 2146 numUncachedAdds++; 2147 } 2148 2149 updateAuthzCount(m.getAlternateAuthorizationDN()); 2150 } 2151 2152 2153 2154 /** 2155 * Performs any necessary processing for a bind result message. 2156 * 2157 * @param m The log message to be processed. 2158 */ 2159 private void processBindResult(@NotNull final BindResultAccessLogMessage m) 2160 { 2161 numBinds++; 2162 2163 updateCommonResult(m); 2164 2165 if (m.getAuthenticationType() != null) 2166 { 2167 final String authType; 2168 switch (m.getAuthenticationType()) 2169 { 2170 case SIMPLE: 2171 authType = "Simple"; 2172 break; 2173 2174 case SASL: 2175 final String saslMechanism = m.getSASLMechanismName(); 2176 if (saslMechanism == null) 2177 { 2178 authType = "SASL {unknown mechanism}"; 2179 } 2180 else 2181 { 2182 authType = "SASL " + saslMechanism; 2183 } 2184 break; 2185 2186 case INTERNAL: 2187 authType = "Internal"; 2188 break; 2189 2190 default: 2191 authType = m.getAuthenticationType().name(); 2192 break; 2193 } 2194 2195 AtomicLong l = authenticationTypes.get(authType); 2196 if (l == null) 2197 { 2198 l = new AtomicLong(0L); 2199 authenticationTypes.put(authType, l); 2200 } 2201 l.incrementAndGet(); 2202 } 2203 2204 updateResultCodeCount(m.getResultCode(), bindResultCodes); 2205 bindProcessingDuration += 2206 doubleValue(m.getProcessingTimeMillis(), bindProcessingTimes); 2207 2208 String authenticationDN = getDNString(m.getAuthenticationDN()); 2209 if (m.getResultCode() == ResultCode.SUCCESS) 2210 { 2211 if (authenticationDN != null) 2212 { 2213 AtomicLong l = successfulBindDNs.get(authenticationDN); 2214 if (l == null) 2215 { 2216 l = new AtomicLong(0L); 2217 successfulBindDNs.put(authenticationDN, l); 2218 } 2219 l.incrementAndGet(); 2220 2221 final AtomicLong outstandingFailures = 2222 outstandingFailedBindDNs.remove(authenticationDN); 2223 if (outstandingFailures != null) 2224 { 2225 final AtomicLong consecutiveFailures = 2226 consecutiveFailedBindsByDN.get(authenticationDN); 2227 if ((consecutiveFailures == null) || 2228 (outstandingFailures.get() > consecutiveFailures.get())) 2229 { 2230 consecutiveFailedBindsByDN.put(authenticationDN, 2231 new AtomicLong(outstandingFailures.get())); 2232 } 2233 } 2234 } 2235 2236 final String ccp = m.getClientConnectionPolicy(); 2237 if (ccp != null) 2238 { 2239 AtomicLong l = clientConnectionPolicies.get(ccp); 2240 if (l == null) 2241 { 2242 l = new AtomicLong(0L); 2243 clientConnectionPolicies.put(ccp, l); 2244 } 2245 l.incrementAndGet(); 2246 } 2247 } 2248 else if ((m.getResultCode() != ResultCode.SASL_BIND_IN_PROGRESS) && 2249 (m.getResultCode() != ResultCode.REFERRAL)) 2250 { 2251 if (authenticationDN == null) 2252 { 2253 authenticationDN = getDNString(m.getDN()); 2254 } 2255 2256 if (authenticationDN != null) 2257 { 2258 AtomicLong l = bindFailuresByDN.get(authenticationDN); 2259 if (l == null) 2260 { 2261 l = new AtomicLong(0L); 2262 bindFailuresByDN.put(authenticationDN, l); 2263 } 2264 l.incrementAndGet(); 2265 2266 l = outstandingFailedBindDNs.get(authenticationDN); 2267 if (l == null) 2268 { 2269 l = new AtomicLong(0L); 2270 outstandingFailedBindDNs.put(authenticationDN, l); 2271 } 2272 l.incrementAndGet(); 2273 } 2274 2275 String ipAddress = m.getRequesterIPAddress(); 2276 if (ipAddress == null) 2277 { 2278 final Long connectionID = m.getConnectionID(); 2279 if (connectionID != null) 2280 { 2281 ipAddress = ipAddressesByConnectionID.get(connectionID); 2282 } 2283 } 2284 2285 if (ipAddress != null) 2286 { 2287 AtomicLong l = bindFailuresByIPAddress.get(ipAddress); 2288 if (l == null) 2289 { 2290 l = new AtomicLong(0L); 2291 bindFailuresByIPAddress.put(ipAddress, l); 2292 } 2293 l.incrementAndGet(); 2294 } 2295 } 2296 2297 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2298 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2299 { 2300 numUncachedBinds++; 2301 } 2302 2303 updateAuthzCount(m.getAuthorizationDN()); 2304 } 2305 2306 2307 2308 /** 2309 * Performs any necessary processing for a compare result message. 2310 * 2311 * @param m The log message to be processed. 2312 */ 2313 private void processCompareResult( 2314 @NotNull final CompareResultAccessLogMessage m) 2315 { 2316 numCompares++; 2317 2318 updateCommonResult(m); 2319 2320 updateResultCodeCount(m.getResultCode(), compareResultCodes); 2321 compareProcessingDuration += 2322 doubleValue(m.getProcessingTimeMillis(), compareProcessingTimes); 2323 2324 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2325 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2326 { 2327 numUncachedCompares++; 2328 } 2329 2330 updateAuthzCount(m.getAlternateAuthorizationDN()); 2331 } 2332 2333 2334 2335 /** 2336 * Performs any necessary processing for a delete result message. 2337 * 2338 * @param m The log message to be processed. 2339 */ 2340 private void processDeleteResult( 2341 @NotNull final DeleteResultAccessLogMessage m) 2342 { 2343 numDeletes++; 2344 2345 updateCommonResult(m); 2346 2347 updateResultCodeCount(m.getResultCode(), deleteResultCodes); 2348 deleteProcessingDuration += 2349 doubleValue(m.getProcessingTimeMillis(), deleteProcessingTimes); 2350 2351 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2352 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2353 { 2354 numUncachedDeletes++; 2355 } 2356 2357 updateAuthzCount(m.getAlternateAuthorizationDN()); 2358 } 2359 2360 2361 2362 /** 2363 * Performs any necessary processing for an extended result message. 2364 * 2365 * @param m The log message to be processed. 2366 */ 2367 private void processExtendedResult( 2368 @NotNull final ExtendedResultAccessLogMessage m) 2369 { 2370 numExtended++; 2371 2372 updateCommonResult(m); 2373 2374 final String id = m.getConnectionID() + "-" + m.getOperationID(); 2375 if (!processedRequests.remove(id)) 2376 { 2377 processExtendedRequestInternal(m); 2378 } 2379 2380 updateResultCodeCount(m.getResultCode(), extendedResultCodes); 2381 extendedProcessingDuration += 2382 doubleValue(m.getProcessingTimeMillis(), extendedProcessingTimes); 2383 2384 final String ccp = m.getClientConnectionPolicy(); 2385 if (ccp != null) 2386 { 2387 AtomicLong l = clientConnectionPolicies.get(ccp); 2388 if (l == null) 2389 { 2390 l = new AtomicLong(0L); 2391 clientConnectionPolicies.put(ccp, l); 2392 } 2393 l.incrementAndGet(); 2394 } 2395 2396 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2397 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2398 { 2399 numUncachedExtended++; 2400 } 2401 } 2402 2403 2404 2405 /** 2406 * Performs any necessary processing for a modify result message. 2407 * 2408 * @param m The log message to be processed. 2409 */ 2410 private void processModifyResult( 2411 @NotNull final ModifyResultAccessLogMessage m) 2412 { 2413 numModifies++; 2414 2415 updateCommonResult(m); 2416 2417 updateResultCodeCount(m.getResultCode(), modifyResultCodes); 2418 modifyProcessingDuration += 2419 doubleValue(m.getProcessingTimeMillis(), modifyProcessingTimes); 2420 2421 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2422 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2423 { 2424 numUncachedModifies++; 2425 } 2426 2427 updateAuthzCount(m.getAlternateAuthorizationDN()); 2428 } 2429 2430 2431 2432 /** 2433 * Performs any necessary processing for a modify DN result message. 2434 * 2435 * @param m The log message to be processed. 2436 */ 2437 private void processModifyDNResult( 2438 @NotNull final ModifyDNResultAccessLogMessage m) 2439 { 2440 numModifyDNs++; 2441 2442 updateCommonResult(m); 2443 2444 updateResultCodeCount(m.getResultCode(), modifyDNResultCodes); 2445 modifyDNProcessingDuration += 2446 doubleValue(m.getProcessingTimeMillis(), modifyDNProcessingTimes); 2447 2448 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2449 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2450 { 2451 numUncachedModifyDNs++; 2452 } 2453 2454 updateAuthzCount(m.getAlternateAuthorizationDN()); 2455 } 2456 2457 2458 2459 /** 2460 * Performs any necessary processing for a search result message. 2461 * 2462 * @param m The log message to be processed. 2463 */ 2464 private void processSearchResult( 2465 @NotNull final SearchResultAccessLogMessage m) 2466 { 2467 numSearches++; 2468 2469 updateCommonResult(m); 2470 2471 final String id = m.getConnectionID() + "-" + m.getOperationID(); 2472 if (! processedRequests.remove(id)) 2473 { 2474 processSearchRequestInternal(m); 2475 } 2476 2477 final ResultCode resultCode = m.getResultCode(); 2478 updateResultCodeCount(resultCode, searchResultCodes); 2479 searchProcessingDuration += 2480 doubleValue(m.getProcessingTimeMillis(), searchProcessingTimes); 2481 2482 final String filterString = prepareFilter(m.getFilter()); 2483 2484 final Long entryCount = m.getEntriesReturned(); 2485 if (entryCount != null) 2486 { 2487 AtomicLong l = searchEntryCounts.get(entryCount); 2488 if (l == null) 2489 { 2490 l = new AtomicLong(0L); 2491 searchEntryCounts.put(entryCount, l); 2492 } 2493 l.incrementAndGet(); 2494 2495 final Map<String,AtomicLong> filterCountMap; 2496 switch (entryCount.intValue()) 2497 { 2498 case 0: 2499 filterCountMap = noEntryFilters; 2500 break; 2501 case 1: 2502 filterCountMap = oneEntryFilters; 2503 break; 2504 default: 2505 filterCountMap = multiEntryFilters; 2506 break; 2507 } 2508 2509 if (filterString != null) 2510 { 2511 AtomicLong filterCount = filterCountMap.get(filterString); 2512 if (filterCount == null) 2513 { 2514 filterCount = new AtomicLong(0L); 2515 filterCountMap.put(filterString, filterCount); 2516 } 2517 filterCount.incrementAndGet(); 2518 } 2519 } 2520 2521 final Boolean isUnindexed = m.getUnindexed(); 2522 if ((isUnindexed != null) && isUnindexed) 2523 { 2524 numUnindexedAttempts++; 2525 if (resultCode == ResultCode.SUCCESS) 2526 { 2527 numUnindexedSuccessful++; 2528 } 2529 else 2530 { 2531 numUnindexedFailed++; 2532 } 2533 2534 if (filterString != null) 2535 { 2536 AtomicLong l = unindexedFilters.get(filterString); 2537 if (l == null) 2538 { 2539 l = new AtomicLong(0L); 2540 unindexedFilters.put(filterString, l); 2541 } 2542 l.incrementAndGet(); 2543 } 2544 } 2545 2546 final Boolean uncachedDataAccessed = m.getUncachedDataAccessed(); 2547 if ((uncachedDataAccessed != null) && uncachedDataAccessed) 2548 { 2549 numUncachedSearches++; 2550 } 2551 2552 updateAuthzCount(m.getAlternateAuthorizationDN()); 2553 2554 final Double processingTimeMillis = m.getProcessingTimeMillis(); 2555 if ((processingTimeMillis != null) && (filterString != null)) 2556 { 2557 final long processingTimeMicros = 2558 Math.round(processingTimeMillis * 1_000.0); 2559 2560 AtomicLong l = mostExpensiveFilters.get(filterString); 2561 if (l == null) 2562 { 2563 l = new AtomicLong(processingTimeMicros); 2564 mostExpensiveFilters.put(filterString, l); 2565 } 2566 else 2567 { 2568 final long previousProcessingTimeMicros = l.get(); 2569 if (processingTimeMicros > previousProcessingTimeMicros) 2570 { 2571 l.set(processingTimeMicros); 2572 } 2573 } 2574 } 2575 } 2576 2577 2578 2579 /** 2580 * Updates a number of statistics that are common to all types of result log 2581 * messages. 2582 * 2583 * @param m The result log message to examine. 2584 */ 2585 private void updateCommonResult( 2586 @NotNull final OperationResultAccessLogMessage m) 2587 { 2588 // Handle the work queue wait time. 2589 totalWorkQueueWaitTime += 2590 doubleValue(m.getWorkQueueWaitTimeMillis(), workQueueWaitTimes); 2591 2592 2593 // Handle request and response control OIDs. 2594 for (final String oid : m.getRequestControlOIDs()) 2595 { 2596 numRequestControls++; 2597 updateCount(requestControlOIDs, oid); 2598 } 2599 2600 for (final String oid : m.getResponseControlOIDs()) 2601 { 2602 numResponseControls++; 2603 updateCount(responseControlOIDs, oid); 2604 } 2605 2606 2607 // Handle used and missing privileges. 2608 for (final String privilegeName : m.getPreAuthorizationUsedPrivileges()) 2609 { 2610 updateCount(preAuthzPrivilegesUsed, privilegeName); 2611 } 2612 2613 for (final String privilegeName : m.getUsedPrivileges()) 2614 { 2615 updateCount(privilegesUsed, privilegeName); 2616 } 2617 2618 for (final String privilegeName : m.getMissingPrivileges()) 2619 { 2620 updateCount(privilegesMissing, privilegeName); 2621 } 2622 } 2623 2624 2625 2626 /** 2627 * Updates the counter for the given key in the provided map. If the key does 2628 * not exist, it will be added to the map. 2629 * 2630 * @param m The map to be updated. 2631 * @param key The key for which to update the count. 2632 */ 2633 private static void updateCount(@NotNull final Map<String,AtomicLong> m, 2634 @NotNull final String key) 2635 { 2636 AtomicLong count = m.get(key); 2637 if (count == null) 2638 { 2639 count = new AtomicLong(0L); 2640 m.put(key, count); 2641 } 2642 2643 count.incrementAndGet(); 2644 } 2645 2646 2647 2648 /** 2649 * Updates the count for the provided result code in the given map. 2650 * 2651 * @param rc The result code for which to update the count. 2652 * @param m The map used to hold counts by result code. 2653 */ 2654 private static void updateResultCodeCount(@Nullable final ResultCode rc, 2655 @NotNull final HashMap<ResultCode,AtomicLong> m) 2656 { 2657 if (rc == null) 2658 { 2659 return; 2660 } 2661 2662 AtomicLong l = m.get(rc); 2663 if (l == null) 2664 { 2665 l = new AtomicLong(0L); 2666 m.put(rc, l); 2667 } 2668 l.incrementAndGet(); 2669 } 2670 2671 2672 2673 /** 2674 * Retrieves the double value for the provided {@code Double} object. 2675 * 2676 * @param d The {@code Double} object for which to retrieve the value. 2677 * @param m The processing time histogram map to be updated. 2678 * 2679 * @return The double value of the provided {@code Double} object if it was 2680 * non-{@code null}, or 0.0 if it was {@code null}. 2681 */ 2682 private static double doubleValue(@Nullable final Double d, 2683 @NotNull final HashMap<Long,AtomicLong> m) 2684 { 2685 if (d == null) 2686 { 2687 return 0.0; 2688 } 2689 else 2690 { 2691 for (final Map.Entry<Long,AtomicLong> e : m.entrySet()) 2692 { 2693 if (d <= e.getKey()) 2694 { 2695 e.getValue().incrementAndGet(); 2696 break; 2697 } 2698 } 2699 2700 return d; 2701 } 2702 } 2703 2704 2705 2706 /** 2707 * Updates the provided list with the most frequently-occurring elements in 2708 * the provided map, paired with the number of times each value occurred. 2709 * 2710 * @param <K> The type of object used as the key for the 2711 * provided map. 2712 * @param countMap The map to be examined. It is expected that 2713 * the values of the map will be the count of 2714 * occurrences for the keys. 2715 * @param mostCommonElementList The list to which the values will be 2716 * updated. It must not be {@code null}, must 2717 * be empty, and must be updatable. 2718 * @param maxListSize The maximum number of items to add to the 2719 * provided list. It must be greater than 2720 * zero. 2721 * @param skippedWithSameCount A counter that will be incremented for each 2722 * map entry that is skipped with the same 2723 * count as a value that was not skipped. It 2724 * must not be {@code null} and must initially 2725 * be zero. 2726 * @param skippedWithLowerCount A counter that will be incremented for each 2727 * map entry that is skipped with a lower count 2728 * as the last value that was not skipped. It 2729 * must not be {@code null} and must initially 2730 * be zero. 2731 * 2732 * @return A list of the most frequently-occurring elements in the provided 2733 * map. 2734 */ 2735 @NotNull() 2736 private static <K> List<ObjectPair<K,Long>> getMostCommonElements( 2737 @NotNull final Map<K,AtomicLong> countMap, 2738 @NotNull final List<ObjectPair<K,Long>> mostCommonElementList, 2739 final int maxListSize, 2740 @NotNull final AtomicLong skippedWithSameCount, 2741 @NotNull final AtomicLong skippedWithLowerCount) 2742 { 2743 final TreeMap<Long,List<K>> reverseMap = 2744 new TreeMap<>(new ReverseComparator<Long>()); 2745 for (final Map.Entry<K,AtomicLong> e : countMap.entrySet()) 2746 { 2747 final Long count = e.getValue().get(); 2748 List<K> list = reverseMap.get(count); 2749 if (list == null) 2750 { 2751 list = new ArrayList<>(); 2752 reverseMap.put(count, list); 2753 } 2754 list.add(e.getKey()); 2755 } 2756 2757 for (final Map.Entry<Long,List<K>> e : reverseMap.entrySet()) 2758 { 2759 final Long l = e.getKey(); 2760 int numNotSkipped = 0; 2761 for (final K k : e.getValue()) 2762 { 2763 if (mostCommonElementList.size() >= maxListSize) 2764 { 2765 if (numNotSkipped > 0) 2766 { 2767 skippedWithSameCount.incrementAndGet(); 2768 } 2769 else 2770 { 2771 skippedWithLowerCount.incrementAndGet(); 2772 } 2773 } 2774 else 2775 { 2776 numNotSkipped++; 2777 mostCommonElementList.add(new ObjectPair<>(k, l)); 2778 } 2779 } 2780 } 2781 2782 return mostCommonElementList; 2783 } 2784 2785 2786 2787 /** 2788 * Updates the count of alternate authorization identities for the provided 2789 * DN. 2790 * 2791 * @param authzDN The DN of the alternate authorization identity that was 2792 * used. It may be {@code null} if no alternate 2793 * authorization identity was used. 2794 */ 2795 private void updateAuthzCount(@Nullable final String authzDN) 2796 { 2797 if (authzDN == null) 2798 { 2799 return; 2800 } 2801 2802 final String dnString = getDNString(authzDN); 2803 2804 AtomicLong l = authzDNs.get(dnString); 2805 if (l == null) 2806 { 2807 l = new AtomicLong(0L); 2808 authzDNs.put(dnString, l); 2809 } 2810 } 2811 2812 2813 2814 /** 2815 * Retrieves a string representation of the provided DN. It may either be 2816 * anonymized, using question marks in place of specific attribute values, or 2817 * it may be the actual string representation of the given DN. 2818 * 2819 * @param dn The DN for which to retrieve the string representation. 2820 * 2821 * @return A string representation of the provided DN, or {@code null} if the 2822 * given DN was {@code null}. 2823 */ 2824 @Nullable() 2825 private String getDNString(@Nullable final String dn) 2826 { 2827 if (dn == null) 2828 { 2829 return null; 2830 } 2831 2832 final DN parsedDN; 2833 try 2834 { 2835 parsedDN = new DN(dn); 2836 } 2837 catch (final Exception e) 2838 { 2839 Debug.debugException(e); 2840 return dn.toLowerCase(); 2841 } 2842 2843 if (parsedDN.isNullDN()) 2844 { 2845 return "{Null DN}"; 2846 } 2847 2848 if (doNotAnonymize.isPresent()) 2849 { 2850 return parsedDN.toNormalizedString(); 2851 } 2852 2853 final StringBuilder buffer = new StringBuilder(); 2854 final RDN[] rdns = parsedDN.getRDNs(); 2855 for (int i=0; i < rdns.length; i++) 2856 { 2857 if (i > 0) 2858 { 2859 buffer.append(','); 2860 } 2861 2862 final RDN rdn = rdns[i]; 2863 final String[] attributeNames = rdn.getAttributeNames(); 2864 for (int j=0; j < attributeNames.length; j++) 2865 { 2866 if (j > 0) 2867 { 2868 buffer.append('+'); 2869 } 2870 buffer.append(attributeNames[j].toLowerCase()); 2871 buffer.append("=?"); 2872 } 2873 } 2874 2875 return buffer.toString(); 2876 } 2877 2878 2879 2880 /** 2881 * Retrieves a prepared string representation of the provided search filter. 2882 * It may potentially be de-anonymized to include specific values. 2883 * 2884 * @param filterString The string representation of the filter to prepare. 2885 * It may be {@code null} if the log message does not 2886 * have a filter. 2887 * 2888 * @return A string representation of the provided filter (which may or may 2889 * not be anonymized), or {@code null} if the provided filter is 2890 * {@code null} or cannot be prepared. 2891 */ 2892 @Nullable() 2893 private String prepareFilter(@Nullable final String filterString) 2894 { 2895 if (filterString == null) 2896 { 2897 return null; 2898 } 2899 2900 if (doNotAnonymize.isPresent()) 2901 { 2902 return filterString.toLowerCase(); 2903 } 2904 2905 try 2906 { 2907 return new GenericFilter(Filter.create(filterString)).toString(). 2908 toLowerCase(); 2909 } 2910 catch (final Exception e) 2911 { 2912 Debug.debugException(e); 2913 return null; 2914 } 2915 } 2916 2917 2918 2919 /** 2920 * Writes a breakdown of the processing times for a specified type of 2921 * operation. 2922 * 2923 * @param t The name of the operation type. 2924 * @param n The total number of operations of the specified type that were 2925 * processed by the server. 2926 * @param m The map of operation counts by processing time bucket. 2927 */ 2928 private void printProcessingTimeHistogram(@NotNull final String t, 2929 final long n, 2930 @NotNull final LinkedHashMap<Long,AtomicLong> m) 2931 { 2932 printHistogram("Count of " + t + " operations by processing time:", n, m); 2933 } 2934 2935 2936 2937 /** 2938 * Writes a breakdown of the processing times for a specified type of 2939 * operation. 2940 * 2941 * @param h The header to display at the beginning of the histogram. 2942 * @param n The total number of operations that were processed by the 2943 * server. 2944 * @param m The map of operation counts by processing time bucket. 2945 */ 2946 private void printHistogram(@NotNull final String h, 2947 final long n, 2948 @NotNull final LinkedHashMap<Long,AtomicLong> m) 2949 { 2950 if (n <= 0) 2951 { 2952 return; 2953 } 2954 2955 out(); 2956 out(h); 2957 2958 long lowerBound = 0; 2959 long accumulatedCount = 0; 2960 final Iterator<Map.Entry<Long,AtomicLong>> i = m.entrySet().iterator(); 2961 while (i.hasNext()) 2962 { 2963 final Map.Entry<Long,AtomicLong> e = i.next(); 2964 final long upperBound = e.getKey(); 2965 final long count = e.getValue().get(); 2966 final double categoryPercent = 100.0 * count / n; 2967 2968 accumulatedCount += count; 2969 final double accumulatedPercent = 100.0 * accumulatedCount / n; 2970 2971 if (i.hasNext()) 2972 { 2973 final String lowerBoundString; 2974 if (lowerBound == 0L) 2975 { 2976 lowerBoundString = "0 milliseconds"; 2977 } 2978 else 2979 { 2980 final long lowerBoundNanos = lowerBound * 1_000_000L; 2981 lowerBoundString = DurationArgument.nanosToDuration(lowerBoundNanos); 2982 } 2983 2984 final long upperBoundNanos = upperBound * 1_000_000L; 2985 final String upperBoundString = 2986 DurationArgument.nanosToDuration(upperBoundNanos); 2987 2988 2989 out("Between ", lowerBoundString, " and ", upperBoundString, ": ", 2990 count, " (", decimalFormat.format(categoryPercent), "%, ", 2991 decimalFormat.format(accumulatedPercent), "% accumulated)"); 2992 lowerBound = upperBound; 2993 } 2994 else 2995 { 2996 final long lowerBoundNanos = lowerBound * 1_000_000L; 2997 final String lowerBoundString = 2998 DurationArgument.nanosToDuration(lowerBoundNanos); 2999 3000 out("Greater than ", lowerBoundString, ": ", count, " (", 3001 decimalFormat.format(categoryPercent), "%, ", 3002 decimalFormat.format(accumulatedPercent), "% accumulated)"); 3003 } 3004 } 3005 } 3006 3007 3008 3009 /** 3010 * Optionally prints information about the number and percent of operations of 3011 * the specified type that involved access to uncached data. 3012 * 3013 * @param operationType The type of operation. 3014 * @param numUncached The number of operations of the specified type that 3015 * involved access to uncached data. 3016 * @param numTotal The total number of operations of the specified 3017 * type. 3018 */ 3019 private void printUncached(@NotNull final String operationType, 3020 final long numUncached, 3021 final long numTotal) 3022 { 3023 if (numUncached == 0) 3024 { 3025 return; 3026 } 3027 3028 out(operationType, ": ", numUncached, " (", 3029 decimalFormat.format(100.0 * numUncached / numTotal), "%)"); 3030 } 3031 3032 3033 3034 /** 3035 * Prints data from the provided map of counts. 3036 * 3037 * @param countMap The map containing the data to print. 3038 * @param heading The heading to display before printing the contents 3039 * of the map. 3040 * @param singularItem The name to use for a single item represented by the 3041 * key of the given map. 3042 * @param pluralItem The name to use for zero or multiple items 3043 * represented by the key of the given map. 3044 */ 3045 private void printCounts(@Nullable final Map<String,AtomicLong> countMap, 3046 @NotNull final String heading, 3047 @NotNull final String singularItem, 3048 @NotNull final String pluralItem) 3049 { 3050 if ((countMap == null) || countMap.isEmpty()) 3051 { 3052 return; 3053 } 3054 3055 long totalCount = 0L; 3056 for (final AtomicLong l : countMap.values()) 3057 { 3058 totalCount += l.get(); 3059 } 3060 3061 out(); 3062 out(heading); 3063 3064 int displayCount = reportCount.getValue(); 3065 if (displayCount <= 0L) 3066 { 3067 displayCount = Integer.MAX_VALUE; 3068 } 3069 3070 final List<ObjectPair<String,Long>> countList = new ArrayList<>(); 3071 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 3072 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 3073 getMostCommonElements(countMap, countList, displayCount, 3074 skippedWithSameCount, skippedWithLowerCount); 3075 3076 long count = -1L; 3077 for (final ObjectPair<String,Long> p : countList) 3078 { 3079 count = p.getSecond(); 3080 3081 if (totalCount > 0L) 3082 { 3083 final double percent = 100.0 * count / totalCount; 3084 out(p.getFirst(), ": ", count, " (", decimalFormat.format(percent), 3085 ")"); 3086 } 3087 else 3088 { 3089 out(p.getFirst(), ": ", count); 3090 } 3091 } 3092 3093 if (skippedWithSameCount.get() > 0L) 3094 { 3095 out("{ Skipped " + skippedWithSameCount.get() + " additional " + 3096 getSingularOrPlural(skippedWithSameCount.get(), singularItem, 3097 pluralItem) + 3098 " with a count of " + count + " }"); 3099 } 3100 3101 if (skippedWithLowerCount.get() > 0L) 3102 { 3103 out("{ Skipped " + skippedWithLowerCount.get() + " additional " + 3104 getSingularOrPlural(skippedWithLowerCount.get(), singularItem, 3105 pluralItem) + 3106 " with a count that is less than " + count + " }"); 3107 } 3108 } 3109 3110 3111 3112 /** 3113 * Prints data from the provided map of counts. 3114 * 3115 * @param countMap The map containing the data to print. 3116 * @param operationType The type of operation represented by the keys of 3117 * the map. 3118 */ 3119 private void printResultCodeCounts( 3120 @Nullable final Map<ResultCode,AtomicLong> countMap, 3121 @NotNull final String operationType) 3122 { 3123 if ((countMap == null) || countMap.isEmpty()) 3124 { 3125 return; 3126 } 3127 3128 long totalCount = 0L; 3129 for (final AtomicLong l : countMap.values()) 3130 { 3131 totalCount += l.get(); 3132 } 3133 3134 out(); 3135 out("Most common " + operationType + " operation result codes:"); 3136 3137 int displayCount = reportCount.getValue(); 3138 if (displayCount <= 0L) 3139 { 3140 displayCount = Integer.MAX_VALUE; 3141 } 3142 3143 final List<ObjectPair<ResultCode,Long>> resultCodeList = new ArrayList<>(); 3144 final AtomicLong skippedWithSameCount = new AtomicLong(0L); 3145 final AtomicLong skippedWithLowerCount = new AtomicLong(0L); 3146 getMostCommonElements(countMap, resultCodeList, displayCount, 3147 skippedWithSameCount, skippedWithLowerCount); 3148 3149 long count = -1L; 3150 for (final ObjectPair<ResultCode,Long> p : resultCodeList) 3151 { 3152 count = p.getSecond(); 3153 3154 if (totalCount > 0L) 3155 { 3156 final double percent = 100.0 * count / totalCount; 3157 out(p.getFirst().getName(), " (", p.getFirst().intValue(), "): ", 3158 count, " (", decimalFormat.format(percent), ")"); 3159 } 3160 else 3161 { 3162 out(p.getFirst(), ": ", count); 3163 } 3164 } 3165 3166 if (skippedWithSameCount.get() > 0L) 3167 { 3168 out("{ Skipped " + skippedWithSameCount.get() + " additional result " + 3169 getSingularOrPlural(skippedWithSameCount.get(), "code", "codes") + 3170 " with a count of " + count + " }"); 3171 } 3172 3173 if (skippedWithLowerCount.get() > 0L) 3174 { 3175 out("{ Skipped " + skippedWithLowerCount.get() + " additional result " + 3176 getSingularOrPlural(skippedWithLowerCount.get(), "code", "codes") + 3177 " with a count that is less than " + count + " }"); 3178 } 3179 } 3180 3181 3182 3183 /** 3184 * Retrieves the appropriate singular or plural form based on the given 3185 * value. 3186 * 3187 * @param count The count that will be used to determine whether to 3188 * retrieve the singular or plural form. 3189 * @param singular The singular form for the value to return. 3190 * @param plural The plural form for the value to return. 3191 * 3192 * @return The singular form if the count is 1, or the plural form if the 3193 * count is any other value. 3194 */ 3195 @NotNull() 3196 private String getSingularOrPlural(final long count, 3197 @NotNull final String singular, 3198 @NotNull final String plural) 3199 { 3200 if (count == 1L) 3201 { 3202 return singular; 3203 } 3204 else 3205 { 3206 return plural; 3207 } 3208 } 3209}