001    /*
002     * Copyright 2013-2016 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2013-2016 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.OutputStream;
026    import java.util.Collections;
027    import java.util.LinkedHashMap;
028    import java.util.List;
029    import java.util.Map;
030    import java.util.TreeMap;
031    import java.util.concurrent.atomic.AtomicLong;
032    
033    import com.unboundid.asn1.ASN1OctetString;
034    import com.unboundid.ldap.sdk.Attribute;
035    import com.unboundid.ldap.sdk.DN;
036    import com.unboundid.ldap.sdk.Filter;
037    import com.unboundid.ldap.sdk.LDAPConnectionOptions;
038    import com.unboundid.ldap.sdk.LDAPConnectionPool;
039    import com.unboundid.ldap.sdk.LDAPException;
040    import com.unboundid.ldap.sdk.LDAPSearchException;
041    import com.unboundid.ldap.sdk.ResultCode;
042    import com.unboundid.ldap.sdk.SearchRequest;
043    import com.unboundid.ldap.sdk.SearchResult;
044    import com.unboundid.ldap.sdk.SearchResultEntry;
045    import com.unboundid.ldap.sdk.SearchResultReference;
046    import com.unboundid.ldap.sdk.SearchResultListener;
047    import com.unboundid.ldap.sdk.SearchScope;
048    import com.unboundid.ldap.sdk.Version;
049    import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
050    import com.unboundid.util.Debug;
051    import com.unboundid.util.LDAPCommandLineTool;
052    import com.unboundid.util.StaticUtils;
053    import com.unboundid.util.ThreadSafety;
054    import com.unboundid.util.ThreadSafetyLevel;
055    import com.unboundid.util.args.ArgumentException;
056    import com.unboundid.util.args.ArgumentParser;
057    import com.unboundid.util.args.DNArgument;
058    import com.unboundid.util.args.IntegerArgument;
059    import com.unboundid.util.args.StringArgument;
060    
061    
062    
063    /**
064     * This class provides a tool that may be used to identify references to entries
065     * that do not exist.  This tool can be useful for verifying existing data in
066     * directory servers that provide support for referential integrity.
067     * <BR><BR>
068     * All of the necessary information is provided using command line arguments.
069     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
070     * class, as well as the following additional arguments:
071     * <UL>
072     *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
073     *       for the searches.  At least one base DN must be provided.</LI>
074     *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
075     *       that is expected to contain references to other entries.  This
076     *       attribute should be indexed for equality searches, and its values
077     *       should be DNs.  At least one attribute must be provided.</LI>
078     *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
079     *       to find entries with references to other entries should use the simple
080     *       paged results control to iterate across entries in fixed-size pages
081     *       rather than trying to use a single search to identify all entries that
082     *       reference other entries.</LI>
083     * </UL>
084     */
085    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
086    public final class IdentifyReferencesToMissingEntries
087           extends LDAPCommandLineTool
088           implements SearchResultListener
089    {
090      /**
091       * The serial version UID for this serializable class.
092       */
093      private static final long serialVersionUID = 1981894839719501258L;
094    
095    
096    
097      // The number of entries examined so far.
098      private final AtomicLong entriesExamined;
099    
100      // The argument used to specify the base DNs to use for searches.
101      private DNArgument baseDNArgument;
102    
103      // The argument used to specify the search page size.
104      private IntegerArgument pageSizeArgument;
105    
106      // The connection to use for retrieving referenced entries.
107      private LDAPConnectionPool getReferencedEntriesPool;
108    
109      // A map with counts of missing references by attribute type.
110      private final Map<String,AtomicLong> missingReferenceCounts;
111    
112      // The names of the attributes for which to find missing references.
113      private String[] attributes;
114    
115      // The argument used to specify the attributes for which to find missing
116      // references.
117      private StringArgument attributeArgument;
118    
119    
120    
121      /**
122       * Parse the provided command line arguments and perform the appropriate
123       * processing.
124       *
125       * @param  args  The command line arguments provided to this program.
126       */
127      public static void main(final String... args)
128      {
129        final ResultCode resultCode = main(args, System.out, System.err);
130        if (resultCode != ResultCode.SUCCESS)
131        {
132          System.exit(resultCode.intValue());
133        }
134      }
135    
136    
137    
138      /**
139       * Parse the provided command line arguments and perform the appropriate
140       * processing.
141       *
142       * @param  args       The command line arguments provided to this program.
143       * @param  outStream  The output stream to which standard out should be
144       *                    written.  It may be {@code null} if output should be
145       *                    suppressed.
146       * @param  errStream  The output stream to which standard error should be
147       *                    written.  It may be {@code null} if error messages
148       *                    should be suppressed.
149       *
150       * @return A result code indicating whether the processing was successful.
151       */
152      public static ResultCode main(final String[] args,
153                                    final OutputStream outStream,
154                                    final OutputStream errStream)
155      {
156        final IdentifyReferencesToMissingEntries tool =
157             new IdentifyReferencesToMissingEntries(outStream, errStream);
158        return tool.runTool(args);
159      }
160    
161    
162    
163      /**
164       * Creates a new instance of this tool.
165       *
166       * @param  outStream  The output stream to which standard out should be
167       *                    written.  It may be {@code null} if output should be
168       *                    suppressed.
169       * @param  errStream  The output stream to which standard error should be
170       *                    written.  It may be {@code null} if error messages
171       *                    should be suppressed.
172       */
173      public IdentifyReferencesToMissingEntries(final OutputStream outStream,
174                                                final OutputStream errStream)
175      {
176        super(outStream, errStream);
177    
178        baseDNArgument = null;
179        pageSizeArgument = null;
180        attributeArgument = null;
181        getReferencedEntriesPool = null;
182    
183        entriesExamined = new AtomicLong(0L);
184        missingReferenceCounts = new TreeMap<String, AtomicLong>();
185      }
186    
187    
188    
189      /**
190       * Retrieves the name of this tool.  It should be the name of the command used
191       * to invoke this tool.
192       *
193       * @return  The name for this tool.
194       */
195      @Override()
196      public String getToolName()
197      {
198        return "identify-references-to-missing-entries";
199      }
200    
201    
202    
203      /**
204       * Retrieves a human-readable description for this tool.
205       *
206       * @return  A human-readable description for this tool.
207       */
208      @Override()
209      public String getToolDescription()
210      {
211        return "This tool may be used to identify entries containing one or more " +
212             "attributes which reference entries that do not exist.  This may " +
213             "require the ability to perform unindexed searches and/or the " +
214             "ability to use the simple paged results control.";
215      }
216    
217    
218    
219      /**
220       * Retrieves a version string for this tool, if available.
221       *
222       * @return  A version string for this tool, or {@code null} if none is
223       *          available.
224       */
225      @Override()
226      public String getToolVersion()
227      {
228        return Version.NUMERIC_VERSION_STRING;
229      }
230    
231    
232    
233      /**
234       * Indicates whether this tool should provide support for an interactive mode,
235       * in which the tool offers a mode in which the arguments can be provided in
236       * a text-driven menu rather than requiring them to be given on the command
237       * line.  If interactive mode is supported, it may be invoked using the
238       * "--interactive" argument.  Alternately, if interactive mode is supported
239       * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
240       * interactive mode may be invoked by simply launching the tool without any
241       * arguments.
242       *
243       * @return  {@code true} if this tool supports interactive mode, or
244       *          {@code false} if not.
245       */
246      @Override()
247      public boolean supportsInteractiveMode()
248      {
249        return true;
250      }
251    
252    
253    
254      /**
255       * Indicates whether this tool defaults to launching in interactive mode if
256       * the tool is invoked without any command-line arguments.  This will only be
257       * used if {@link #supportsInteractiveMode()} returns {@code true}.
258       *
259       * @return  {@code true} if this tool defaults to using interactive mode if
260       *          launched without any command-line arguments, or {@code false} if
261       *          not.
262       */
263      @Override()
264      public boolean defaultsToInteractiveMode()
265      {
266        return true;
267      }
268    
269    
270    
271      /**
272       * Indicates whether this tool should provide arguments for redirecting output
273       * to a file.  If this method returns {@code true}, then the tool will offer
274       * an "--outputFile" argument that will specify the path to a file to which
275       * all standard output and standard error content will be written, and it will
276       * also offer a "--teeToStandardOut" argument that can only be used if the
277       * "--outputFile" argument is present and will cause all output to be written
278       * to both the specified output file and to standard output.
279       *
280       * @return  {@code true} if this tool should provide arguments for redirecting
281       *          output to a file, or {@code false} if not.
282       */
283      @Override()
284      protected boolean supportsOutputFile()
285      {
286        return true;
287      }
288    
289    
290    
291      /**
292       * Indicates whether this tool should default to interactively prompting for
293       * the bind password if a password is required but no argument was provided
294       * to indicate how to get the password.
295       *
296       * @return  {@code true} if this tool should default to interactively
297       *          prompting for the bind password, or {@code false} if not.
298       */
299      @Override()
300      protected boolean defaultToPromptForBindPassword()
301      {
302        return true;
303      }
304    
305    
306    
307      /**
308       * Indicates whether this tool supports the use of a properties file for
309       * specifying default values for arguments that aren't specified on the
310       * command line.
311       *
312       * @return  {@code true} if this tool supports the use of a properties file
313       *          for specifying default values for arguments that aren't specified
314       *          on the command line, or {@code false} if not.
315       */
316      @Override()
317      public boolean supportsPropertiesFile()
318      {
319        return true;
320      }
321    
322    
323    
324      /**
325       * Indicates whether the LDAP-specific arguments should include alternate
326       * versions of all long identifiers that consist of multiple words so that
327       * they are available in both camelCase and dash-separated versions.
328       *
329       * @return  {@code true} if this tool should provide multiple versions of
330       *          long identifiers for LDAP-specific arguments, or {@code false} if
331       *          not.
332       */
333      @Override()
334      protected boolean includeAlternateLongIdentifiers()
335      {
336        return true;
337      }
338    
339    
340    
341      /**
342       * Adds the arguments needed by this command-line tool to the provided
343       * argument parser which are not related to connecting or authenticating to
344       * the directory server.
345       *
346       * @param  parser  The argument parser to which the arguments should be added.
347       *
348       * @throws  ArgumentException  If a problem occurs while adding the arguments.
349       */
350      @Override()
351      public void addNonLDAPArguments(final ArgumentParser parser)
352             throws ArgumentException
353      {
354        String description = "The search base DN(s) to use to find entries with " +
355             "references to other entries.  At least one base DN must be " +
356             "specified.";
357        baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
358             description);
359        baseDNArgument.addLongIdentifier("base-dn");
360        parser.addArgument(baseDNArgument);
361    
362        description = "The attribute(s) for which to find missing references.  " +
363             "At least one attribute must be specified, and each attribute " +
364             "must be indexed for equality searches and have values which are DNs.";
365        attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
366             description);
367        parser.addArgument(attributeArgument);
368    
369        description = "The maximum number of entries to retrieve at a time when " +
370             "attempting to find entries with references to other entries.  This " +
371             "requires that the authenticated user have permission to use the " +
372             "simple paged results control, but it can avoid problems with the " +
373             "server sending entries too quickly for the client to handle.  By " +
374             "default, the simple paged results control will not be used.";
375        pageSizeArgument =
376             new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
377                  description, 1, Integer.MAX_VALUE);
378        pageSizeArgument.addLongIdentifier("simple-page-size");
379        parser.addArgument(pageSizeArgument);
380      }
381    
382    
383    
384      /**
385       * Retrieves the connection options that should be used for connections that
386       * are created with this command line tool.  Subclasses may override this
387       * method to use a custom set of connection options.
388       *
389       * @return  The connection options that should be used for connections that
390       *          are created with this command line tool.
391       */
392      @Override()
393      public LDAPConnectionOptions getConnectionOptions()
394      {
395        final LDAPConnectionOptions options = new LDAPConnectionOptions();
396    
397        options.setUseSynchronousMode(true);
398        options.setResponseTimeoutMillis(0L);
399    
400        return options;
401      }
402    
403    
404    
405      /**
406       * Performs the core set of processing for this tool.
407       *
408       * @return  A result code that indicates whether the processing completed
409       *          successfully.
410       */
411      @Override()
412      public ResultCode doToolProcessing()
413      {
414        // Establish a connection to the target directory server to use for
415        // finding references to entries.
416        final LDAPConnectionPool findReferencesPool;
417        try
418        {
419          findReferencesPool = getConnectionPool(1, 1);
420          findReferencesPool.setRetryFailedOperationsDueToInvalidConnections(true);
421        }
422        catch (final LDAPException le)
423        {
424          Debug.debugException(le);
425          err("Unable to establish a connection to the directory server:  ",
426               StaticUtils.getExceptionMessage(le));
427          return le.getResultCode();
428        }
429    
430        try
431        {
432          // Establish a second connection to use for retrieving referenced entries.
433          try
434          {
435            getReferencedEntriesPool = getConnectionPool(1,1);
436            getReferencedEntriesPool.
437                 setRetryFailedOperationsDueToInvalidConnections(true);
438          }
439          catch (final LDAPException le)
440          {
441            Debug.debugException(le);
442            err("Unable to establish a connection to the directory server:  ",
443                 StaticUtils.getExceptionMessage(le));
444            return le.getResultCode();
445          }
446    
447    
448          // Get the set of attributes for which to find missing references.
449          final List<String> attrList = attributeArgument.getValues();
450          attributes = new String[attrList.size()];
451          attrList.toArray(attributes);
452    
453    
454          // Construct a search filter that will be used to find all entries with
455          // references to other entries.
456          final Filter filter;
457          if (attributes.length == 1)
458          {
459            filter = Filter.createPresenceFilter(attributes[0]);
460            missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
461          }
462          else
463          {
464            final Filter[] orComps = new Filter[attributes.length];
465            for (int i=0; i < attributes.length; i++)
466            {
467              orComps[i] = Filter.createPresenceFilter(attributes[i]);
468              missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
469            }
470            filter = Filter.createORFilter(orComps);
471          }
472    
473    
474          // Iterate across all of the search base DNs and perform searches to find
475          // missing references.
476          for (final DN baseDN : baseDNArgument.getValues())
477          {
478            ASN1OctetString cookie = null;
479            do
480            {
481              final SearchRequest searchRequest = new SearchRequest(this,
482                   baseDN.toString(), SearchScope.SUB, filter, attributes);
483              if (pageSizeArgument.isPresent())
484              {
485                searchRequest.addControl(new SimplePagedResultsControl(
486                     pageSizeArgument.getValue(), cookie, false));
487              }
488    
489              SearchResult searchResult;
490              try
491              {
492                searchResult = findReferencesPool.search(searchRequest);
493              }
494              catch (final LDAPSearchException lse)
495              {
496                Debug.debugException(lse);
497                try
498                {
499                  searchResult = findReferencesPool.search(searchRequest);
500                }
501                catch (final LDAPSearchException lse2)
502                {
503                  Debug.debugException(lse2);
504                  searchResult = lse2.getSearchResult();
505                }
506              }
507    
508              if (searchResult.getResultCode() != ResultCode.SUCCESS)
509              {
510                err("An error occurred while attempting to search for missing " +
511                     "references to entries below " + baseDN + ":  " +
512                     searchResult.getDiagnosticMessage());
513                return searchResult.getResultCode();
514              }
515    
516              final SimplePagedResultsControl pagedResultsResponse;
517              try
518              {
519                pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
520              }
521              catch (final LDAPException le)
522              {
523                Debug.debugException(le);
524                err("An error occurred while attempting to decode a simple " +
525                     "paged results response control in the response to a " +
526                     "search for entries below " + baseDN + ":  " +
527                     StaticUtils.getExceptionMessage(le));
528                return le.getResultCode();
529              }
530    
531              if (pagedResultsResponse != null)
532              {
533                if (pagedResultsResponse.moreResultsToReturn())
534                {
535                  cookie = pagedResultsResponse.getCookie();
536                }
537                else
538                {
539                  cookie = null;
540                }
541              }
542            }
543            while (cookie != null);
544          }
545    
546    
547          // See if there were any missing references found.
548          boolean missingReferenceFound = false;
549          for (final Map.Entry<String,AtomicLong> e :
550               missingReferenceCounts.entrySet())
551          {
552            final long numMissing = e.getValue().get();
553            if (numMissing > 0L)
554            {
555              if (! missingReferenceFound)
556              {
557                err();
558                missingReferenceFound = true;
559              }
560    
561              err("Found " + numMissing + ' ' + e.getKey() +
562                   " references to entries that do not exist.");
563            }
564          }
565    
566          if (missingReferenceFound)
567          {
568            return ResultCode.CONSTRAINT_VIOLATION;
569          }
570          else
571          {
572            out("No references were found to entries that do not exist.");
573            return ResultCode.SUCCESS;
574          }
575        }
576        finally
577        {
578          findReferencesPool.close();
579    
580          if (getReferencedEntriesPool != null)
581          {
582            getReferencedEntriesPool.close();
583          }
584        }
585      }
586    
587    
588    
589      /**
590       * Retrieves a map that correlates the number of missing references found by
591       * attribute type.
592       *
593       * @return  A map that correlates the number of missing references found by
594       *          attribute type.
595       */
596      public Map<String,AtomicLong> getMissingReferenceCounts()
597      {
598        return Collections.unmodifiableMap(missingReferenceCounts);
599      }
600    
601    
602    
603      /**
604       * Retrieves a set of information that may be used to generate example usage
605       * information.  Each element in the returned map should consist of a map
606       * between an example set of arguments and a string that describes the
607       * behavior of the tool when invoked with that set of arguments.
608       *
609       * @return  A set of information that may be used to generate example usage
610       *          information.  It may be {@code null} or empty if no example usage
611       *          information is available.
612       */
613      @Override()
614      public LinkedHashMap<String[],String> getExampleUsages()
615      {
616        final LinkedHashMap<String[],String> exampleMap =
617             new LinkedHashMap<String[],String>(1);
618    
619        final String[] args =
620        {
621          "--hostname", "server.example.com",
622          "--port", "389",
623          "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
624          "--bindPassword", "password",
625          "--baseDN", "dc=example,dc=com",
626          "--attribute", "member",
627          "--attribute", "uniqueMember",
628          "--simplePageSize", "100"
629        };
630        exampleMap.put(args,
631             "Identify all entries below dc=example,dc=com in which either the " +
632                  "member or uniqueMember attribute references an entry that " +
633                  "does not exist.");
634    
635        return exampleMap;
636      }
637    
638    
639    
640      /**
641       * Indicates that the provided search result entry has been returned by the
642       * server and may be processed by this search result listener.
643       *
644       * @param  searchEntry  The search result entry that has been returned by the
645       *                      server.
646       */
647      public void searchEntryReturned(final SearchResultEntry searchEntry)
648      {
649        try
650        {
651          // Find attributes which references to entries that do not exist.
652          for (final String attr : attributes)
653          {
654            final List<Attribute> attrList =
655                 searchEntry.getAttributesWithOptions(attr, null);
656            for (final Attribute a : attrList)
657            {
658              for (final String value : a.getValues())
659              {
660                try
661                {
662                  final SearchResultEntry e =
663                       getReferencedEntriesPool.getEntry(value, "1.1");
664                  if (e == null)
665                  {
666                    err("Entry '", searchEntry.getDN(), "' includes attribute ",
667                         a.getName(), " that references entry '", value,
668                         "' which does not exist.");
669                    missingReferenceCounts.get(attr).incrementAndGet();
670                  }
671                }
672                catch (final LDAPException le)
673                {
674                  Debug.debugException(le);
675                  err("An error occurred while attempting to determine whether " +
676                       "entry '" + value + "' referenced in attribute " +
677                       a.getName() + " of entry '" + searchEntry.getDN() +
678                       "' exists:  " + StaticUtils.getExceptionMessage(le));
679                  missingReferenceCounts.get(attr).incrementAndGet();
680                }
681              }
682            }
683          }
684        }
685        finally
686        {
687          final long count = entriesExamined.incrementAndGet();
688          if ((count % 1000L) == 0L)
689          {
690            out(count, " entries examined");
691          }
692        }
693      }
694    
695    
696    
697      /**
698       * Indicates that the provided search result reference has been returned by
699       * the server and may be processed by this search result listener.
700       *
701       * @param  searchReference  The search result reference that has been returned
702       *                          by the server.
703       */
704      public void searchReferenceReturned(
705                       final SearchResultReference searchReference)
706      {
707        // No implementation is required.  This tool will not follow referrals.
708      }
709    }