001    /*
002     * Copyright 2013-2015 UnboundID Corp.
003     * All Rights Reserved.
004     */
005    /*
006     * Copyright (C) 2013-2015 UnboundID Corp.
007     *
008     * This program is free software; you can redistribute it and/or modify
009     * it under the terms of the GNU General Public License (GPLv2 only)
010     * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011     * as published by the Free Software Foundation.
012     *
013     * This program is distributed in the hope that it will be useful,
014     * but WITHOUT ANY WARRANTY; without even the implied warranty of
015     * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016     * GNU General Public License for more details.
017     *
018     * You should have received a copy of the GNU General Public License
019     * along with this program; if not, see <http://www.gnu.org/licenses>.
020     */
021    package com.unboundid.ldap.sdk.examples;
022    
023    
024    
025    import java.io.OutputStream;
026    import java.util.Collections;
027    import java.util.LinkedHashMap;
028    import java.util.List;
029    import java.util.Map;
030    import java.util.TreeMap;
031    import java.util.concurrent.atomic.AtomicLong;
032    
033    import com.unboundid.asn1.ASN1OctetString;
034    import com.unboundid.ldap.sdk.Attribute;
035    import com.unboundid.ldap.sdk.DN;
036    import com.unboundid.ldap.sdk.Filter;
037    import com.unboundid.ldap.sdk.LDAPConnection;
038    import com.unboundid.ldap.sdk.LDAPException;
039    import com.unboundid.ldap.sdk.LDAPSearchException;
040    import com.unboundid.ldap.sdk.ResultCode;
041    import com.unboundid.ldap.sdk.SearchRequest;
042    import com.unboundid.ldap.sdk.SearchResult;
043    import com.unboundid.ldap.sdk.SearchResultEntry;
044    import com.unboundid.ldap.sdk.SearchResultReference;
045    import com.unboundid.ldap.sdk.SearchResultListener;
046    import com.unboundid.ldap.sdk.SearchScope;
047    import com.unboundid.ldap.sdk.Version;
048    import com.unboundid.ldap.sdk.controls.SimplePagedResultsControl;
049    import com.unboundid.util.Debug;
050    import com.unboundid.util.LDAPCommandLineTool;
051    import com.unboundid.util.StaticUtils;
052    import com.unboundid.util.ThreadSafety;
053    import com.unboundid.util.ThreadSafetyLevel;
054    import com.unboundid.util.args.ArgumentException;
055    import com.unboundid.util.args.ArgumentParser;
056    import com.unboundid.util.args.DNArgument;
057    import com.unboundid.util.args.IntegerArgument;
058    import com.unboundid.util.args.StringArgument;
059    
060    
061    
062    /**
063     * This class provides a tool that may be used to identify references to entries
064     * that do not exist.  This tool can be useful for verifying existing data in
065     * directory servers that provide support for referential integrity.
066     * <BR><BR>
067     * All of the necessary information is provided using command line arguments.
068     * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
069     * class, as well as the following additional arguments:
070     * <UL>
071     *   <LI>"-b {baseDN}" or "--baseDN {baseDN}" -- specifies the base DN to use
072     *       for the searches.  At least one base DN must be provided.</LI>
073     *   <LI>"-A {attribute}" or "--attribute {attribute}" -- specifies an attribute
074     *       that is expected to contain references to other entries.  This
075     *       attribute should be indexed for equality searches, and its values
076     *       should be DNs.  At least one attribute must be provided.</LI>
077     *   <LI>"-z {size}" or "--simplePageSize {size}" -- indicates that the search
078     *       to find entries with references to other entries should use the simple
079     *       paged results control to iterate across entries in fixed-size pages
080     *       rather than trying to use a single search to identify all entries that
081     *       reference other entries.</LI>
082     * </UL>
083     */
084    @ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
085    public final class IdentifyReferencesToMissingEntries
086           extends LDAPCommandLineTool
087           implements SearchResultListener
088    {
089      /**
090       * The serial version UID for this serializable class.
091       */
092      private static final long serialVersionUID = 1981894839719501258L;
093    
094    
095    
096      // The number of entries examined so far.
097      private final AtomicLong entriesExamined;
098    
099      // The argument used to specify the base DNs to use for searches.
100      private DNArgument baseDNArgument;
101    
102      // The argument used to specify the search page size.
103      private IntegerArgument pageSizeArgument;
104    
105      // The connection to use for retrieving referenced entries.
106      private LDAPConnection getReferencedEntriesConnection;
107    
108      // A map with counts of missing references by attribute type.
109      private final Map<String,AtomicLong> missingReferenceCounts;
110    
111      // The names of the attributes for which to find missing references.
112      private String[] attributes;
113    
114      // The argument used to specify the attributes for which to find missing
115      // references.
116      private StringArgument attributeArgument;
117    
118    
119    
120      /**
121       * Parse the provided command line arguments and perform the appropriate
122       * processing.
123       *
124       * @param  args  The command line arguments provided to this program.
125       */
126      public static void main(final String... args)
127      {
128        final ResultCode resultCode = main(args, System.out, System.err);
129        if (resultCode != ResultCode.SUCCESS)
130        {
131          System.exit(resultCode.intValue());
132        }
133      }
134    
135    
136    
137      /**
138       * Parse the provided command line arguments and perform the appropriate
139       * processing.
140       *
141       * @param  args       The command line arguments provided to this program.
142       * @param  outStream  The output stream to which standard out should be
143       *                    written.  It may be {@code null} if output should be
144       *                    suppressed.
145       * @param  errStream  The output stream to which standard error should be
146       *                    written.  It may be {@code null} if error messages
147       *                    should be suppressed.
148       *
149       * @return A result code indicating whether the processing was successful.
150       */
151      public static ResultCode main(final String[] args,
152                                    final OutputStream outStream,
153                                    final OutputStream errStream)
154      {
155        final IdentifyReferencesToMissingEntries tool =
156             new IdentifyReferencesToMissingEntries(outStream, errStream);
157        return tool.runTool(args);
158      }
159    
160    
161    
162      /**
163       * Creates a new instance of this tool.
164       *
165       * @param  outStream  The output stream to which standard out should be
166       *                    written.  It may be {@code null} if output should be
167       *                    suppressed.
168       * @param  errStream  The output stream to which standard error should be
169       *                    written.  It may be {@code null} if error messages
170       *                    should be suppressed.
171       */
172      public IdentifyReferencesToMissingEntries(final OutputStream outStream,
173                                                final OutputStream errStream)
174      {
175        super(outStream, errStream);
176    
177        baseDNArgument = null;
178        pageSizeArgument = null;
179        attributeArgument = null;
180        getReferencedEntriesConnection = null;
181    
182        entriesExamined = new AtomicLong(0L);
183        missingReferenceCounts = new TreeMap<String, AtomicLong>();
184      }
185    
186    
187    
188      /**
189       * Retrieves the name of this tool.  It should be the name of the command used
190       * to invoke this tool.
191       *
192       * @return  The name for this tool.
193       */
194      @Override()
195      public String getToolName()
196      {
197        return "identify-references-to-missing-entries";
198      }
199    
200    
201    
202      /**
203       * Retrieves a human-readable description for this tool.
204       *
205       * @return  A human-readable description for this tool.
206       */
207      @Override()
208      public String getToolDescription()
209      {
210        return "This tool may be used to identify entries containing one or more " +
211             "attributes which reference entries that do not exist.  This may " +
212             "require the ability to perform unindexed searches and/or the " +
213             "ability to use the simple paged results control.";
214      }
215    
216    
217    
218      /**
219       * Retrieves a version string for this tool, if available.
220       *
221       * @return  A version string for this tool, or {@code null} if none is
222       *          available.
223       */
224      @Override()
225      public String getToolVersion()
226      {
227        return Version.NUMERIC_VERSION_STRING;
228      }
229    
230    
231    
232      /**
233       * Adds the arguments needed by this command-line tool to the provided
234       * argument parser which are not related to connecting or authenticating to
235       * the directory server.
236       *
237       * @param  parser  The argument parser to which the arguments should be added.
238       *
239       * @throws  ArgumentException  If a problem occurs while adding the arguments.
240       */
241      @Override()
242      public void addNonLDAPArguments(final ArgumentParser parser)
243             throws ArgumentException
244      {
245        String description = "The search base DN(s) to use to find entries with " +
246             "references to other entries.  At least one base DN must be " +
247             "specified.";
248        baseDNArgument = new DNArgument('b', "baseDN", true, 0, "{dn}",
249             description);
250        parser.addArgument(baseDNArgument);
251    
252        description = "The attribute(s) for which to find missing references.  " +
253             "At least one attribute must be specified, and each attribute " +
254             "must be indexed for equality searches and have values which are DNs.";
255        attributeArgument = new StringArgument('A', "attribute", true, 0, "{attr}",
256             description);
257        parser.addArgument(attributeArgument);
258    
259        description = "The maximum number of entries to retrieve at a time when " +
260             "attempting to find entries with references to other entries.  This " +
261             "requires that the authenticated user have permission to use the " +
262             "simple paged results control, but it can avoid problems with the " +
263             "server sending entries too quickly for the client to handle.  By " +
264             "default, the simple paged results control will not be used.";
265        pageSizeArgument =
266             new IntegerArgument('z', "simplePageSize", false, 1, "{num}",
267                  description, 1, Integer.MAX_VALUE);
268        parser.addArgument(pageSizeArgument);
269      }
270    
271    
272    
273      /**
274       * Performs the core set of processing for this tool.
275       *
276       * @return  A result code that indicates whether the processing completed
277       *          successfully.
278       */
279      @Override()
280      public ResultCode doToolProcessing()
281      {
282        // Establish a connection to the target directory server to use for
283        // finding references to entries.
284        final LDAPConnection findReferencesConnection;
285        try
286        {
287          findReferencesConnection = getConnection();
288        }
289        catch (final LDAPException le)
290        {
291          Debug.debugException(le);
292          err("Unable to establish a connection to the directory server:  ",
293               StaticUtils.getExceptionMessage(le));
294          return le.getResultCode();
295        }
296    
297        try
298        {
299          // Establish a second connection to use for retrieving referenced entries.
300          try
301          {
302            getReferencedEntriesConnection = getConnection();
303          }
304          catch (final LDAPException le)
305          {
306            Debug.debugException(le);
307            err("Unable to establish a connection to the directory server:  ",
308                 StaticUtils.getExceptionMessage(le));
309            return le.getResultCode();
310          }
311    
312    
313          // Get the set of attributes for which to find missing references.
314          final List<String> attrList = attributeArgument.getValues();
315          attributes = new String[attrList.size()];
316          attrList.toArray(attributes);
317    
318    
319          // Construct a search filter that will be used to find all entries with
320          // references to other entries.
321          final Filter filter;
322          if (attributes.length == 1)
323          {
324            filter = Filter.createPresenceFilter(attributes[0]);
325            missingReferenceCounts.put(attributes[0], new AtomicLong(0L));
326          }
327          else
328          {
329            final Filter[] orComps = new Filter[attributes.length];
330            for (int i=0; i < attributes.length; i++)
331            {
332              orComps[i] = Filter.createPresenceFilter(attributes[i]);
333              missingReferenceCounts.put(attributes[i], new AtomicLong(0L));
334            }
335            filter = Filter.createORFilter(orComps);
336          }
337    
338    
339          // Iterate across all of the search base DNs and perform searches to find
340          // missing references.
341          for (final DN baseDN : baseDNArgument.getValues())
342          {
343            ASN1OctetString cookie = null;
344            do
345            {
346              final SearchRequest searchRequest = new SearchRequest(this,
347                   baseDN.toString(), SearchScope.SUB, filter, attributes);
348              if (pageSizeArgument.isPresent())
349              {
350                searchRequest.addControl(new SimplePagedResultsControl(
351                     pageSizeArgument.getValue(), cookie, false));
352              }
353    
354              SearchResult searchResult;
355              try
356              {
357                searchResult = findReferencesConnection.search(searchRequest);
358              }
359              catch (final LDAPSearchException lse)
360              {
361                Debug.debugException(lse);
362                searchResult = lse.getSearchResult();
363              }
364    
365              if (searchResult.getResultCode() != ResultCode.SUCCESS)
366              {
367                err("An error occurred while attempting to search for missing " +
368                     "references to entries below " + baseDN + ":  " +
369                     searchResult.getDiagnosticMessage());
370                return searchResult.getResultCode();
371              }
372    
373              final SimplePagedResultsControl pagedResultsResponse;
374              try
375              {
376                pagedResultsResponse = SimplePagedResultsControl.get(searchResult);
377              }
378              catch (final LDAPException le)
379              {
380                Debug.debugException(le);
381                err("An error occurred while attempting to decode a simple " +
382                     "paged results response control in the response to a " +
383                     "search for entries below " + baseDN + ":  " +
384                     StaticUtils.getExceptionMessage(le));
385                return le.getResultCode();
386              }
387    
388              if (pagedResultsResponse != null)
389              {
390                if (pagedResultsResponse.moreResultsToReturn())
391                {
392                  cookie = pagedResultsResponse.getCookie();
393                }
394                else
395                {
396                  cookie = null;
397                }
398              }
399            }
400            while (cookie != null);
401          }
402    
403    
404          // See if there were any missing references found.
405          boolean missingReferenceFound = false;
406          for (final Map.Entry<String,AtomicLong> e :
407               missingReferenceCounts.entrySet())
408          {
409            final long numMissing = e.getValue().get();
410            if (numMissing > 0L)
411            {
412              if (! missingReferenceFound)
413              {
414                err();
415                missingReferenceFound = true;
416              }
417    
418              err("Found " + numMissing + ' ' + e.getKey() +
419                   " references to entries that do not exist.");
420            }
421          }
422    
423          if (missingReferenceFound)
424          {
425            return ResultCode.CONSTRAINT_VIOLATION;
426          }
427          else
428          {
429            out("No references were found to entries that do not exist.");
430            return ResultCode.SUCCESS;
431          }
432        }
433        finally
434        {
435          findReferencesConnection.close();
436    
437          if (getReferencedEntriesConnection != null)
438          {
439            getReferencedEntriesConnection.close();
440          }
441        }
442      }
443    
444    
445    
446      /**
447       * Retrieves a map that correlates the number of missing references found by
448       * attribute type.
449       *
450       * @return  A map that correlates the number of missing references found by
451       *          attribute type.
452       */
453      public Map<String,AtomicLong> getMissingReferenceCounts()
454      {
455        return Collections.unmodifiableMap(missingReferenceCounts);
456      }
457    
458    
459    
460      /**
461       * Retrieves a set of information that may be used to generate example usage
462       * information.  Each element in the returned map should consist of a map
463       * between an example set of arguments and a string that describes the
464       * behavior of the tool when invoked with that set of arguments.
465       *
466       * @return  A set of information that may be used to generate example usage
467       *          information.  It may be {@code null} or empty if no example usage
468       *          information is available.
469       */
470      @Override()
471      public LinkedHashMap<String[],String> getExampleUsages()
472      {
473        final LinkedHashMap<String[],String> exampleMap =
474             new LinkedHashMap<String[],String>(1);
475    
476        final String[] args =
477        {
478          "--hostname", "server.example.com",
479          "--port", "389",
480          "--bindDN", "uid=john.doe,ou=People,dc=example,dc=com",
481          "--bindPassword", "password",
482          "--baseDN", "dc=example,dc=com",
483          "--attribute", "member",
484          "--attribute", "uniqueMember",
485          "--simplePageSize", "100"
486        };
487        exampleMap.put(args,
488             "Identify all entries below dc=example,dc=com in which either the " +
489                  "member or uniqueMember attribute references an entry that " +
490                  "does not exist.");
491    
492        return exampleMap;
493      }
494    
495    
496    
497      /**
498       * Indicates that the provided search result entry has been returned by the
499       * server and may be processed by this search result listener.
500       *
501       * @param  searchEntry  The search result entry that has been returned by the
502       *                      server.
503       */
504      public void searchEntryReturned(final SearchResultEntry searchEntry)
505      {
506        try
507        {
508          // Find attributes which references to entries that do not exist.
509          for (final String attr : attributes)
510          {
511            final List<Attribute> attrList =
512                 searchEntry.getAttributesWithOptions(attr, null);
513            for (final Attribute a : attrList)
514            {
515              for (final String value : a.getValues())
516              {
517                try
518                {
519                  final SearchResultEntry e =
520                       getReferencedEntriesConnection.getEntry(value, "1.1");
521                  if (e == null)
522                  {
523                    err("Entry '", searchEntry.getDN(), "' includes attribute ",
524                         a.getName(), " that references entry '", value,
525                         "' which does not exist.");
526                    missingReferenceCounts.get(attr).incrementAndGet();
527                  }
528                }
529                catch (final LDAPException le)
530                {
531                  Debug.debugException(le);
532                  err("An error occurred while attempting to determine whether " +
533                       "entry '" + value + "' referenced in attribute " +
534                       a.getName() + " of entry '" + searchEntry.getDN() +
535                       "' exists:  " + StaticUtils.getExceptionMessage(le));
536                  missingReferenceCounts.get(attr).incrementAndGet();
537                }
538              }
539            }
540          }
541        }
542        finally
543        {
544          final long count = entriesExamined.incrementAndGet();
545          if ((count % 1000L) == 0L)
546          {
547            out(count, " entries examined");
548          }
549        }
550      }
551    
552    
553    
554      /**
555       * Indicates that the provided search result reference has been returned by
556       * the server and may be processed by this search result listener.
557       *
558       * @param  searchReference  The search result reference that has been returned
559       *                          by the server.
560       */
561      public void searchReferenceReturned(
562                       final SearchResultReference searchReference)
563      {
564        // No implementation is required.  This tool will not follow referrals.
565      }
566    }