source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.search;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.admin.model.RepositoryAdminException;
  23 import org.apache.archiva.admin.model.beans.ManagedRepository;
  24 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
  25 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
  26 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
  27 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
  28 import org.apache.archiva.indexer.util.SearchUtil;
  29 import org.apache.commons.lang.StringUtils;
  30 import org.apache.lucene.search.BooleanClause.Occur;
  31 import org.apache.lucene.search.BooleanQuery;
  32 import org.apache.maven.index.ArtifactInfo;
  33 import org.apache.maven.index.FlatSearchRequest;
  34 import org.apache.maven.index.FlatSearchResponse;
  35 import org.apache.maven.index.MAVEN;
  36 import org.apache.maven.index.NexusIndexer;
  37 import org.apache.maven.index.OSGI;
  38 import org.apache.maven.index.context.IndexCreator;
  39 import org.apache.maven.index.context.IndexingContext;
  40 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
  41 import org.apache.maven.index.expr.StringSearchExpression;
  42 import org.slf4j.Logger;
  43 import org.slf4j.LoggerFactory;
  44 import org.springframework.stereotype.Service;
  45
  46 import javax.inject.Inject;
  47 import java.io.File;
  48 import java.io.IOException;
  49 import java.util.ArrayList;
  50 import java.util.Collections;
  51 import java.util.List;
  52 import java.util.Map;
  53 import java.util.Set;
  54
  55 /**
  56  * RepositorySearch implementation which uses the Nexus Indexer for searching.
  57  */
  58 @Service( "nexusSearch" )
  59 public class NexusRepositorySearch
  60     implements RepositorySearch
  61 {
  62     private Logger log = LoggerFactory.getLogger( getClass() );
  63
  64     private NexusIndexer indexer;
  65
  66     private ManagedRepositoryAdmin managedRepositoryAdmin;
  67
  68     private MavenIndexerUtils mavenIndexerUtils;
  69
  70     @Inject
  71     public NexusRepositorySearch( PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
  72                                   MavenIndexerUtils mavenIndexerUtils )
  73         throws PlexusSisuBridgeException
  74     {
  75         this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
  76         this.managedRepositoryAdmin = managedRepositoryAdmin;
  77         this.mavenIndexerUtils = mavenIndexerUtils;
  78
  79     }
  80
  81     /**
  82      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
  83      */
  84     public SearchResults search( String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
  85                                  List<String> previousSearchTerms )
  86         throws RepositorySearchException
  87     {
  88         List<String> indexingContextIds = addIndexingContexts( selectedRepos );
  89
  90         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
  91         //      resulting to more wildcard searches so we need to increase max clause count
  92         BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
  93         BooleanQuery q = new BooleanQuery();
  94
  95         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
  96         {
  97             constructQuery( term, q );
  98         }
  99         else
 100         {
 101             for ( String previousTerm : previousSearchTerms )
 102             {
 103                 BooleanQuery iQuery = new BooleanQuery();
 104                 constructQuery( previousTerm, iQuery );
 105
 106                 q.add( iQuery, Occur.MUST );
 107             }
 108
 109             BooleanQuery iQuery = new BooleanQuery();
 110             constructQuery( term, iQuery );
 111             q.add( iQuery, Occur.MUST );
 112         }
 113
 114         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
 115         // FIXME  cannot find a way currently to setup this in constructQuery !!!
 116         return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST );
 117
 118     }
 119
 120     /**
 121      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
 122      */
 123     public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
 124         throws RepositorySearchException
 125     {
 126         if ( searchFields.getRepositories() == null )
 127         {
 128             throw new RepositorySearchException( "Repositories cannot be null." );
 129         }
 130
 131         List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() );
 132
 133         BooleanQuery q = new BooleanQuery();
 134         if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
 135         {
 136             q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( searchFields.getGroupId() ) ),
 137                    Occur.MUST );
 138         }
 139
 140         if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
 141         {
 142             q.add(
 143                 indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( searchFields.getArtifactId() ) ),
 144                 Occur.MUST );
 145         }
 146
 147         if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
 148         {
 149             q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( searchFields.getVersion() ) ),
 150                    Occur.MUST );
 151         }
 152
 153         if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
 154         {
 155             q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( searchFields.getPackaging() ) ),
 156                    Occur.MUST );
 157         }
 158
 159         if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
 160         {
 161             q.add(
 162                 indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( searchFields.getClassName() ) ),
 163                 Occur.MUST );
 164         }
 165
 166         if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
 167         {
 168             q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
 169                                            new StringSearchExpression( searchFields.getBundleSymbolicName() ) ),
 170                    Occur.MUST );
 171         }
 172
 173         if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
 174         {
 175             q.add(
 176                 indexer.constructQuery( OSGI.VERSION, new StringSearchExpression( searchFields.getBundleVersion() ) ),
 177                 Occur.MUST );
 178         }
 179
 180         if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
 181         {
 182             q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
 183                                            new StringSearchExpression( searchFields.getBundleExportPackage() ) ),
 184                    Occur.MUST );
 185         }
 186
 187         if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
 188         {
 189             q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
 190                                            new StringSearchExpression( searchFields.getBundleExportService() ) ),
 191                    Occur.MUST );
 192         }
 193
 194         if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
 195         {
 196             q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
 197                                            new StringSearchExpression( searchFields.getBundleImportPackage() ) ),
 198                    Occur.MUST );
 199         }
 200
 201         if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
 202         {
 203             q.add( indexer.constructQuery( OSGI.NAME, new StringSearchExpression( searchFields.getBundleName() ) ),
 204                    Occur.MUST );
 205         }
 206
 207         if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
 208         {
 209             q.add(
 210                 indexer.constructQuery( MAVEN.CLASSIFIER, new StringSearchExpression( searchFields.getClassifier() ) ),
 211                 Occur.MUST );
 212         }
 213
 214         if ( q.getClauses() == null || q.getClauses().length <= 0 )
 215         {
 216             throw new RepositorySearchException( "No search fields set." );
 217         }
 218
 219         return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList() );
 220     }
 221
 222     private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
 223                                   List<? extends ArtifactInfoFiler> filters )
 224         throws RepositorySearchException
 225     {
 226
 227         try
 228         {
 229             FlatSearchRequest request = new FlatSearchRequest( q );
 230             request.setContexts( getIndexingContexts( indexingContextIds ) );
 231             FlatSearchResponse response = indexer.searchFlat( request );
 232
 233             if ( response == null || response.getTotalHits() == 0 )
 234             {
 235                 SearchResults results = new SearchResults();
 236                 results.setLimits( limits );
 237                 return results;
 238             }
 239
 240             return convertToSearchResults( response, limits, filters );
 241         }
 242         catch ( IOException e )
 243         {
 244             throw new RepositorySearchException( e );
 245         }
 246         /*
 247         olamy : don't understand why this ?? it remove content from index ??
 248         comment until someone explain WTF ?? :-))
 249         finally
 250         {
 251             Map<String, IndexingContext> indexingContexts = indexer.getIndexingContexts();
 252
 253             for ( Map.Entry<String, IndexingContext> entry : indexingContexts.entrySet() )
 254             {
 255                 try
 256                 {
 257                     indexer.removeIndexingContext( entry.getValue(), false );
 258                     log.debug( "Indexing context '{}' removed from search.", entry.getKey() );
 259                 }
 260                 catch ( IOException e )
 261                 {
 262                     log.warn( "IOException occurred while removing indexing content '" + entry.getKey() + "'." );
 263                     continue;
 264                 }
 265             }
 266         }*/
 267     }
 268
 269     private List<IndexingContext> getIndexingContexts( List<String> ids )
 270     {
 271         List<IndexingContext> contexts = new ArrayList<IndexingContext>( ids.size() );
 272
 273         for ( String id : ids )
 274         {
 275             IndexingContext context = indexer.getIndexingContexts().get( id );
 276             if ( context != null )
 277             {
 278                 contexts.add( context );
 279             }
 280             else
 281             {
 282                 log.warn( "context with id {} not exists", id );
 283             }
 284         }
 285
 286         return contexts;
 287     }
 288
 289     private void constructQuery( String term, BooleanQuery q )
 290     {
 291         q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
 292         q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
 293         q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( term ) ), Occur.SHOULD );
 294         q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( term ) ), Occur.SHOULD );
 295         q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( term ) ), Occur.SHOULD );
 296
 297         //Query query =
 298         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
 299         //q.add( query, Occur.MUST_NOT );
 300         // olamy IMHO we could set this option as at least one must match
 301         //q.setMinimumNumberShouldMatch( 1 );
 302     }
 303
 304
 305     /**
 306      * @param selectedRepos
 307      * @return indexing contextId used
 308      */
 309     private List<String> addIndexingContexts( List<String> selectedRepos )
 310     {
 311         List<String> indexingContextIds = new ArrayList<String>();
 312         for ( String repo : selectedRepos )
 313         {
 314             try
 315             {
 316                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository( repo );
 317
 318                 if ( repoConfig != null )
 319                 {
 320                     String indexDir = repoConfig.getIndexDirectory();
 321                     File indexDirectory = null;
 322                     if ( indexDir != null && !"".equals( indexDir ) )
 323                     {
 324                         indexDirectory = new File( repoConfig.getIndexDirectory() );
 325                     }
 326                     else
 327                     {
 328                         indexDirectory = new File( repoConfig.getLocation(), ".indexer" );
 329                     }
 330
 331                     IndexingContext context = indexer.getIndexingContexts().get( repoConfig.getId() );
 332                     if ( context != null )
 333                     {
 334                         // alreday here so no need to record it again
 335                         log.debug( "index with id {} already exists skip adding it", repoConfig.getId() );
 336                         // set searchable flag
 337                         context.setSearchable( repoConfig.isScanned() );
 338                         indexingContextIds.add( context.getId() );
 339                         continue;
 340                     }
 341
 342                     context = indexer.addIndexingContext( repoConfig.getId(), repoConfig.getId(),
 343                                                           new File( repoConfig.getLocation() ), indexDirectory, null,
 344                                                           null, getAllIndexCreators() );
 345                     context.setSearchable( repoConfig.isScanned() );
 346                     if ( context.isSearchable() )
 347                     {
 348                         indexingContextIds.add( context.getId() );
 349                     }
 350                     else
 351                     {
 352                         log.warn( "indexingContext with id {} not searchable", repoConfig.getId() );
 353                     }
 354
 355                 }
 356                 else
 357                 {
 358                     log.warn( "Repository '" + repo + "' not found in configuration." );
 359                 }
 360             }
 361             catch ( UnsupportedExistingLuceneIndexException e )
 362             {
 363                 log.warn( "Error accessing index of repository '" + repo + "' : " + e.getMessage() );
 364                 continue;
 365             }
 366             catch ( IOException e )
 367             {
 368                 log.warn( "IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage() );
 369                 continue;
 370             }
 371             catch ( RepositoryAdminException e )
 372             {
 373                 log.warn( "RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
 374                               + e.getMessage() );
 375                 continue;
 376             }
 377         }
 378         return indexingContextIds;
 379     }
 380
 381
 382     protected List<? extends IndexCreator> getAllIndexCreators()
 383     {
 384         return mavenIndexerUtils.getAllIndexCreators();
 385     }
 386
 387
 388     private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits,
 389                                                   List<? extends ArtifactInfoFiler> artifactInfoFilers )
 390     {
 391         SearchResults results = new SearchResults();
 392         Set<ArtifactInfo> artifactInfos = response.getResults();
 393
 394         for ( ArtifactInfo artifactInfo : artifactInfos )
 395         {
 396             String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
 397                                              artifactInfo.packaging );
 398             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
 399
 400             if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilers, hitsMap ) )
 401             {
 402                 continue;
 403             }
 404
 405             SearchResultHit hit = hitsMap.get( id );
 406             if ( hit != null )
 407             {
 408                 if ( !hit.getVersions().contains( artifactInfo.version ) )
 409                 {
 410                     hit.addVersion( artifactInfo.version );
 411                 }
 412             }
 413             else
 414             {
 415                 hit = new SearchResultHit();
 416                 hit.setArtifactId( artifactInfo.artifactId );
 417                 hit.setGroupId( artifactInfo.groupId );
 418                 hit.setRepositoryId( artifactInfo.repository );
 419                 // FIXME archiva url ??
 420                 hit.setUrl( artifactInfo.repository + "/" + artifactInfo.fname );
 421                 hit.addVersion( artifactInfo.version );
 422                 hit.setBundleExportPackage( artifactInfo.bundleExportPackage );
 423                 hit.setBundleExportService( artifactInfo.bundleExportService );
 424                 hit.setBundleSymbolicName( artifactInfo.bundleSymbolicName );
 425                 hit.setBundleVersion( artifactInfo.bundleVersion );
 426                 hit.setBundleDescription( artifactInfo.bundleDescription );
 427                 hit.setBundleDocUrl( artifactInfo.bundleDocUrl );
 428
 429                 hit.setBundleRequireBundle( artifactInfo.bundleRequireBundle );
 430                 hit.setBundleImportPackage( artifactInfo.bundleImportPackage );
 431                 hit.setBundleLicense( artifactInfo.bundleLicense );
 432                 hit.setBundleName( artifactInfo.bundleName );
 433                 hit.setContext( artifactInfo.context );
 434                 hit.setGoals( artifactInfo.goals );
 435                 hit.setPrefix( artifactInfo.prefix );
 436                 hit.setPackaging( artifactInfo.packaging );
 437                 hit.setClassifier( artifactInfo.classifier );
 438                 // sure ??
 439                 hit.setUrl( artifactInfo.remoteUrl );
 440             }
 441
 442             results.addHit( id, hit );
 443         }
 444
 445         results.setTotalHits( response.getTotalHitsCount() );
 446         results.setReturnedHitsCount( response.getReturnedHitsCount() );
 447         results.setLimits( limits );
 448
 449         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
 450         {
 451             return results;
 452         }
 453         else
 454         {
 455             return paginate( results );
 456         }
 457     }
 458
 459     private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo,
 460                                               List<? extends ArtifactInfoFiler> artifactInfoFilers,
 461                                               Map<String, SearchResultHit> currentResult )
 462     {
 463         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
 464         {
 465             return true;
 466         }
 467
 468         for ( ArtifactInfoFiler filter : artifactInfoFilers )
 469         {
 470             if ( !filter.addArtifactInResult( artifactInfo, currentResult ) )
 471             {
 472                 return false;
 473             }
 474         }
 475         return true;
 476     }
 477
 478     private SearchResults paginate( SearchResults results )
 479     {
 480         SearchResultLimits limits = results.getLimits();
 481         SearchResults paginated = new SearchResults();
 482
 483         int fetchCount = limits.getPageSize();
 484         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
 485
 486         if ( fetchCount > results.getTotalHits() )
 487         {
 488             fetchCount = results.getTotalHits();
 489         }
 490
 491         // Goto offset.
 492         if ( offset < results.getTotalHits() )
 493         {
 494             // only process if the offset is within the hit count.
 495             for ( int i = 0; i < fetchCount; i++ )
 496             {
 497                 // Stop fetching if we are past the total # of available hits.
 498                 if ( offset + i >= results.getHits().size() )
 499                 {
 500                     break;
 501                 }
 502
 503                 SearchResultHit hit = results.getHits().get( ( offset + i ) );
 504                 if ( hit != null )
 505                 {
 506                     String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
 507                                                      hit.getPackaging() );
 508                     paginated.addHit( id, hit );
 509                 }
 510                 else
 511                 {
 512                     break;
 513                 }
 514             }
 515         }
 516         paginated.setTotalHits( results.getTotalHits() );
 517         paginated.setReturnedHitsCount( paginated.getHits().size() );
 518         paginated.setLimits( limits );
 519
 520         return paginated;
 521     }
 522 }