source.dussan.org Git - archiva.git/blob

   1 package org.apache.archiva.indexer.search;
   2
   3 /*
   4  * Licensed to the Apache Software Foundation (ASF) under one
   5  * or more contributor license agreements.  See the NOTICE file
   6  * distributed with this work for additional information
   7  * regarding copyright ownership.  The ASF licenses this file
   8  * to you under the Apache License, Version 2.0 (the
   9  * "License"); you may not use this file except in compliance
  10  * with the License.  You may obtain a copy of the License at
  11  *
  12  *  http://www.apache.org/licenses/LICENSE-2.0
  13  *
  14  * Unless required by applicable law or agreed to in writing,
  15  * software distributed under the License is distributed on an
  16  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
  17  * KIND, either express or implied.  See the License for the
  18  * specific language governing permissions and limitations
  19  * under the License.
  20  */
  21
  22 import org.apache.archiva.admin.model.RepositoryAdminException;
  23 import org.apache.archiva.admin.model.beans.ManagedRepository;
  24 import org.apache.archiva.admin.model.beans.ProxyConnector;
  25 import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin;
  26 import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin;
  27 import org.apache.archiva.common.plexusbridge.MavenIndexerUtils;
  28 import org.apache.archiva.common.plexusbridge.PlexusSisuBridge;
  29 import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException;
  30 import org.apache.archiva.indexer.util.SearchUtil;
  31 import org.apache.commons.lang.StringUtils;
  32 import org.apache.lucene.search.BooleanClause.Occur;
  33 import org.apache.lucene.search.BooleanQuery;
  34 import org.apache.maven.index.ArtifactInfo;
  35 import org.apache.maven.index.FlatSearchRequest;
  36 import org.apache.maven.index.FlatSearchResponse;
  37 import org.apache.maven.index.MAVEN;
  38 import org.apache.maven.index.NexusIndexer;
  39 import org.apache.maven.index.OSGI;
  40 import org.apache.maven.index.context.IndexCreator;
  41 import org.apache.maven.index.context.IndexingContext;
  42 import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException;
  43 import org.apache.maven.index.expr.StringSearchExpression;
  44 import org.slf4j.Logger;
  45 import org.slf4j.LoggerFactory;
  46 import org.springframework.stereotype.Service;
  47
  48 import javax.inject.Inject;
  49 import java.io.File;
  50 import java.io.IOException;
  51 import java.util.ArrayList;
  52 import java.util.Collections;
  53 import java.util.HashSet;
  54 import java.util.List;
  55 import java.util.Map;
  56 import java.util.Set;
  57
  58 /**
  59  * RepositorySearch implementation which uses the Nexus Indexer for searching.
  60  */
  61 @Service( "nexusSearch" )
  62 public class NexusRepositorySearch
  63     implements RepositorySearch
  64 {
  65     private Logger log = LoggerFactory.getLogger( getClass() );
  66
  67     private NexusIndexer indexer;
  68
  69     private ManagedRepositoryAdmin managedRepositoryAdmin;
  70
  71     private ProxyConnectorAdmin proxyConnectorAdmin;
  72
  73     private MavenIndexerUtils mavenIndexerUtils;
  74
  75     @Inject
  76     public NexusRepositorySearch( PlexusSisuBridge plexusSisuBridge, ManagedRepositoryAdmin managedRepositoryAdmin,
  77                                   MavenIndexerUtils mavenIndexerUtils, ProxyConnectorAdmin proxyConnectorAdmin )
  78         throws PlexusSisuBridgeException
  79     {
  80         this.indexer = plexusSisuBridge.lookup( NexusIndexer.class );
  81         this.managedRepositoryAdmin = managedRepositoryAdmin;
  82         this.mavenIndexerUtils = mavenIndexerUtils;
  83         this.proxyConnectorAdmin = proxyConnectorAdmin;
  84     }
  85
  86     /**
  87      * @see RepositorySearch#search(String, List, String, SearchResultLimits, List)
  88      */
  89     public SearchResults search( String principal, List<String> selectedRepos, String term, SearchResultLimits limits,
  90                                  List<String> previousSearchTerms )
  91         throws RepositorySearchException
  92     {
  93         List<String> indexingContextIds = addIndexingContexts( selectedRepos );
  94
  95         // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]*
  96         //      resulting to more wildcard searches so we need to increase max clause count
  97         BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE );
  98         BooleanQuery q = new BooleanQuery();
  99
 100         if ( previousSearchTerms == null || previousSearchTerms.isEmpty() )
 101         {
 102             constructQuery( term, q );
 103         }
 104         else
 105         {
 106             for ( String previousTerm : previousSearchTerms )
 107             {
 108                 BooleanQuery iQuery = new BooleanQuery();
 109                 constructQuery( previousTerm, iQuery );
 110
 111                 q.add( iQuery, Occur.MUST );
 112             }
 113
 114             BooleanQuery iQuery = new BooleanQuery();
 115             constructQuery( term, iQuery );
 116             q.add( iQuery, Occur.MUST );
 117         }
 118
 119         // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty
 120         // FIXME  cannot find a way currently to setup this in constructQuery !!!
 121         return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFiler.LIST );
 122
 123     }
 124
 125     /**
 126      * @see RepositorySearch#search(String, SearchFields, SearchResultLimits)
 127      */
 128     public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits )
 129         throws RepositorySearchException
 130     {
 131         if ( searchFields.getRepositories() == null )
 132         {
 133             throw new RepositorySearchException( "Repositories cannot be null." );
 134         }
 135
 136         List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() );
 137
 138         BooleanQuery q = new BooleanQuery();
 139         if ( StringUtils.isNotBlank( searchFields.getGroupId() ) )
 140         {
 141             q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( searchFields.getGroupId() ) ),
 142                    Occur.MUST );
 143         }
 144
 145         if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) )
 146         {
 147             q.add(
 148                 indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( searchFields.getArtifactId() ) ),
 149                 Occur.MUST );
 150         }
 151
 152         if ( StringUtils.isNotBlank( searchFields.getVersion() ) )
 153         {
 154             q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( searchFields.getVersion() ) ),
 155                    Occur.MUST );
 156         }
 157
 158         if ( StringUtils.isNotBlank( searchFields.getPackaging() ) )
 159         {
 160             q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( searchFields.getPackaging() ) ),
 161                    Occur.MUST );
 162         }
 163
 164         if ( StringUtils.isNotBlank( searchFields.getClassName() ) )
 165         {
 166             q.add(
 167                 indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( searchFields.getClassName() ) ),
 168                 Occur.MUST );
 169         }
 170
 171         if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) )
 172         {
 173             q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME,
 174                                            new StringSearchExpression( searchFields.getBundleSymbolicName() ) ),
 175                    Occur.MUST );
 176         }
 177
 178         if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) )
 179         {
 180             q.add(
 181                 indexer.constructQuery( OSGI.VERSION, new StringSearchExpression( searchFields.getBundleVersion() ) ),
 182                 Occur.MUST );
 183         }
 184
 185         if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) )
 186         {
 187             q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE,
 188                                            new StringSearchExpression( searchFields.getBundleExportPackage() ) ),
 189                    Occur.MUST );
 190         }
 191
 192         if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) )
 193         {
 194             q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE,
 195                                            new StringSearchExpression( searchFields.getBundleExportService() ) ),
 196                    Occur.MUST );
 197         }
 198
 199         if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) )
 200         {
 201             q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE,
 202                                            new StringSearchExpression( searchFields.getBundleImportPackage() ) ),
 203                    Occur.MUST );
 204         }
 205
 206         if ( StringUtils.isNotBlank( searchFields.getBundleName() ) )
 207         {
 208             q.add( indexer.constructQuery( OSGI.NAME, new StringSearchExpression( searchFields.getBundleName() ) ),
 209                    Occur.MUST );
 210         }
 211
 212         if ( StringUtils.isNotBlank( searchFields.getClassifier() ) )
 213         {
 214             q.add(
 215                 indexer.constructQuery( MAVEN.CLASSIFIER, new StringSearchExpression( searchFields.getClassifier() ) ),
 216                 Occur.MUST );
 217         }
 218
 219         if ( q.getClauses() == null || q.getClauses().length <= 0 )
 220         {
 221             throw new RepositorySearchException( "No search fields set." );
 222         }
 223
 224         return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFiler>emptyList() );
 225     }
 226
 227     private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds,
 228                                   List<? extends ArtifactInfoFiler> filters )
 229         throws RepositorySearchException
 230     {
 231
 232         try
 233         {
 234             FlatSearchRequest request = new FlatSearchRequest( q );
 235             request.setContexts( getIndexingContexts( indexingContextIds ) );
 236             FlatSearchResponse response = indexer.searchFlat( request );
 237
 238             if ( response == null || response.getTotalHits() == 0 )
 239             {
 240                 SearchResults results = new SearchResults();
 241                 results.setLimits( limits );
 242                 return results;
 243             }
 244
 245             return convertToSearchResults( response, limits, filters );
 246         }
 247         catch ( IOException e )
 248         {
 249             throw new RepositorySearchException( e );
 250         }
 251
 252     }
 253
 254     private List<IndexingContext> getIndexingContexts( List<String> ids )
 255     {
 256         List<IndexingContext> contexts = new ArrayList<IndexingContext>( ids.size() );
 257
 258         for ( String id : ids )
 259         {
 260             IndexingContext context = indexer.getIndexingContexts().get( id );
 261             if ( context != null )
 262             {
 263                 contexts.add( context );
 264             }
 265             else
 266             {
 267                 log.warn( "context with id {} not exists", id );
 268             }
 269         }
 270
 271         return contexts;
 272     }
 273
 274     private void constructQuery( String term, BooleanQuery q )
 275     {
 276         q.add( indexer.constructQuery( MAVEN.GROUP_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
 277         q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new StringSearchExpression( term ) ), Occur.SHOULD );
 278         q.add( indexer.constructQuery( MAVEN.VERSION, new StringSearchExpression( term ) ), Occur.SHOULD );
 279         q.add( indexer.constructQuery( MAVEN.PACKAGING, new StringSearchExpression( term ) ), Occur.SHOULD );
 280         q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new StringSearchExpression( term ) ), Occur.SHOULD );
 281
 282         //Query query =
 283         //    new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) );
 284         //q.add( query, Occur.MUST_NOT );
 285         // olamy IMHO we could set this option as at least one must match
 286         //q.setMinimumNumberShouldMatch( 1 );
 287     }
 288
 289
 290     /**
 291      * @param selectedRepos
 292      * @return indexing contextId used
 293      */
 294     private List<String> addIndexingContexts( List<String> selectedRepos )
 295     {
 296         Set<String> indexingContextIds = new HashSet<String>();
 297         for ( String repo : selectedRepos )
 298         {
 299             try
 300             {
 301                 ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository( repo );
 302
 303                 if ( repoConfig != null )
 304                 {
 305                     String indexDir = repoConfig.getIndexDirectory();
 306                     File indexDirectory = null;
 307                     if ( indexDir != null && !"".equals( indexDir ) )
 308                     {
 309                         indexDirectory = new File( repoConfig.getIndexDirectory() );
 310                     }
 311                     else
 312                     {
 313                         indexDirectory = new File( repoConfig.getLocation(), ".indexer" );
 314                     }
 315
 316                     IndexingContext context = indexer.getIndexingContexts().get( repoConfig.getId() );
 317                     if ( context != null )
 318                     {
 319                         // alreday here so no need to record it again
 320                         log.debug( "index with id {} already exists skip adding it", repoConfig.getId() );
 321                         // set searchable flag
 322                         context.setSearchable( repoConfig.isScanned() );
 323                         indexingContextIds.add( context.getId() );
 324                         indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) );
 325                         continue;
 326                     }
 327
 328                     context = indexer.addIndexingContext( repoConfig.getId(), repoConfig.getId(),
 329                                                           new File( repoConfig.getLocation() ), indexDirectory, null,
 330                                                           null, getAllIndexCreators() );
 331                     context.setSearchable( repoConfig.isScanned() );
 332                     if ( context.isSearchable() )
 333                     {
 334                         indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) );
 335                         indexingContextIds.add( context.getId() );
 336                     }
 337                     else
 338                     {
 339                         log.warn( "indexingContext with id {} not searchable", repoConfig.getId() );
 340                     }
 341
 342                 }
 343                 else
 344                 {
 345                     log.warn( "Repository '" + repo + "' not found in configuration." );
 346                 }
 347             }
 348             catch ( UnsupportedExistingLuceneIndexException e )
 349             {
 350                 log.warn( "Error accessing index of repository '" + repo + "' : " + e.getMessage() );
 351                 continue;
 352             }
 353             catch ( IOException e )
 354             {
 355                 log.warn( "IO error occured while accessing index of repository '" + repo + "' : " + e.getMessage() );
 356                 continue;
 357             }
 358             catch ( RepositoryAdminException e )
 359             {
 360                 log.warn( "RepositoryAdminException occured while accessing index of repository '" + repo + "' : "
 361                               + e.getMessage() );
 362                 continue;
 363             }
 364         }
 365
 366         return new ArrayList<String>( indexingContextIds );
 367     }
 368
 369
 370     private Set<String> getRemoteIndexingContextIds( String managedRepoId )
 371         throws RepositoryAdminException
 372     {
 373         Set<String> ids = new HashSet<String>();
 374
 375         List<ProxyConnector> proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get( managedRepoId );
 376
 377         if ( proxyConnectors == null || proxyConnectors.isEmpty() )
 378         {
 379             return ids;
 380         }
 381
 382         for ( ProxyConnector proxyConnector : proxyConnectors )
 383         {
 384             String remoteId = "remote-" + proxyConnector.getTargetRepoId();
 385             IndexingContext context = indexer.getIndexingContexts().get( remoteId );
 386             if ( context != null && context.isSearchable() )
 387             {
 388                 ids.add( remoteId );
 389             }
 390         }
 391
 392         return ids;
 393     }
 394
 395
 396     protected List<? extends IndexCreator> getAllIndexCreators()
 397     {
 398         return mavenIndexerUtils.getAllIndexCreators();
 399     }
 400
 401
 402     private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits,
 403                                                   List<? extends ArtifactInfoFiler> artifactInfoFilers )
 404     {
 405         SearchResults results = new SearchResults();
 406         Set<ArtifactInfo> artifactInfos = response.getResults();
 407
 408         for ( ArtifactInfo artifactInfo : artifactInfos )
 409         {
 410             String id = SearchUtil.getHitId( artifactInfo.groupId, artifactInfo.artifactId, artifactInfo.classifier,
 411                                              artifactInfo.packaging );
 412             Map<String, SearchResultHit> hitsMap = results.getHitsMap();
 413
 414             if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilers, hitsMap ) )
 415             {
 416                 continue;
 417             }
 418
 419             SearchResultHit hit = hitsMap.get( id );
 420             if ( hit != null )
 421             {
 422                 if ( !hit.getVersions().contains( artifactInfo.version ) )
 423                 {
 424                     hit.addVersion( artifactInfo.version );
 425                 }
 426             }
 427             else
 428             {
 429                 hit = new SearchResultHit();
 430                 hit.setArtifactId( artifactInfo.artifactId );
 431                 hit.setGroupId( artifactInfo.groupId );
 432                 hit.setRepositoryId( artifactInfo.repository );
 433                 hit.addVersion( artifactInfo.version );
 434                 hit.setBundleExportPackage( artifactInfo.bundleExportPackage );
 435                 hit.setBundleExportService( artifactInfo.bundleExportService );
 436                 hit.setBundleSymbolicName( artifactInfo.bundleSymbolicName );
 437                 hit.setBundleVersion( artifactInfo.bundleVersion );
 438                 hit.setBundleDescription( artifactInfo.bundleDescription );
 439                 hit.setBundleDocUrl( artifactInfo.bundleDocUrl );
 440                 hit.setBundleRequireBundle( artifactInfo.bundleRequireBundle );
 441                 hit.setBundleImportPackage( artifactInfo.bundleImportPackage );
 442                 hit.setBundleLicense( artifactInfo.bundleLicense );
 443                 hit.setBundleName( artifactInfo.bundleName );
 444                 hit.setContext( artifactInfo.context );
 445                 hit.setGoals( artifactInfo.goals );
 446                 hit.setPrefix( artifactInfo.prefix );
 447                 hit.setPackaging( artifactInfo.packaging );
 448                 hit.setClassifier( artifactInfo.classifier );
 449                 hit.setUrl( getBaseUrl( artifactInfo ) );
 450             }
 451
 452             results.addHit( id, hit );
 453         }
 454
 455         results.setTotalHits( response.getTotalHitsCount() );
 456         results.setReturnedHitsCount( response.getReturnedHitsCount() );
 457         results.setLimits( limits );
 458
 459         if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES )
 460         {
 461             return results;
 462         }
 463         else
 464         {
 465             return paginate( results );
 466         }
 467     }
 468
 469     /**
 470      * calculate baseUrl without the context and base Archiva Url
 471      * @param artifactInfo
 472      * @return
 473      */
 474     protected String getBaseUrl( ArtifactInfo artifactInfo )
 475     {
 476         StringBuilder sb = new StringBuilder( );
 477
 478         sb.append( '/' ).append( StringUtils.replaceChars( artifactInfo.groupId, '.', '/' ) );
 479         sb.append( '/' ).append( artifactInfo.artifactId );
 480         sb.append( '/' ).append( artifactInfo.version );
 481         sb.append( '/' ).append( artifactInfo.artifactId );
 482         sb.append( '-' ).append( artifactInfo.version );
 483         if ( StringUtils.isNotBlank( artifactInfo.classifier ) )
 484         {
 485             sb.append( '-' ).append( artifactInfo.classifier );
 486         }
 487         sb.append( '.' ).append( artifactInfo.packaging );
 488
 489         return sb.toString();
 490     }
 491
 492     private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo,
 493                                               List<? extends ArtifactInfoFiler> artifactInfoFilers,
 494                                               Map<String, SearchResultHit> currentResult )
 495     {
 496         if ( artifactInfoFilers == null || artifactInfoFilers.isEmpty() )
 497         {
 498             return true;
 499         }
 500
 501         for ( ArtifactInfoFiler filter : artifactInfoFilers )
 502         {
 503             if ( !filter.addArtifactInResult( artifactInfo, currentResult ) )
 504             {
 505                 return false;
 506             }
 507         }
 508         return true;
 509     }
 510
 511     private SearchResults paginate( SearchResults results )
 512     {
 513         SearchResultLimits limits = results.getLimits();
 514         SearchResults paginated = new SearchResults();
 515
 516         int fetchCount = limits.getPageSize();
 517         int offset = ( limits.getSelectedPage() * limits.getPageSize() );
 518
 519         if ( fetchCount > results.getTotalHits() )
 520         {
 521             fetchCount = results.getTotalHits();
 522         }
 523
 524         // Goto offset.
 525         if ( offset < results.getTotalHits() )
 526         {
 527             // only process if the offset is within the hit count.
 528             for ( int i = 0; i < fetchCount; i++ )
 529             {
 530                 // Stop fetching if we are past the total # of available hits.
 531                 if ( offset + i >= results.getHits().size() )
 532                 {
 533                     break;
 534                 }
 535
 536                 SearchResultHit hit = results.getHits().get( ( offset + i ) );
 537                 if ( hit != null )
 538                 {
 539                     String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(),
 540                                                      hit.getPackaging() );
 541                     paginated.addHit( id, hit );
 542                 }
 543                 else
 544                 {
 545                     break;
 546                 }
 547             }
 548         }
 549         paginated.setTotalHits( results.getTotalHits() );
 550         paginated.setReturnedHitsCount( paginated.getHits().size() );
 551         paginated.setLimits( limits );
 552
 553         return paginated;
 554     }
 555 }