Implementation of new archivaindexer Not finished yet, but uses the maven index service via spring dependency injection Moving some maven specific classes to subpackages
Project: http://git-wip-us.apache.org/repos/asf/archiva/repo Commit: http://git-wip-us.apache.org/repos/asf/archiva/commit/c6e4e563 Tree: http://git-wip-us.apache.org/repos/asf/archiva/tree/c6e4e563 Diff: http://git-wip-us.apache.org/repos/asf/archiva/diff/c6e4e563 Branch: refs/heads/master Commit: c6e4e563d6698aa7cb5772878bc0fe22066b7884 Parents: 8f52099 Author: Martin Stockhammer <martin.stockham...@ars.de> Authored: Sat Nov 11 00:07:56 2017 +0100 Committer: Martin Stockhammer <martin.stockham...@ars.de> Committed: Sat Nov 11 00:07:56 2017 +0100 ---------------------------------------------------------------------- .../configuration/ArchivaConfiguration.java | 3 + .../DefaultArchivaConfiguration.java | 10 + .../archiva-base/archiva-maven2-indexer/pom.xml | 33 +- .../maven/DefaultIndexUpdateSideEffect.java | 47 + .../indexer/maven/MavenIndexContext.java | 122 +++ .../indexer/maven/MavenIndexManager.java | 171 ++++ .../maven/merger/DefaultIndexMerger.java | 177 ++++ .../maven/search/MavenRepositorySearch.java | 738 +++++++++++++++ .../indexer/merger/DefaultIndexMerger.java | 173 ---- .../indexer/search/MavenRepositorySearch.java | 737 --------------- .../main/resources/META-INF/spring-context.xml | 12 +- .../search/AbstractMavenRepositorySearch.java | 306 ++++++ .../search/MavenRepositorySearchOSGITest.java | 81 ++ .../MavenRepositorySearchPaginateTest.java | 109 +++ .../maven/search/MavenRepositorySearchTest.java | 933 ++++++++++++++++++ .../search/AbstractMavenRepositorySearch.java | 303 ------ .../search/MavenRepositorySearchOSGITest.java | 78 -- .../MavenRepositorySearchPaginateTest.java | 105 --- .../search/MavenRepositorySearchTest.java | 937 ------------------- .../src/test/resources/spring-context.xml | 1 + .../apache/archiva/proxy/MockConfiguration.java | 24 +- .../archiva/indexer/ArchivaIndexManager.java | 13 +- .../archiva/indexer/ArchivaIndexingContext.java | 7 +- .../archiva-webapp/src/test/log4j2-test.xml | 2 +- .../configuration/TestConfiguration.java | 11 + .../storage/maven2/conf/MockConfiguration.java | 11 + .../configuration/StubConfiguration.java | 11 + 27 files changed, 2786 insertions(+), 2369 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaConfiguration.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaConfiguration.java b/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaConfiguration.java index 02d2b13..acda2b7 100644 --- a/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaConfiguration.java +++ b/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/ArchivaConfiguration.java @@ -22,6 +22,7 @@ package org.apache.archiva.configuration; import org.apache.archiva.redback.components.registry.RegistryException; import org.apache.archiva.redback.components.registry.RegistryListener; +import java.nio.file.Path; import java.util.List; import java.util.Locale; @@ -92,5 +93,7 @@ public interface ArchivaConfiguration public Locale getDefaultLocale(); public List<Locale.LanguageRange> getLanguagePriorities(); + + public Path getAppServerBaseDir(); } http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/DefaultArchivaConfiguration.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/DefaultArchivaConfiguration.java b/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/DefaultArchivaConfiguration.java index 6983be0..0222a42 100644 --- a/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/DefaultArchivaConfiguration.java +++ b/archiva-modules/archiva-base/archiva-configuration/src/main/java/org/apache/archiva/configuration/DefaultArchivaConfiguration.java @@ -892,6 +892,16 @@ public class DefaultArchivaConfiguration } @Override + public Path getAppServerBaseDir() { + String basePath = registry.getString("appserver.base"); + if (!StringUtils.isEmpty(basePath)) { + return Paths.get(basePath); + } else { + return Paths.get(""); + } + } + + @Override public void beforeConfigurationChange( Registry registry, String propertyName, Object propertyValue ) { // nothing to do here http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml b/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml index f0a90a5..bc141b6 100644 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/pom.xml @@ -124,11 +124,6 @@ </dependency> <dependency> <groupId>org.apache.archiva</groupId> - <artifactId>archiva-configuration</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.archiva</groupId> <artifactId>archiva-repository-admin-default</artifactId> <scope>test</scope> </dependency> @@ -183,7 +178,35 @@ <artifactId>hsqldb</artifactId> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.archiva</groupId> + <artifactId>archiva-configuration</artifactId> + </dependency> + <dependency> + <groupId>org.apache.maven.indexer</groupId> + <artifactId>indexer-core</artifactId> + <classifier>shaded-lucene</classifier> + </dependency> + <dependency> + <groupId>org.apache.maven</groupId> + <artifactId>maven-core</artifactId> + <exclusions> + <exclusion> + <groupId>org.sonatype.sisu</groupId> + <artifactId>sisu-guava</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.sisu</groupId> + <artifactId>sisu-inject</artifactId> + </exclusion> + <exclusion> + <groupId>org.sonatype.sisu</groupId> + <artifactId>sisu-guice</artifactId> + </exclusion> + </exclusions> + </dependency> </dependencies> + <build> <pluginManagement> <plugins> http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/DefaultIndexUpdateSideEffect.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/DefaultIndexUpdateSideEffect.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/DefaultIndexUpdateSideEffect.java new file mode 100644 index 0000000..cb66cbb --- /dev/null +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/DefaultIndexUpdateSideEffect.java @@ -0,0 +1,47 @@ +package org.apache.archiva.indexer.maven; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.updater.IndexUpdateSideEffect; +import org.apache.maven.index_shaded.lucene.store.Directory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +/** + * Not doing much but required at least one implementation + * + * @since 3.0.0 + */ +@Service("archivaIndexUpdater") +public class DefaultIndexUpdateSideEffect + implements IndexUpdateSideEffect +{ + private static final Logger LOGGER = LoggerFactory.getLogger( DefaultIndexUpdateSideEffect.class ); + + @Override + public void updateIndex( Directory directory, IndexingContext indexingContext, boolean b ) + { + LOGGER.info( "updating index: {} with directory: {}", // + indexingContext.getId(), // + directory.toString() ); + } +} http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexContext.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexContext.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexContext.java new file mode 100644 index 0000000..30cc727 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexContext.java @@ -0,0 +1,122 @@ +package org.apache.archiva.indexer.maven; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.indexer.ArchivaIndexingContext; +import org.apache.archiva.repository.Repository; +import org.apache.maven.index.context.IndexingContext; + +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.sql.Date; +import java.time.ZonedDateTime; +import java.util.Set; + +/** + * Maven implementation of index context + */ +public class MavenIndexContext implements ArchivaIndexingContext { + + private IndexingContext delegate; + private Repository repository; + + MavenIndexContext(Repository repository, IndexingContext delegate) { + this.delegate = delegate; + this.repository = repository; + + } + + @Override + public String getId() { + return delegate.getId(); + } + + @Override + public Repository getRepository() { + return repository; + } + + @Override + public URI getPath() { + return delegate.getIndexDirectoryFile().toURI(); + } + + @Override + public boolean isEmpty() throws IOException { + return Files.list(delegate.getIndexDirectoryFile().toPath()).count()==0; + } + + @Override + public void commit() throws IOException { + delegate.commit(); + } + + @Override + public void rollback() throws IOException { + delegate.rollback(); + } + + @Override + public void optimize() throws IOException { + delegate.optimize(); + } + + @Override + public void close(boolean deleteFiles) throws IOException { + delegate.close(deleteFiles); + } + + @Override + public void purge() throws IOException { + delegate.purge(); + } + + @Override + public boolean supports(Class<?> clazz) { + return IndexingContext.class.equals(clazz); + } + + @Override + public <T> T getBaseContext(Class<T> clazz) throws UnsupportedOperationException { + if (IndexingContext.class.equals(clazz)) { + return (T) delegate; + } else { + throw new UnsupportedOperationException("The class "+clazz+" is not supported by the maven indexer"); + } + } + + @Override + public Set<String> getGroups() throws IOException { + return delegate.getAllGroups(); + } + + @Override + public void updateTimestamp(boolean save) throws IOException { + delegate.updateTimestamp(save); + } + + @Override + public void updateTimestamp(boolean save, ZonedDateTime time) throws IOException { + delegate.updateTimestamp(save, Date.from(time.toInstant())); + } + + +} http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java new file mode 100644 index 0000000..12d8586 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/MavenIndexManager.java @@ -0,0 +1,171 @@ +package org.apache.archiva.indexer.maven; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.common.utils.PathUtil; +import org.apache.archiva.configuration.ArchivaConfiguration; +import org.apache.archiva.indexer.ArchivaIndexManager; +import org.apache.archiva.indexer.ArchivaIndexingContext; +import org.apache.archiva.model.ArtifactReference; +import org.apache.archiva.repository.ManagedRepository; +import org.apache.archiva.repository.RemoteRepository; +import org.apache.archiva.repository.Repository; +import org.apache.archiva.repository.RepositoryType; +import org.apache.archiva.repository.features.RemoteIndexFeature; +import org.apache.maven.index.Indexer; +import org.apache.maven.index.context.IndexCreator; +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index_shaded.lucene.index.IndexFormatTooOldException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.io.IOException; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** + * Maven implementation of index manager + */ +@Service("archivaIndexManager#maven") +public class MavenIndexManager implements ArchivaIndexManager { + + private static final Logger log = LoggerFactory.getLogger(MavenIndexManager.class); + + @Inject + private Indexer indexer; + + @Inject + private List<? extends IndexCreator> indexCreators; + + @Inject + private ArchivaConfiguration archivaConfiguration; + + @Override + public void pack(ArchivaIndexingContext context) { + + } + + @Override + public void scan(ArchivaIndexingContext context, boolean update) { + + } + + @Override + public void update(ArchivaIndexingContext context, URI remoteUpdateUri, boolean fullUpdate) { + + } + + @Override + public void addArtifactToIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) { + + } + + @Override + public void removeArtifactFromIndex(ArchivaIndexingContext context, ArtifactReference artifactReference) { + + } + + @Override + public boolean supportsRepository(RepositoryType type) { + return false; + } + + @Override + public ArchivaIndexingContext createContext(Repository remoteRepository) throws IOException { + IndexingContext mvnCtx = null; + if (remoteRepository instanceof RemoteRepository) { + mvnCtx = createRemoteContext((RemoteRepository) remoteRepository); + } else if (remoteRepository instanceof ManagedRepository) { + // TODO: Implement managed repository index creation + mvnCtx = null; + } + MavenIndexContext context = new MavenIndexContext(remoteRepository, mvnCtx); + return null; + } + + private IndexingContext createRemoteContext(RemoteRepository remoteRepository) throws IOException { + Path appServerBase = archivaConfiguration.getAppServerBaseDir(); + + String contextKey = "remote-" + remoteRepository.getId(); + + // create remote repository path + Path repoDir = appServerBase.resolve( "data").resolve( "remotes" ).resolve( remoteRepository.getId() ); + if ( !Files.exists(repoDir) ) + { + Files.createDirectories(repoDir); + } + + Path indexDirectory = null; + + // is there configured indexDirectory ? + if (remoteRepository.supportsFeature(RemoteIndexFeature.class)) { + RemoteIndexFeature rif = remoteRepository.getFeature(RemoteIndexFeature.class).get(); + indexDirectory = PathUtil.getPathFromUri(rif.getIndexUri()); + if (!indexDirectory.isAbsolute()) { + indexDirectory = repoDir.resolve(indexDirectory); + } + + // if not configured use a default value + if (indexDirectory == null) { + indexDirectory = repoDir.resolve(".index"); + } + if (!Files.exists(indexDirectory)) { + Files.createDirectories(indexDirectory); + } + + try { + + return indexer.createIndexingContext(contextKey, remoteRepository.getId(), repoDir.toFile(), indexDirectory.toFile(), + remoteRepository.getLocation() == null ? null : remoteRepository.getLocation().toString(), + calculateIndexRemoteUrl(remoteRepository.getLocation(), rif), + true, false, + indexCreators); + } catch (IndexFormatTooOldException e) { + // existing index with an old lucene format so we need to delete it!!! + // delete it first then recreate it. + log.warn("the index of repository {} is too old we have to delete and recreate it", // + remoteRepository.getId()); + org.apache.archiva.common.utils.FileUtils.deleteDirectory(indexDirectory); + return indexer.createIndexingContext(contextKey, remoteRepository.getId(), repoDir.toFile(), indexDirectory.toFile(), + remoteRepository.getLocation() == null ? null : remoteRepository.getLocation().toString(), + calculateIndexRemoteUrl(remoteRepository.getLocation(), rif), + true, false, + indexCreators); + + } + } else { + throw new IOException("No remote index defined"); + } + } + + private String calculateIndexRemoteUrl(URI baseUri, RemoteIndexFeature rif) { + if (rif.getIndexUri()==null) { + return baseUri.resolve(".index").toString(); + } else { + return baseUri.resolve(rif.getIndexUri()).toString(); + } + } + + +} http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java new file mode 100644 index 0000000..0e0cd90 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/merger/DefaultIndexMerger.java @@ -0,0 +1,177 @@ +package org.apache.archiva.indexer.maven.merger; +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.common.utils.FileUtils; +import org.apache.archiva.indexer.merger.IndexMerger; +import org.apache.archiva.indexer.merger.IndexMergerException; +import org.apache.archiva.indexer.merger.IndexMergerRequest; +import org.apache.archiva.indexer.merger.TemporaryGroupIndex; +import org.apache.commons.lang.time.StopWatch; +import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.context.IndexCreator; +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; +import org.apache.maven.index.packer.IndexPacker; +import org.apache.maven.index.packer.IndexPackingRequest; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * @author Olivier Lamy + * @since 1.4-M2 + */ +@Service("indexMerger#default") +public class DefaultIndexMerger + implements IndexMerger +{ + + private Logger log = LoggerFactory.getLogger( getClass() ); + + private final NexusIndexer indexer; + + private final IndexPacker indexPacker; + + private final List<IndexCreator> indexCreators; + + private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<>(); + + private List<String> runningGroups = new CopyOnWriteArrayList<>(); + + @Inject + public DefaultIndexMerger( NexusIndexer nexusIndexer, IndexPacker indexPacker, List<IndexCreator> indexCreators ) + { + this.indexer = nexusIndexer; + this.indexPacker = indexPacker; + this.indexCreators = indexCreators; + } + + @Override + public IndexingContext buildMergedIndex( IndexMergerRequest indexMergerRequest ) + throws IndexMergerException + { + String groupId = indexMergerRequest.getGroupId(); + + if ( runningGroups.contains( groupId ) ) + { + log.info( "skip build merge remote indexes for id: '{}' as already running", groupId ); + return null; + } + + runningGroups.add( groupId ); + + StopWatch stopWatch = new StopWatch(); + stopWatch.reset(); + stopWatch.start(); + + Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory(); + + String tempRepoId = mergedIndexDirectory.getFileName().toString(); + + try + { + Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() ); + IndexingContext indexingContext = + indexer.addIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), indexLocation.toFile(), null, null, + indexCreators ); + + for ( String repoId : indexMergerRequest.getRepositoriesIds() ) + { + IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId ); + if ( idxToMerge != null ) + { + indexingContext.merge( idxToMerge.getIndexDirectory() ); + } + } + + indexingContext.optimize(); + + if ( indexMergerRequest.isPackIndex() ) + { + IndexPackingRequest request = new IndexPackingRequest( indexingContext, // + indexingContext.acquireIndexSearcher().getIndexReader(), // + indexLocation.toFile() ); + indexPacker.packIndex( request ); + } + + if ( indexMergerRequest.isTemporary() ) + { + temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId, + indexMergerRequest.getMergedIndexTtl() ) ); + } + stopWatch.stop(); + log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(), + stopWatch.getTime() ); + return indexingContext; + } + catch ( IOException | UnsupportedExistingLuceneIndexException e ) + { + throw new IndexMergerException( e.getMessage(), e ); + } + finally + { + runningGroups.remove( groupId ); + } + } + + @Async + @Override + public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ) + { + if ( temporaryGroupIndex == null ) + { + return; + } + + try + { + IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); + if ( indexingContext != null ) + { + indexer.removeIndexingContext( indexingContext, true ); + } + Path directory = temporaryGroupIndex.getDirectory(); + if ( directory != null && Files.exists(directory) ) + { + FileUtils.deleteDirectory( directory ); + } + temporaryGroupIndexes.remove( temporaryGroupIndex ); + } + catch ( IOException e ) + { + log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e ); + } + } + + @Override + public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes() + { + return this.temporaryGroupIndexes; + } +} http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java new file mode 100644 index 0000000..590cdd2 --- /dev/null +++ b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/maven/search/MavenRepositorySearch.java @@ -0,0 +1,738 @@ +package org.apache.archiva.indexer.maven.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.archiva.admin.model.RepositoryAdminException; +import org.apache.archiva.admin.model.beans.ManagedRepository; +import org.apache.archiva.admin.model.beans.ProxyConnector; +import org.apache.archiva.admin.model.managed.ManagedRepositoryAdmin; +import org.apache.archiva.admin.model.proxyconnector.ProxyConnectorAdmin; +import org.apache.archiva.common.plexusbridge.PlexusSisuBridgeException; +import org.apache.archiva.indexer.search.*; +import org.apache.archiva.indexer.util.SearchUtil; +import org.apache.archiva.model.ArchivaArtifactModel; +import org.apache.archiva.model.ArtifactReference; +import org.apache.commons.lang.StringUtils; +import org.apache.maven.index.ArtifactInfo; +import org.apache.maven.index.FlatSearchRequest; +import org.apache.maven.index.FlatSearchResponse; +import org.apache.maven.index.MAVEN; +import org.apache.maven.index.NexusIndexer; +import org.apache.maven.index.OSGI; +import org.apache.maven.index.QueryCreator; +import org.apache.maven.index.SearchType; +import org.apache.maven.index.context.IndexingContext; +import org.apache.maven.index.expr.SearchExpression; +import org.apache.maven.index.expr.SearchTyped; +import org.apache.maven.index.expr.SourcedSearchExpression; +import org.apache.maven.index.expr.UserInputSearchExpression; +import org.apache.maven.index_shaded.lucene.search.BooleanClause; +import org.apache.maven.index_shaded.lucene.search.BooleanClause.Occur; +import org.apache.maven.index_shaded.lucene.search.BooleanQuery; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * RepositorySearch implementation which uses the Maven Indexer for searching. + */ +@Service( "repositorySearch#maven" ) +public class MavenRepositorySearch + implements RepositorySearch +{ + private Logger log = LoggerFactory.getLogger( getClass() ); + + private NexusIndexer indexer; + + private QueryCreator queryCreator; + + private ManagedRepositoryAdmin managedRepositoryAdmin; + + private ProxyConnectorAdmin proxyConnectorAdmin; + + protected MavenRepositorySearch() + { + // for test purpose + } + + @Inject + public MavenRepositorySearch( NexusIndexer nexusIndexer, ManagedRepositoryAdmin managedRepositoryAdmin, + ProxyConnectorAdmin proxyConnectorAdmin, QueryCreator queryCreator ) + throws PlexusSisuBridgeException + { + this.indexer = nexusIndexer; + this.queryCreator = queryCreator; + this.managedRepositoryAdmin = managedRepositoryAdmin; + this.proxyConnectorAdmin = proxyConnectorAdmin; + } + + /** + * @see RepositorySearch#search(String, List, String, SearchResultLimits, List) + */ + @Override + public SearchResults search(String principal, List<String> selectedRepos, String term, SearchResultLimits limits, + List<String> previousSearchTerms ) + throws RepositorySearchException + { + List<String> indexingContextIds = addIndexingContexts( selectedRepos ); + + // since upgrade to nexus 2.0.0, query has changed from g:[QUERIED TERM]* to g:*[QUERIED TERM]* + // resulting to more wildcard searches so we need to increase max clause count + BooleanQuery.setMaxClauseCount( Integer.MAX_VALUE ); + BooleanQuery q = new BooleanQuery(); + + if ( previousSearchTerms == null || previousSearchTerms.isEmpty() ) + { + constructQuery( term, q ); + } + else + { + for ( String previousTerm : previousSearchTerms ) + { + BooleanQuery iQuery = new BooleanQuery(); + constructQuery( previousTerm, iQuery ); + + q.add( iQuery, BooleanClause.Occur.MUST ); + } + + BooleanQuery iQuery = new BooleanQuery(); + constructQuery( term, iQuery ); + q.add( iQuery, BooleanClause.Occur.MUST ); + } + + // we retun only artifacts without classifier in quick search, olamy cannot find a way to say with this field empty + // FIXME cannot find a way currently to setup this in constructQuery !!! + return search( limits, q, indexingContextIds, NoClassifierArtifactInfoFilter.LIST, selectedRepos, true ); + + } + + /** + * @see RepositorySearch#search(String, SearchFields, SearchResultLimits) + */ + @Override + public SearchResults search( String principal, SearchFields searchFields, SearchResultLimits limits ) + throws RepositorySearchException + { + if ( searchFields.getRepositories() == null ) + { + throw new RepositorySearchException( "Repositories cannot be null." ); + } + + List<String> indexingContextIds = addIndexingContexts( searchFields.getRepositories() ); + + // if no index found in the specified ones return an empty search result instead of doing a search on all index + // olamy: IMHO doesn't make sense + if ( !searchFields.getRepositories().isEmpty() && ( indexingContextIds == null + || indexingContextIds.isEmpty() ) ) + { + return new SearchResults(); + } + + BooleanQuery q = new BooleanQuery(); + if ( StringUtils.isNotBlank( searchFields.getGroupId() ) ) + { + q.add( indexer.constructQuery( MAVEN.GROUP_ID, searchFields.isExactSearch() ? new SourcedSearchExpression( + searchFields.getGroupId() ) : new UserInputSearchExpression( searchFields.getGroupId() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getArtifactId() ) ) + { + q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, + searchFields.isExactSearch() + ? new SourcedSearchExpression( searchFields.getArtifactId() ) + : new UserInputSearchExpression( searchFields.getArtifactId() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getVersion() ) ) + { + q.add( indexer.constructQuery( MAVEN.VERSION, searchFields.isExactSearch() ? new SourcedSearchExpression( + searchFields.getVersion() ) : new SourcedSearchExpression( searchFields.getVersion() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getPackaging() ) ) + { + q.add( indexer.constructQuery( MAVEN.PACKAGING, searchFields.isExactSearch() ? new SourcedSearchExpression( + searchFields.getPackaging() ) : new UserInputSearchExpression( searchFields.getPackaging() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getClassName() ) ) + { + q.add( indexer.constructQuery( MAVEN.CLASSNAMES, + new UserInputSearchExpression( searchFields.getClassName() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleSymbolicName() ) ) + { + q.add( indexer.constructQuery( OSGI.SYMBOLIC_NAME, + new UserInputSearchExpression( searchFields.getBundleSymbolicName() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleVersion() ) ) + { + q.add( indexer.constructQuery( OSGI.VERSION, + new UserInputSearchExpression( searchFields.getBundleVersion() ) ), + BooleanClause.Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleExportPackage() ) ) + { + q.add( indexer.constructQuery( OSGI.EXPORT_PACKAGE, + new UserInputSearchExpression( searchFields.getBundleExportPackage() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleExportService() ) ) + { + q.add( indexer.constructQuery( OSGI.EXPORT_SERVICE, + new UserInputSearchExpression( searchFields.getBundleExportService() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) ) + { + q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, + new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleName() ) ) + { + q.add( indexer.constructQuery( OSGI.NAME, new UserInputSearchExpression( searchFields.getBundleName() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleImportPackage() ) ) + { + q.add( indexer.constructQuery( OSGI.IMPORT_PACKAGE, + new UserInputSearchExpression( searchFields.getBundleImportPackage() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getBundleRequireBundle() ) ) + { + q.add( indexer.constructQuery( OSGI.REQUIRE_BUNDLE, + new UserInputSearchExpression( searchFields.getBundleRequireBundle() ) ), + Occur.MUST ); + } + + if ( StringUtils.isNotBlank( searchFields.getClassifier() ) ) + { + q.add( indexer.constructQuery( MAVEN.CLASSIFIER, searchFields.isExactSearch() ? new SourcedSearchExpression( + searchFields.getClassifier() ) : new UserInputSearchExpression( searchFields.getClassifier() ) ), + Occur.MUST ); + } + else if ( searchFields.isExactSearch() ) + { + //TODO improvement in case of exact search and no classifier we must query for classifier with null value + // currently it's done in DefaultSearchService with some filtering + } + + if ( q.getClauses() == null || q.getClauses().length <= 0 ) + { + throw new RepositorySearchException( "No search fields set." ); + } + System.err.println("CLAUSES "+q.getClauses()); + if (q.getClauses()!=null) { + for (BooleanClause cl : q.getClauses()) { + System.err.println("Clause "+cl); + } + } + + return search( limits, q, indexingContextIds, Collections.<ArtifactInfoFilter>emptyList(), + searchFields.getRepositories(), searchFields.isIncludePomArtifacts() ); + } + + private static class NullSearch + implements SearchTyped, SearchExpression + { + private static final NullSearch INSTANCE = new NullSearch(); + + @Override + public String getStringValue() + { + return "[[NULL_VALUE]]"; + } + + @Override + public SearchType getSearchType() + { + return SearchType.EXACT; + } + } + + private SearchResults search( SearchResultLimits limits, BooleanQuery q, List<String> indexingContextIds, + List<? extends ArtifactInfoFilter> filters, List<String> selectedRepos, + boolean includePoms ) + throws RepositorySearchException + { + + try + { + FlatSearchRequest request = new FlatSearchRequest( q ); + + request.setContexts( getIndexingContexts( indexingContextIds ) ); + if ( limits != null ) + { + // we apply limits only when first page asked + if ( limits.getSelectedPage() == 0 ) + { + request.setCount( limits.getPageSize() * ( Math.max( 1, limits.getSelectedPage() ) ) ); + } + } + + FlatSearchResponse response = indexer.searchFlat( request ); + + if ( response == null || response.getTotalHits() == 0 ) + { + SearchResults results = new SearchResults(); + results.setLimits( limits ); + return results; + } + + return convertToSearchResults( response, limits, filters, selectedRepos, includePoms ); + } + catch ( IOException e ) + { + throw new RepositorySearchException( e.getMessage(), e ); + } + catch ( RepositoryAdminException e ) + { + throw new RepositorySearchException( e.getMessage(), e ); + } + + } + + private List<IndexingContext> getIndexingContexts( List<String> ids ) + { + List<IndexingContext> contexts = new ArrayList<>( ids.size() ); + + for ( String id : ids ) + { + IndexingContext context = indexer.getIndexingContexts().get( id ); + if ( context != null ) + { + contexts.add( context ); + } + else + { + log.warn( "context with id {} not exists", id ); + } + } + + return contexts; + } + + private void constructQuery( String term, BooleanQuery q ) + { + q.add( indexer.constructQuery( MAVEN.GROUP_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD ); + q.add( indexer.constructQuery( MAVEN.ARTIFACT_ID, new UserInputSearchExpression( term ) ), Occur.SHOULD ); + q.add( indexer.constructQuery( MAVEN.VERSION, new UserInputSearchExpression( term ) ), Occur.SHOULD ); + q.add( indexer.constructQuery( MAVEN.PACKAGING, new UserInputSearchExpression( term ) ), Occur.SHOULD ); + q.add( indexer.constructQuery( MAVEN.CLASSNAMES, new UserInputSearchExpression( term ) ), Occur.SHOULD ); + + //Query query = + // new WildcardQuery( new Term( MAVEN.CLASSNAMES.getFieldName(), "*" ) ); + //q.add( query, Occur.MUST_NOT ); + // olamy IMHO we could set this option as at least one must match + //q.setMinimumNumberShouldMatch( 1 ); + } + + + /** + * @param selectedRepos + * @return indexing contextId used + */ + private List<String> addIndexingContexts( List<String> selectedRepos ) + { + Set<String> indexingContextIds = new HashSet<>(); + for ( String repo : selectedRepos ) + { + try + { + ManagedRepository repoConfig = managedRepositoryAdmin.getManagedRepository( repo ); + + if ( repoConfig != null ) + { + + IndexingContext context = managedRepositoryAdmin.createIndexContext( repoConfig ); + if ( context.isSearchable() ) + { + indexingContextIds.addAll( getRemoteIndexingContextIds( repo ) ); + indexingContextIds.add( context.getId() ); + } + else + { + log.warn( "indexingContext with id {} not searchable", repoConfig.getId() ); + } + + } + else + { + log.warn( "Repository '{}' not found in configuration.", repo ); + } + } + catch ( RepositoryAdminException e ) + { + log.warn( "RepositoryAdminException occured while accessing index of repository '{}' : {}", repo, + e.getMessage() ); + continue; + } + catch ( RepositorySearchException e ) + { + log.warn( "RepositorySearchException occured while accessing index of repository '{}' : {}", repo, + e.getMessage() ); + continue; + } + } + + return new ArrayList<>( indexingContextIds ); + } + + + @Override + public Set<String> getRemoteIndexingContextIds( String managedRepoId ) + throws RepositorySearchException + { + Set<String> ids = new HashSet<>(); + + List<ProxyConnector> proxyConnectors = null; + try + { + proxyConnectors = proxyConnectorAdmin.getProxyConnectorAsMap().get( managedRepoId ); + } + catch ( RepositoryAdminException e ) + { + throw new RepositorySearchException( e.getMessage(), e ); + } + + if ( proxyConnectors == null || proxyConnectors.isEmpty() ) + { + return ids; + } + + for ( ProxyConnector proxyConnector : proxyConnectors ) + { + String remoteId = "remote-" + proxyConnector.getTargetRepoId(); + IndexingContext context = indexer.getIndexingContexts().get( remoteId ); + if ( context != null && context.isSearchable() ) + { + ids.add( remoteId ); + } + } + + return ids; + } + + @Override + public Collection<String> getAllGroupIds( String principal, List<String> selectedRepos ) + throws RepositorySearchException + { + List<IndexingContext> indexContexts = getIndexingContexts( selectedRepos ); + + if ( indexContexts == null || indexContexts.isEmpty() ) + { + return Collections.emptyList(); + } + + try + { + Set<String> allGroupIds = new HashSet<>(); + for ( IndexingContext indexingContext : indexContexts ) + { + allGroupIds.addAll( indexingContext.getAllGroups() ); + } + return allGroupIds; + } + catch ( IOException e ) + { + throw new RepositorySearchException( e.getMessage(), e ); + } + + } + + private SearchResults convertToSearchResults( FlatSearchResponse response, SearchResultLimits limits, + List<? extends ArtifactInfoFilter> artifactInfoFilters, + List<String> selectedRepos, boolean includePoms ) + throws RepositoryAdminException + { + SearchResults results = new SearchResults(); + Set<ArtifactInfo> artifactInfos = response.getResults(); + + for ( ArtifactInfo artifactInfo : artifactInfos ) + { + if ( StringUtils.equalsIgnoreCase( "pom", artifactInfo.getFileExtension() ) && !includePoms ) + { + continue; + } + String id = SearchUtil.getHitId( artifactInfo.getGroupId(), // + artifactInfo.getArtifactId(), // + artifactInfo.getClassifier(), // + artifactInfo.getPackaging() ); + Map<String, SearchResultHit> hitsMap = results.getHitsMap(); + + + if ( !applyArtifactInfoFilters( artifactInfo, artifactInfoFilters, hitsMap ) ) + { + continue; + } + + SearchResultHit hit = hitsMap.get( id ); + if ( hit != null ) + { + if ( !hit.getVersions().contains( artifactInfo.getVersion() ) ) + { + hit.addVersion( artifactInfo.getVersion() ); + } + } + else + { + hit = new SearchResultHit(); + hit.setArtifactId( artifactInfo.getArtifactId() ); + hit.setGroupId( artifactInfo.getGroupId() ); + hit.setRepositoryId( artifactInfo.getRepository() ); + hit.addVersion( artifactInfo.getVersion() ); + hit.setBundleExportPackage( artifactInfo.getBundleExportPackage() ); + hit.setBundleExportService( artifactInfo.getBundleExportService() ); + hit.setBundleSymbolicName( artifactInfo.getBundleSymbolicName() ); + hit.setBundleVersion( artifactInfo.getBundleVersion() ); + hit.setBundleDescription( artifactInfo.getBundleDescription() ); + hit.setBundleDocUrl( artifactInfo.getBundleDocUrl() ); + hit.setBundleRequireBundle( artifactInfo.getBundleRequireBundle() ); + hit.setBundleImportPackage( artifactInfo.getBundleImportPackage() ); + hit.setBundleLicense( artifactInfo.getBundleLicense() ); + hit.setBundleName( artifactInfo.getBundleName() ); + hit.setContext( artifactInfo.getContext() ); + hit.setGoals( artifactInfo.getGoals() ); + hit.setPrefix( artifactInfo.getPrefix() ); + hit.setPackaging( artifactInfo.getPackaging() ); + hit.setClassifier( artifactInfo.getClassifier() ); + hit.setFileExtension( artifactInfo.getFileExtension() ); + hit.setUrl( getBaseUrl( artifactInfo, selectedRepos ) ); + } + + results.addHit( id, hit ); + } + + results.setTotalHits( response.getTotalHitsCount() ); + results.setTotalHitsMapSize( results.getHitsMap().values().size() ); + results.setReturnedHitsCount( response.getReturnedHitsCount() ); + results.setLimits( limits ); + + if ( limits == null || limits.getSelectedPage() == SearchResultLimits.ALL_PAGES ) + { + return results; + } + else + { + return paginate( results ); + } + } + + /** + * calculate baseUrl without the context and base Archiva Url + * + * @param artifactInfo + * @return + */ + protected String getBaseUrl( ArtifactInfo artifactInfo, List<String> selectedRepos ) + throws RepositoryAdminException + { + StringBuilder sb = new StringBuilder(); + if ( StringUtils.startsWith( artifactInfo.getContext(), "remote-" ) ) + { + // it's a remote index result we search a managed which proxying this remote and on which + // current user has read karma + String managedRepoId = + getManagedRepoId( StringUtils.substringAfter( artifactInfo.getContext(), "remote-" ), selectedRepos ); + if ( managedRepoId != null ) + { + sb.append( '/' ).append( managedRepoId ); + artifactInfo.setContext( managedRepoId ); + } + } + else + { + sb.append( '/' ).append( artifactInfo.getContext() ); + } + + sb.append( '/' ).append( StringUtils.replaceChars( artifactInfo.getGroupId(), '.', '/' ) ); + sb.append( '/' ).append( artifactInfo.getArtifactId() ); + sb.append( '/' ).append( artifactInfo.getVersion() ); + sb.append( '/' ).append( artifactInfo.getArtifactId() ); + sb.append( '-' ).append( artifactInfo.getVersion() ); + if ( StringUtils.isNotBlank( artifactInfo.getClassifier() ) ) + { + sb.append( '-' ).append( artifactInfo.getClassifier() ); + } + // maven-plugin packaging is a jar + if ( StringUtils.equals( "maven-plugin", artifactInfo.getPackaging() ) ) + { + sb.append( "jar" ); + } + else + { + sb.append( '.' ).append( artifactInfo.getPackaging() ); + } + + return sb.toString(); + } + + /** + * return a managed repo for a remote result + * + * @param remoteRepo + * @param selectedRepos + * @return + * @throws RepositoryAdminException + */ + private String getManagedRepoId( String remoteRepo, List<String> selectedRepos ) + throws RepositoryAdminException + { + Map<String, List<ProxyConnector>> proxyConnectorMap = proxyConnectorAdmin.getProxyConnectorAsMap(); + if ( proxyConnectorMap == null || proxyConnectorMap.isEmpty() ) + { + return null; + } + if ( selectedRepos != null && !selectedRepos.isEmpty() ) + { + for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() ) + { + if ( selectedRepos.contains( entry.getKey() ) ) + { + for ( ProxyConnector proxyConnector : entry.getValue() ) + { + if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) ) + { + return proxyConnector.getSourceRepoId(); + } + } + } + } + } + + // we don't find in search selected repos so return the first one + for ( Map.Entry<String, List<ProxyConnector>> entry : proxyConnectorMap.entrySet() ) + { + + for ( ProxyConnector proxyConnector : entry.getValue() ) + { + if ( StringUtils.equals( remoteRepo, proxyConnector.getTargetRepoId() ) ) + { + return proxyConnector.getSourceRepoId(); + } + } + + } + return null; + } + + private boolean applyArtifactInfoFilters( ArtifactInfo artifactInfo, + List<? extends ArtifactInfoFilter> artifactInfoFilters, + Map<String, SearchResultHit> currentResult ) + { + if ( artifactInfoFilters == null || artifactInfoFilters.isEmpty() ) + { + return true; + } + + ArchivaArtifactModel artifact = new ArchivaArtifactModel(); + artifact.setArtifactId( artifactInfo.getArtifactId() ); + artifact.setClassifier( artifactInfo.getClassifier() ); + artifact.setGroupId( artifactInfo.getGroupId() ); + artifact.setRepositoryId( artifactInfo.getRepository() ); + artifact.setVersion( artifactInfo.getVersion() ); + artifact.setChecksumMD5( artifactInfo.getMd5() ); + artifact.setChecksumSHA1( artifactInfo.getSha1() ); + for ( ArtifactInfoFilter filter : artifactInfoFilters ) + { + if ( !filter.addArtifactInResult( artifact, currentResult ) ) + { + return false; + } + } + return true; + } + + protected SearchResults paginate( SearchResults results ) + { + SearchResultLimits limits = results.getLimits(); + SearchResults paginated = new SearchResults(); + + // ( limits.getPageSize() * ( Math.max( 1, limits.getSelectedPage() ) ) ); + + int fetchCount = limits.getPageSize(); + int offset = ( limits.getSelectedPage() * limits.getPageSize() ); + + if ( fetchCount > results.getTotalHits() ) + { + fetchCount = results.getTotalHits(); + } + + // Goto offset. + if ( offset < results.getTotalHits() ) + { + // only process if the offset is within the hit count. + for ( int i = 0; i < fetchCount; i++ ) + { + // Stop fetching if we are past the total # of available hits. + if ( offset + i >= results.getHits().size() ) + { + break; + } + + SearchResultHit hit = results.getHits().get( ( offset + i ) ); + if ( hit != null ) + { + String id = SearchUtil.getHitId( hit.getGroupId(), hit.getArtifactId(), hit.getClassifier(), + hit.getPackaging() ); + paginated.addHit( id, hit ); + } + else + { + break; + } + } + } + paginated.setTotalHits( results.getTotalHits() ); + paginated.setReturnedHitsCount( paginated.getHits().size() ); + paginated.setTotalHitsMapSize( results.getTotalHitsMapSize() ); + paginated.setLimits( limits ); + + return paginated; + } + + +} http://git-wip-us.apache.org/repos/asf/archiva/blob/c6e4e563/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java ---------------------------------------------------------------------- diff --git a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java b/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java deleted file mode 100644 index b4daa8b..0000000 --- a/archiva-modules/archiva-base/archiva-maven2-indexer/src/main/java/org/apache/archiva/indexer/merger/DefaultIndexMerger.java +++ /dev/null @@ -1,173 +0,0 @@ -package org.apache.archiva.indexer.merger; -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import org.apache.archiva.common.utils.FileUtils; -import org.apache.commons.lang.time.StopWatch; -import org.apache.maven.index.NexusIndexer; -import org.apache.maven.index.context.IndexCreator; -import org.apache.maven.index.context.IndexingContext; -import org.apache.maven.index.context.UnsupportedExistingLuceneIndexException; -import org.apache.maven.index.packer.IndexPacker; -import org.apache.maven.index.packer.IndexPackingRequest; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Async; -import org.springframework.stereotype.Service; - -import javax.inject.Inject; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.CopyOnWriteArrayList; - -/** - * @author Olivier Lamy - * @since 1.4-M2 - */ -@Service("indexMerger#default") -public class DefaultIndexMerger - implements IndexMerger -{ - - private Logger log = LoggerFactory.getLogger( getClass() ); - - private final NexusIndexer indexer; - - private final IndexPacker indexPacker; - - private final List<IndexCreator> indexCreators; - - private List<TemporaryGroupIndex> temporaryGroupIndexes = new CopyOnWriteArrayList<>(); - - private List<String> runningGroups = new CopyOnWriteArrayList<>(); - - @Inject - public DefaultIndexMerger( NexusIndexer nexusIndexer, IndexPacker indexPacker, List<IndexCreator> indexCreators ) - { - this.indexer = nexusIndexer; - this.indexPacker = indexPacker; - this.indexCreators = indexCreators; - } - - @Override - public IndexingContext buildMergedIndex( IndexMergerRequest indexMergerRequest ) - throws IndexMergerException - { - String groupId = indexMergerRequest.getGroupId(); - - if ( runningGroups.contains( groupId ) ) - { - log.info( "skip build merge remote indexes for id: '{}' as already running", groupId ); - return null; - } - - runningGroups.add( groupId ); - - StopWatch stopWatch = new StopWatch(); - stopWatch.reset(); - stopWatch.start(); - - Path mergedIndexDirectory = indexMergerRequest.getMergedIndexDirectory(); - - String tempRepoId = mergedIndexDirectory.getFileName().toString(); - - try - { - Path indexLocation = mergedIndexDirectory.resolve( indexMergerRequest.getMergedIndexPath() ); - IndexingContext indexingContext = - indexer.addIndexingContext( tempRepoId, tempRepoId, mergedIndexDirectory.toFile(), indexLocation.toFile(), null, null, - indexCreators ); - - for ( String repoId : indexMergerRequest.getRepositoriesIds() ) - { - IndexingContext idxToMerge = indexer.getIndexingContexts().get( repoId ); - if ( idxToMerge != null ) - { - indexingContext.merge( idxToMerge.getIndexDirectory() ); - } - } - - indexingContext.optimize(); - - if ( indexMergerRequest.isPackIndex() ) - { - IndexPackingRequest request = new IndexPackingRequest( indexingContext, // - indexingContext.acquireIndexSearcher().getIndexReader(), // - indexLocation.toFile() ); - indexPacker.packIndex( request ); - } - - if ( indexMergerRequest.isTemporary() ) - { - temporaryGroupIndexes.add( new TemporaryGroupIndex( mergedIndexDirectory, tempRepoId, groupId, - indexMergerRequest.getMergedIndexTtl() ) ); - } - stopWatch.stop(); - log.info( "merged index for repos {} in {} s", indexMergerRequest.getRepositoriesIds(), - stopWatch.getTime() ); - return indexingContext; - } - catch ( IOException | UnsupportedExistingLuceneIndexException e ) - { - throw new IndexMergerException( e.getMessage(), e ); - } - finally - { - runningGroups.remove( groupId ); - } - } - - @Async - @Override - public void cleanTemporaryGroupIndex( TemporaryGroupIndex temporaryGroupIndex ) - { - if ( temporaryGroupIndex == null ) - { - return; - } - - try - { - IndexingContext indexingContext = indexer.getIndexingContexts().get( temporaryGroupIndex.getIndexId() ); - if ( indexingContext != null ) - { - indexer.removeIndexingContext( indexingContext, true ); - } - Path directory = temporaryGroupIndex.getDirectory(); - if ( directory != null && Files.exists(directory) ) - { - FileUtils.deleteDirectory( directory ); - } - temporaryGroupIndexes.remove( temporaryGroupIndex ); - } - catch ( IOException e ) - { - log.warn( "fail to delete temporary group index {}", temporaryGroupIndex.getIndexId(), e ); - } - } - - @Override - public Collection<TemporaryGroupIndex> getTemporaryGroupIndexes() - { - return this.temporaryGroupIndexes; - } -}