Repository: maven-indexer Updated Branches: refs/heads/maven-indexer-5.x c19388828 -> af8783d8f
MINDEXER-96: Indexer reader Project: http://git-wip-us.apache.org/repos/asf/maven-indexer/repo Commit: http://git-wip-us.apache.org/repos/asf/maven-indexer/commit/af8783d8 Tree: http://git-wip-us.apache.org/repos/asf/maven-indexer/tree/af8783d8 Diff: http://git-wip-us.apache.org/repos/asf/maven-indexer/diff/af8783d8 Branch: refs/heads/maven-indexer-5.x Commit: af8783d8fc8dcab6ef6d9a17d04858fd725037a0 Parents: c193888 Author: Tamas Cservenak <[email protected]> Authored: Sat Oct 31 01:29:30 2015 +0100 Committer: Tamas Cservenak <[email protected]> Committed: Sat Oct 31 01:29:30 2015 +0100 ---------------------------------------------------------------------- indexer-reader/README.md | 8 + indexer-reader/header.txt | 17 + indexer-reader/pom.xml | 48 ++ .../apache/maven/index/reader/ChunkReader.java | 454 +++++++++++++++++++ .../apache/maven/index/reader/IndexReader.java | 281 ++++++++++++ .../org/apache/maven/index/reader/Record.java | 247 ++++++++++ .../maven/index/reader/ResourceHandler.java | 45 ++ .../index/reader/WritableResourceHandler.java | 44 ++ .../index/reader/CachingResourceHandler.java | 60 +++ .../maven/index/reader/ChunkReaderTest.java | 66 +++ .../index/reader/DirectoryResourceHandler.java | 72 +++ .../maven/index/reader/HttpResourceHandler.java | 57 +++ .../maven/index/reader/IndexReaderTest.java | 102 +++++ .../resources/nexus-maven-repository-index.gz | Bin 0 -> 319 bytes .../nexus-maven-repository-index.properties | 6 + pom.xml | 1 + 16 files changed, 1508 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/README.md ---------------------------------------------------------------------- diff --git a/indexer-reader/README.md b/indexer-reader/README.md new file mode 100644 index 0000000..95aa1b8 --- /dev/null +++ b/indexer-reader/README.md @@ -0,0 +1,8 @@ +Indexer Reader Notes +================== + +Indexer Reader is a minimal dep-less library that is able to read published (remote) +index with incremental update support, making this library user able to integrate +published Maven Indexes into any engine without depending on maven-indexer-core +and it's transitive dependencies. + http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/header.txt ---------------------------------------------------------------------- diff --git a/indexer-reader/header.txt b/indexer-reader/header.txt new file mode 100644 index 0000000..1a2ef73 --- /dev/null +++ b/indexer-reader/header.txt @@ -0,0 +1,17 @@ +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. + http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/pom.xml ---------------------------------------------------------------------- diff --git a/indexer-reader/pom.xml b/indexer-reader/pom.xml new file mode 100644 index 0000000..daf364c --- /dev/null +++ b/indexer-reader/pom.xml @@ -0,0 +1,48 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.maven.indexer</groupId> + <artifactId>maven-indexer</artifactId> + <version>5.1.2-SNAPSHOT</version> + </parent> + + <artifactId>indexer-reader</artifactId> + + <name>Maven :: Indexer Reader</name> + <description> + Indexer Reader is a minimal dep-less library that is able to read published (remote) index with incremental update + support, making user able to integrate published Maven Indexes into any engine without depending on + maven-indexer-core and it's transitive dependencies. + </description> + + <dependencies> + <!-- Test --> + <dependency> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + <scope>test</scope> + </dependency> + </dependencies> + +</project> http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java new file mode 100644 index 0000000..89434fd --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ChunkReader.java @@ -0,0 +1,454 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.DataInputStream; +import java.io.EOFException; +import java.io.IOException; +import java.io.InputStream; +import java.io.UTFDataFormatException; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.regex.Pattern; +import java.util.zip.GZIPInputStream; + +import org.apache.maven.index.reader.Record.Type; + +/** + * Maven 2 Index published binary chunk reader. + * + * @since 5.1.2 + */ +public class ChunkReader + implements Closeable, Iterable<Record> +{ + private static final String FIELD_SEPARATOR = "|"; + + private static final String NOT_AVAILABLE = "NA"; + + private static final String UINFO = "u"; + + private static final String INFO = "i"; + + private static final Pattern FS_PATTERN = Pattern.compile(Pattern.quote(FIELD_SEPARATOR)); + + private final String chunkName; + + private final DataInputStream dataInputStream; + + private final int version; + + private final Date timestamp; + + public ChunkReader(final String chunkName, final InputStream inputStream) throws IOException + { + this.chunkName = chunkName.trim(); + this.dataInputStream = new DataInputStream(new GZIPInputStream(inputStream, 2 * 1024)); + this.version = ((int) dataInputStream.readByte()) & 0xff; + this.timestamp = new Date(dataInputStream.readLong()); + } + + /** + * Returns the chunk name. + */ + public String getName() { + return chunkName; + } + + /** + * Returns index getVersion. All releases so far always returned {@code 1}. + */ + public int getVersion() { + return version; + } + + /** + * Returns the getTimestamp of last update of the index. + */ + public Date getTimestamp() { + return timestamp; + } + + /** + * Returns the {@link Record} iterator. + */ + public Iterator<Record> iterator() { + try { + return new IndexIterator(dataInputStream); + } + catch (IOException e) { + throw new RuntimeException("error", e); + } + } + + /** + * Closes this reader and it's underlying input. + */ + public void close() throws IOException { + dataInputStream.close(); + } + + /** + * Low memory footprint index iterator that incrementally parses the underlying stream. + */ + private static class IndexIterator + implements Iterator<Record> + { + private final DataInputStream dataInputStream; + + private Record nextRecord; + + public IndexIterator(final DataInputStream dataInputStream) throws IOException { + this.dataInputStream = dataInputStream; + this.nextRecord = readRecord(); + } + + public boolean hasNext() { + return nextRecord != null; + } + + public Record next() { + if (nextRecord == null) { + throw new NoSuchElementException("chunk depleted"); + } + Record result = nextRecord; + try { + nextRecord = readRecord(); + return result; + } + catch (IOException e) { + throw new RuntimeException("read error", e); + } + } + + /** + * Reads and returns next record from the underlying stream, or {@code null} if no more records. + */ + private Record readRecord() + throws IOException + { + int fieldCount; + try { + fieldCount = dataInputStream.readInt(); + } + catch (EOFException ex) { + return null; // no more documents + } + + Map<String, String> recordMap = new HashMap<String, String>(); + for (int i = 0; i < fieldCount; i++) { + readField(recordMap); + } + + if (recordMap.containsKey("DESCRIPTOR")) { + return new Record(Type.DESCRIPTOR, recordMap, expandDescriptor(recordMap)); + } + else if (recordMap.containsKey("allGroups")) { + return new Record(Type.ALL_GROUPS, recordMap, expandAllGroups(recordMap)); + } + else if (recordMap.containsKey("rootGroups")) { + return new Record(Type.ROOT_GROUPS, recordMap, expandRootGroups(recordMap)); + } + else if (recordMap.containsKey("del")) { + return new Record(Type.ARTIFACT_REMOVE, recordMap, expandDeletedArtifact(recordMap)); + } + else { + // Fix up UINFO field wrt MINDEXER-41 + final String uinfo = recordMap.get(UINFO); + final String info = recordMap.get(INFO); + if (uinfo != null && !(info == null || info.trim().length() == 0)) { + final String[] splitInfo = FS_PATTERN.split(info); + if (splitInfo.length > 6) { + final String extension = splitInfo[6]; + if (uinfo.endsWith(FIELD_SEPARATOR + NOT_AVAILABLE)) { + recordMap.put(UINFO, uinfo + FIELD_SEPARATOR + extension); + } + } + } + return new Record(Type.ARTIFACT_ADD, recordMap, expandAddedArtifact(recordMap)); + } + } + + private void readField(final Map<String, String> record) + throws IOException + { + dataInputStream.read(); // flags: neglect them + String name = dataInputStream.readUTF(); + String value = readUTF(); + record.put(name, value); + } + + private String readUTF() + throws IOException + { + int utflen = dataInputStream.readInt(); + + byte[] bytearr; + char[] chararr; + + try { + bytearr = new byte[utflen]; + chararr = new char[utflen]; + } + catch (OutOfMemoryError e) { + IOException ioex = new IOException("Index data content is corrupt"); + ioex.initCause(e); + throw ioex; + } + + int c, char2, char3; + int count = 0; + int chararr_count = 0; + + dataInputStream.readFully(bytearr, 0, utflen); + + while (count < utflen) { + c = bytearr[count] & 0xff; + if (c > 127) { + break; + } + count++; + chararr[chararr_count++] = (char) c; + } + + while (count < utflen) { + c = bytearr[count] & 0xff; + switch (c >> 4) { + case 0: + case 1: + case 2: + case 3: + case 4: + case 5: + case 6: + case 7: + /* 0xxxxxxx */ + count++; + chararr[chararr_count++] = (char) c; + break; + + case 12: + case 13: + /* 110x xxxx 10xx xxxx */ + count += 2; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 1]; + if ((char2 & 0xC0) != 0x80) { + throw new UTFDataFormatException("malformed input around byte " + count); + } + chararr[chararr_count++] = (char) (((c & 0x1F) << 6) | (char2 & 0x3F)); + break; + + case 14: + /* 1110 xxxx 10xx xxxx 10xx xxxx */ + count += 3; + if (count > utflen) { + throw new UTFDataFormatException("malformed input: partial character at end"); + } + char2 = bytearr[count - 2]; + char3 = bytearr[count - 1]; + if (((char2 & 0xC0) != 0x80) || ((char3 & 0xC0) != 0x80)) { + throw new UTFDataFormatException("malformed input around byte " + (count - 1)); + } + chararr[chararr_count++] = + (char) (((c & 0x0F) << 12) | ((char2 & 0x3F) << 6) | (char3 & 0x3F)); + break; + + default: + /* 10xx xxxx, 1111 xxxx */ + throw new UTFDataFormatException("malformed input around byte " + count); + } + } + + // The number of chars produced may be less than utflen + return new String(chararr, 0, chararr_count); + } + + private Map<String, Object> expandDescriptor(final Map<String, String> raw) { + final Map<String, Object> result = new HashMap<String, Object>(); + String[] r = FS_PATTERN.split(raw.get("IDXINFO")); + result.put(Record.REPOSITORY_ID, r[1]); + return result; + } + + private Map<String, Object> expandAllGroups(final Map<String, String> raw) { + final Map<String, Object> result = new HashMap<String, Object>(); + putIfNotNullAsList(raw, Record.ALL_GROUPS_LIST, result, "allGroups"); + return result; + } + + private Map<String, Object> expandRootGroups(final Map<String, String> raw) { + final Map<String, Object> result = new HashMap<String, Object>(); + putIfNotNullAsList(raw, Record.ROOT_GROUPS_LIST, result, "rootGroups"); + return result; + } + + private Map<String, Object> expandDeletedArtifact(final Map<String, String> raw) { + final Map<String, Object> result = new HashMap<String, Object>(); + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + if (raw.containsKey("del")) { + expandUinfo(raw.get("del"), result); + } + return result; + } + + /** + * Expands the "encoded" Maven Indexer record by splitting the synthetic fields and applying expanded field naming. + */ + private Map<String, Object> expandAddedArtifact(final Map<String, String> raw) { + final Map<String, Object> result = new HashMap<String, Object>(); + + // Minimal + expandUinfo(raw.get(UINFO), result); + final String info = raw.get(INFO); + if (info != null) { + String[] r = FS_PATTERN.split(info); + result.put(Record.PACKAGING, renvl(r[0])); + result.put(Record.FILE_MODIFIED, Long.valueOf(r[1])); + result.put(Record.FILE_SIZE, Long.valueOf(r[2])); + result.put(Record.HAS_SOURCES, "1".equals(r[3]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + result.put(Record.HAS_JAVADOC, "1".equals(r[4]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + result.put(Record.HAS_SIGNATURE, "1".equals(r[5]) ? Boolean.TRUE.toString() : Boolean.FALSE.toString()); + if (r.length > 6) { + result.put(Record.FILE_EXTENSION, r[6]); + } + else { + final String packaging = raw.get(Record.PACKAGING); + if (raw.get(Record.CLASSIFIER) != null + || "pom".equals(packaging) + || "war".equals(packaging) + || "ear".equals(packaging)) { + result.put(Record.FILE_EXTENSION, packaging); + } + else { + result.put(Record.FILE_EXTENSION, "jar"); // best guess + } + } + } + putIfNotNullTS(raw, "m", result, Record.REC_MODIFIED); + putIfNotNull(raw, "n", result, Record.NAME); + putIfNotNull(raw, "d", result, Record.DESCRIPTION); + putIfNotNull(raw, "1", result, Record.SHA1); + + // Jar file contents (optional) + putIfNotNullAsList(raw, "classnames", result, Record.CLASSNAMES); + + // Maven Plugin (optional) + putIfNotNull(raw, "px", result, Record.PLUGIN_PREFIX); + putIfNotNullAsList(raw, "gx", result, Record.PLUGIN_GOALS); + + // OSGi (optional) + putIfNotNull(raw, "Bundle-SymbolicName", result, "Bundle-SymbolicName"); + putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); + putIfNotNull(raw, "Export-Package", result, "Export-Package"); + putIfNotNull(raw, "Export-Service", result, "Export-Service"); + putIfNotNull(raw, "Bundle-Description", result, "Bundle-Description"); + putIfNotNull(raw, "Bundle-Name", result, "Bundle-Name"); + putIfNotNull(raw, "Bundle-License", result, "Bundle-License"); + putIfNotNull(raw, "Bundle-DocURL", result, "Bundle-DocURL"); + putIfNotNull(raw, "Import-Package", result, "Import-Package"); + putIfNotNull(raw, "Require-Bundle", result, "Require-Bundle"); + putIfNotNull(raw, "Bundle-Version", result, "Bundle-Version"); + + return result; + } + + /** + * Expands UINFO synthetic field. Handles {@code null} String inputs. + */ + private void expandUinfo(final String uinfo, final Map<String, Object> result) { + if (uinfo != null) { + String[] r = FS_PATTERN.split(uinfo); + result.put(Record.GROUP_ID, r[0]); + result.put(Record.ARTIFACT_ID, r[1]); + result.put(Record.VERSION, r[2]); + String classifier = renvl(r[3]); + if (classifier != null) { + result.put(Record.CLASSIFIER, classifier); + if (r.length > 4) { + result.put(Record.FILE_EXTENSION, r[4]); + } + } + else if (r.length > 4) { + result.put(Record.PACKAGING, r[4]); + } + } + } + } + + /** + * Helper to put a value from source map into target map, if not null. + */ + private static void putIfNotNull( + final Map<String, String> source, + final String sourceName, + final Map<String, Object> target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, value); + } + } + + /** + * Helper to put a {@link Long} value from source map into target map, if not null. + */ + private static void putIfNotNullTS( + final Map<String, String> source, + final String sourceName, + final Map<String, Object> target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Long.valueOf(value)); + } + } + + /** + * Helper to put a collection value from source map into target map as {@link java.util.List}, if not null. + */ + private static void putIfNotNullAsList( + final Map<String, String> source, + final String sourceName, + final Map<String, Object> target, + final String targetName) + { + String value = source.get(sourceName); + if (value != null && value.trim().length() != 0) { + target.put(targetName, Arrays.asList(FS_PATTERN.split(value))); + } + } + + /** + * Helper to translate the "NA" (not available) input into {@code null} value. + */ + private static String renvl(final String v) { + return NOT_AVAILABLE.equals(v) ? null : v; + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java new file mode 100644 index 0000000..45514b9 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/IndexReader.java @@ -0,0 +1,281 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.Iterator; +import java.util.List; +import java.util.Properties; +import java.util.TimeZone; + +/** + * Maven 2 Index reader that handles incremental updates if possible. + * + * @since 5.1.2 + */ +public class IndexReader + implements Iterable<ChunkReader>, Closeable +{ + private static final String INDEX_FILE_PREFIX = "nexus-maven-repository-index"; + + private static final DateFormat INDEX_DATE_FORMAT; + + static { + INDEX_DATE_FORMAT = new SimpleDateFormat("yyyyMMddHHmmss.SSS Z"); + INDEX_DATE_FORMAT.setTimeZone(TimeZone.getTimeZone("GMT")); + } + + private final WritableResourceHandler local; + + private final ResourceHandler remote; + + private final Properties localIndexProperties; + + private final Properties remoteIndexProperties; + + private final String indexId; + + private final Date publishedTimestamp; + + private final boolean incremental; + + private final List<String> chunkNames; + + public IndexReader(final WritableResourceHandler local, final ResourceHandler remote) throws IOException { + if (remote == null) { + throw new NullPointerException("remote resource handler null"); + } + this.local = local; + this.remote = remote; + remoteIndexProperties = loadProperties(remote.open(INDEX_FILE_PREFIX + ".properties")); + try { + if (local != null) { + localIndexProperties = loadProperties(local.open(INDEX_FILE_PREFIX + ".properties")); + String remoteIndexId = remoteIndexProperties.getProperty("nexus.index.id"); + String localIndexId = localIndexProperties.getProperty("nexus.index.id"); + if (remoteIndexId == null || localIndexId == null || !remoteIndexId.equals(localIndexId)) { + throw new IllegalArgumentException( + "local and remote index IDs does not match or is null: " + localIndexId + ", " + + remoteIndexId); + } + this.indexId = localIndexId; + this.publishedTimestamp = INDEX_DATE_FORMAT.parse(localIndexProperties.getProperty("nexus.index.timestamp")); + this.incremental = canRetrieveAllChunks(); + this.chunkNames = calculateChunkNames(); + } + else { + localIndexProperties = null; + this.indexId = remoteIndexProperties.getProperty("nexus.index.id"); + this.publishedTimestamp = INDEX_DATE_FORMAT.parse(remoteIndexProperties.getProperty("nexus.index.timestamp")); + this.incremental = false; + this.chunkNames = calculateChunkNames(); + } + } + catch (ParseException e) { + IOException ex = new IOException("Index properties corrupted"); + ex.initCause(e); + throw ex; + } + } + + /** + * Returns the index context ID that published index has set. Usually it is equal to "repository ID" used in {@link + * Record.Type#DESCRIPTOR} but does not have to be. + */ + public String getIndexId() { + return indexId; + } + + /** + * Returns the {@link Date} when remote index was last published. + */ + public Date getPublishedTimestamp() { + return publishedTimestamp; + } + + /** + * Returns {@code true} if incremental update is about to happen. If incremental update, the {@link #iterator()} will + * return only the diff from the last update. + */ + public boolean isIncremental() { + return incremental; + } + + /** + * Returns unmodifiable list of actual chunks that needs to be pulled from remote {@link ResourceHandler}. Those are + * incremental chunks or the big main file, depending on result of {@link #isIncremental()}. Empty list means local + * index is up to date, and {@link #iterator()} will return empty iterator. + */ + public List<String> getChunkNames() { + return chunkNames; + } + + /** + * Closes the underlying {@link ResourceHandler}s. In case of incremental update use, it also assumes that user + * consumed all the iterator and integrated it, hence, it will update the {@link WritableResourceHandler} contents to + * prepare it for future incremental update. If this is not desired (ie. due to aborted update), then this method + * should NOT be invoked, but rather the {@link ResourceHandler}s that caller provided in constructor of + * this class should be closed manually. + */ + public void close() throws IOException { + remote.close(); + if (local != null) { + try { + syncLocalWithRemote(); + } + finally { + local.close(); + } + } + } + + /** + * Returns an {@link Iterator} of {@link ChunkReader}s, that if read in sequence, provide all the (incremental) + * updates from the index. It is caller responsibility to either consume fully this iterator, or to close current + * {@link ChunkReader} if aborting. + */ + public Iterator<ChunkReader> iterator() { + return new ChunkReaderIterator(remote, chunkNames.iterator()); + } + + /** + * Stores the remote index properties into local index properties, preparing local {@link WritableResourceHandler} + * for future incremental updates. + */ + private void syncLocalWithRemote() throws IOException { + final ByteArrayOutputStream bos = new ByteArrayOutputStream(); + remoteIndexProperties.store(bos, "Maven Indexer Reader"); + local.save(INDEX_FILE_PREFIX + ".properties", new ByteArrayInputStream(bos.toByteArray())); + } + + /** + * Calculates the chunk names that needs to be fetched. + */ + private List<String> calculateChunkNames() { + if (incremental) { + ArrayList<String> chunkNames = new ArrayList<String>(); + int maxCounter = Integer.parseInt(remoteIndexProperties.getProperty("nexus.index.last-incremental")); + int currentCounter = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + currentCounter++; + while (currentCounter <= maxCounter) { + chunkNames.add(INDEX_FILE_PREFIX + "." + currentCounter++ + ".gz"); + } + return Collections.unmodifiableList(chunkNames); + } + else { + return Collections.singletonList(INDEX_FILE_PREFIX + ".gz"); + } + } + + /** + * Verifies incremental update is possible, as all the diff chunks we need are still enlisted in remote properties. + */ + private boolean canRetrieveAllChunks() + { + String localChainId = localIndexProperties.getProperty("nexus.index.chain-id"); + String remoteChainId = remoteIndexProperties.getProperty("nexus.index.chain-id"); + + // If no chain id, or not the same, do full update + if (localChainId == null || remoteChainId == null || !localChainId.equals(remoteChainId)) { + return false; + } + + try { + int localLastIncremental = Integer.parseInt(localIndexProperties.getProperty("nexus.index.last-incremental")); + String currentLocalCounter = String.valueOf(localLastIncremental); + String nextLocalCounter = String.valueOf(localLastIncremental + 1); + // check remote props for existence of current or next chunk after local + for (Object key : remoteIndexProperties.keySet()) { + String sKey = (String) key; + if (sKey.startsWith("nexus.index.incremental-")) { + String value = remoteIndexProperties.getProperty(sKey); + if (currentLocalCounter.equals(value) || nextLocalCounter.equals(value)) { + return true; + } + } + } + } + catch (NumberFormatException e) { + // fall through + } + return false; + } + + /** + * Internal iterator implementation that lazily opens and closes the returned {@link ChunkReader}s as this iterator + * is being consumed. + */ + private static class ChunkReaderIterator + implements Iterator<ChunkReader> + { + private final ResourceHandler resourceHandler; + + private final Iterator<String> chunkNamesIterator; + + private ChunkReader currentChunkReader; + + private ChunkReaderIterator(final ResourceHandler resourceHandler, final Iterator<String> chunkNamesIterator) { + this.resourceHandler = resourceHandler; + this.chunkNamesIterator = chunkNamesIterator; + } + + public boolean hasNext() { + return chunkNamesIterator.hasNext(); + } + + public ChunkReader next() { + String chunkName = chunkNamesIterator.next(); + try { + if (currentChunkReader != null) { + currentChunkReader.close(); + } + currentChunkReader = new ChunkReader(chunkName, resourceHandler.open(chunkName)); + return currentChunkReader; + } + catch (IOException e) { + throw new RuntimeException("IO problem while switching chunk readers", e); + } + } + } + + /** + * Creates and loads {@link Properties} from provided {@link InputStream} and closes the stream. + */ + private static Properties loadProperties(final InputStream inputStream) throws IOException { + try { + final Properties properties = new Properties(); + properties.load(inputStream); + return properties; + } + finally { + inputStream.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java new file mode 100644 index 0000000..3354008 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/Record.java @@ -0,0 +1,247 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.util.Map; + +/** + * Maven 2 Index record. + * + * @since 5.1.2 + */ +public class Record +{ + /** + * Key of repository ID entry, that contains {@link String}. + */ + public static final String REPOSITORY_ID = "repositoryId"; + + /** + * Key of all groups list entry, that contains {@link java.util.List<String>}. + */ + public static final String ALL_GROUPS_LIST = "allGroupsList"; + + /** + * Key of root groups list entry, that contains {@link java.util.List<String>}. + */ + public static final String ROOT_GROUPS_LIST = "rootGroupsList"; + + /** + * Key of index record modification (added to index or removed from index) timestamp entry, that contains {@link + * Long}. + */ + public static final String REC_MODIFIED = "recordModified"; + + /** + * Key of artifact groupId entry, that contains {@link String}. + */ + public static final String GROUP_ID = "groupId"; + + /** + * Key of artifact artifactId entry, that contains {@link String}. + */ + public static final String ARTIFACT_ID = "artifactId"; + + /** + * Key of artifact version entry, that contains {@link String}. + */ + public static final String VERSION = "version"; + + /** + * Key of artifact classifier entry, that contains {@link String}. + */ + public static final String CLASSIFIER = "classifier"; + + /** + * Key of artifact packaging entry, that contains {@link String}. + */ + public static final String PACKAGING = "packaging"; + + /** + * Key of artifact file extension, that contains {@link String}. + */ + public static final String FILE_EXTENSION = "fileExtension"; + + /** + * Key of artifact file last modified timestamp, that contains {@link Long}. + */ + public static final String FILE_MODIFIED = "fileModified"; + + /** + * Key of artifact file size in bytes, that contains {@link Long}. + */ + public static final String FILE_SIZE = "fileSize"; + + /** + * Key of artifact Sources presence flag, that contains {@link Boolean}. + */ + public static final String HAS_SOURCES = "hasSources"; + + /** + * Key of artifact Javadoc presence flag, that contains {@link Boolean}. + */ + public static final String HAS_JAVADOC = "hasJavadoc"; + + /** + * Key of artifact signature presence flag, that contains {@link Boolean}. + */ + public static final String HAS_SIGNATURE = "hasSignature"; + + /** + * Key of artifact name (as set in POM), that contains {@link String}. + */ + public static final String NAME = "name"; + + /** + * Key of artifact description (as set in POM), that contains {@link String}. + */ + public static final String DESCRIPTION = "description"; + + /** + * Key of artifact SHA1 digest, that contains {@link String}. + */ + public static final String SHA1 = "sha1"; + + /** + * Key of artifact contained class names, that contains {@link java.util.List<String>}. + */ + public static final String CLASSNAMES = "classNames"; + + /** + * Key of plugin artifact prefix, that contains {@link String}. + */ + public static final String PLUGIN_PREFIX = "pluginPrefix"; + + /** + * Key of plugin artifact goals, that contains {@link java.util.List<String>}. + */ + public static final String PLUGIN_GOALS = "pluginGoals"; + + /** + * Types of returned records returned from index. + */ + public enum Type + { + /** + * Descriptor record. Can be safely ignored. + * Contains following entries: + * <ul> + * <li>{@link #REPOSITORY_ID}</li> + * </ul> + */ + DESCRIPTOR, + + /** + * Artifact ADD record. Records of this type should be added to your indexing system. + * Contains following entries: + * <ul> + * <li>{@link #REC_MODIFIED} (when record was added/modified on index)</li> + * <li>{@link #GROUP_ID}</li> + * <li>{@link #ARTIFACT_ID}</li> + * <li>{@link #VERSION}</li> + * <li>{@link #CLASSIFIER} (optional)</li> + * <li>{@link #FILE_EXTENSION}</li> + * <li>{@link #FILE_MODIFIED}</li> + * <li>{@link #FILE_SIZE}</li> + * <li>{@link #PACKAGING}</li> + * <li>{@link #HAS_SOURCES}</li> + * <li>{@link #HAS_JAVADOC}</li> + * <li>{@link #HAS_SIGNATURE}</li> + * <li>{@link #NAME}</li> + * <li>{@link #DESCRIPTION}</li> + * <li>{@link #SHA1}</li> + * <li>{@link #CLASSNAMES} (optional)</li> + * <li>{@link #PLUGIN_PREFIX} (optional, for maven-plugins only)</li> + * <li>{@link #PLUGIN_GOALS} (optional, for maven-plugins only)</li> + * </ul> + */ + ARTIFACT_ADD, + + /** + * Artifact REMOTE record. In case of incremental updates, notes that this artifact was removed. Records of this + * type should be removed from your indexing system. + * Contains following entries: + * <ul> + * <li>{@link #REC_MODIFIED} (when record was deleted from index)</li> + * <li>{@link #GROUP_ID}</li> + * <li>{@link #ARTIFACT_ID}</li> + * <li>{@link #VERSION}</li> + * <li>{@link #CLASSIFIER} (optional)</li> + * <li>{@link #FILE_EXTENSION} (if {@link #CLASSIFIER} present)</li> + * <li>{@link #PACKAGING} (optional)</li> + * </ul> + */ + ARTIFACT_REMOVE, + + /** + * Special record, containing all the Maven "groupId"s that are enlisted on the index. Can be safely ignored. + * Contains following entries: + * <ul> + * <li>{@link #ALL_GROUPS_LIST}</li> + * </ul> + */ + ALL_GROUPS, + + /** + * Special record, containing all the root groups of Maven "groupId"s that are enlisted on the index. Can be safely + * ignored. + * Contains following entries: + * <ul> + * <li>{@link #ROOT_GROUPS_LIST}</li> + * </ul> + */ + ROOT_GROUPS + } + + private final Type type; + + private final Map<String, String> raw; + + private final Map<String, Object> expanded; + + public Record(final Type type, final Map<String, String> raw, final Map<String, Object> expanded) { + this.type = type; + this.raw = raw; + this.expanded = expanded; + } + + /** + * Returns the {@link Type} of this record. Usually users would be interested in {@link Type#ARTIFACT_ADD} and {@link + * Type#ARTIFACT_REMOVE} types only to maintain their own index. Still, indexer offers extra records too, see {@link + * Type} for all existing types. + */ + public Type getType() { + return type; + } + + /** + * Returns the "raw", Maven Indexer specific record as a {@link Map}. + */ + public Map<String, String> getRaw() { + return raw; + } + + /** + * Returns the expanded (processed and expanded synthetic fields) record as {@link Map} ready for consumption. + */ + public Map<String, Object> getExpanded() { + return expanded; + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java new file mode 100644 index 0000000..9780067 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/ResourceHandler.java @@ -0,0 +1,45 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStream; + +/** + * Maven 2 Index resource abstraction, that should be handled as a resource (is {@link Closeable}. That means, that + * implementations could perform any extra activity as FS locking or so (if uses FS as backing store). If the + * implementation plans to fetch from remote, it could implement very simple "cache" mechanism, to fetch only once + * during the lifespan of the instance, as for indexer there is no reason to re-fetch during single session, nor + * yo have any advanced caching (ie. TTLs etc). + * + * @since 5.1.2 + */ +public interface ResourceHandler + extends Closeable +{ + /** + * Returns the {@link InputStream} of resource with {@code name} or {@code null} if no such resource. Closing the + * stream is the responsibility of the caller. + * + * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + */ + InputStream open(String name) throws IOException; +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java new file mode 100644 index 0000000..7fe3896 --- /dev/null +++ b/indexer-reader/src/main/java/org/apache/maven/index/reader/WritableResourceHandler.java @@ -0,0 +1,44 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +/** + * Maven 2 Index writable {@link ResourceHandler}, is capable of saving resources too. Needed only if incremental index + * updates are wanted, to store the index state locally, and be able to calculate incremental diffs on next {@link + * IndexReader} invocation. + * + * @see ResourceHandler + * @since 5.1.2 + */ +public interface WritableResourceHandler + extends ResourceHandler +{ + /** + * Stores (creates or overwrites if resource with name exists) the resource under {@code name} with content provided + * by the stream. The {@link InputStream} should be closed when method returns. + * + * @param name Resource name, guaranteed to be non-{@code null} and is FS name and URL safe string. + * @param inputStream the content of the resource, guaranteed to be non-{@code null}. + */ + void save(final String name, final InputStream inputStream) throws IOException; +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java new file mode 100644 index 0000000..6afe222 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/CachingResourceHandler.java @@ -0,0 +1,60 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.IOException; +import java.io.InputStream; + +/** + * A trivial caching {@link ResourceHandler} that caches forever during single session (existence of the instance). + */ +public class CachingResourceHandler + implements ResourceHandler +{ + private final DirectoryResourceHandler local; + + private final ResourceHandler remote; + + public CachingResourceHandler(final DirectoryResourceHandler local, final ResourceHandler remote) { + if (local == null || remote == null) { + throw new NullPointerException("null resource handler"); + } + this.local = local; + this.remote = remote; + } + + public InputStream open(final String name) throws IOException { + InputStream inputStream = local.open(name); + if (inputStream != null) { + return inputStream; + } + inputStream = remote.open(name); + if (inputStream == null) { + return null; + } + local.save(name, inputStream); + return local.open(name); + } + + public void close() throws IOException { + remote.close(); + local.close(); + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java new file mode 100644 index 0000000..0d1915d --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/ChunkReaderTest.java @@ -0,0 +1,66 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.FileInputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.maven.index.reader.Record.Type; +import org.junit.Test; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link ChunkReader} + */ +public class ChunkReaderTest +{ + @Test + public void simple() throws IOException { + final Map<Type, Integer> recordTypes = new HashMap<Type, Integer>(); + recordTypes.put(Type.DESCRIPTOR, 0); + recordTypes.put(Type.ROOT_GROUPS, 0); + recordTypes.put(Type.ALL_GROUPS, 0); + recordTypes.put(Type.ARTIFACT_ADD, 0); + recordTypes.put(Type.ARTIFACT_REMOVE, 0); + + final ChunkReader chunkReader = new ChunkReader("full", + new FileInputStream("src/test/resources/nexus-maven-repository-index.gz")); + try { + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : chunkReader) { + recordTypes.put(record.getType(), recordTypes.get(record.getType()) + 1); + } + } + finally { + chunkReader.close(); + } + + assertThat(recordTypes.get(Type.DESCRIPTOR), equalTo(1)); + assertThat(recordTypes.get(Type.ROOT_GROUPS), equalTo(1)); + assertThat(recordTypes.get(Type.ALL_GROUPS), equalTo(1)); + assertThat(recordTypes.get(Type.ARTIFACT_ADD), equalTo(2)); + assertThat(recordTypes.get(Type.ARTIFACT_REMOVE), equalTo(0)); + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java new file mode 100644 index 0000000..edb614a --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/DirectoryResourceHandler.java @@ -0,0 +1,72 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; + +/** + * A trivial {@link File} directory handler that does not perform any locking or extra bits, and just serves up files + * by name from specified existing directory. + */ +public class DirectoryResourceHandler + implements WritableResourceHandler +{ + private final File rootDirectory; + + public DirectoryResourceHandler(final File rootDirectory) { + if (rootDirectory == null) { + throw new NullPointerException("null rootDirectory"); + } + if (!rootDirectory.isDirectory()) { + throw new IllegalArgumentException("rootDirectory exists and is not a directory"); + } + this.rootDirectory = rootDirectory; + } + + public InputStream open(final String name) throws IOException { + return new BufferedInputStream(new FileInputStream(new File(rootDirectory, name))); + } + + public void save(final String name, final InputStream inputStream) throws IOException { + try { + final BufferedOutputStream outputStream = new BufferedOutputStream( + new FileOutputStream(new File(rootDirectory, name))); + int read; + byte[] bytes = new byte[8192]; + while ((read = inputStream.read(bytes)) != -1) { + outputStream.write(bytes, 0, read); + } + outputStream.close(); + } + finally { + inputStream.close(); + } + } + + public void close() throws IOException { + // nop + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java new file mode 100644 index 0000000..933cfbd --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/HttpResourceHandler.java @@ -0,0 +1,57 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.BufferedInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.HttpURLConnection; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.URL; + +/** + * A trivial HTTP {@link ResourceHandler} that uses {@link URL} to fetch remote content. This implementation does not + * handle any advanced cases, like redirects, authentication, etc. + */ +public class HttpResourceHandler + implements ResourceHandler +{ + private final URI root; + + public HttpResourceHandler(final URL root) throws URISyntaxException { + if (root == null) { + throw new NullPointerException("root URL null"); + } + this.root = root.toURI(); + } + + public InputStream open(final String name) throws IOException { + URL target = root.resolve(name).toURL(); + HttpURLConnection conn = (HttpURLConnection) target.openConnection(); + conn.setRequestMethod("GET"); + conn.setRequestProperty("User-Agent", "ASF Maven-Indexer-Reader/1.0"); + return new BufferedInputStream(conn.getInputStream()); + } + + public void close() throws IOException { + // nop + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java new file mode 100644 index 0000000..e5d7df1 --- /dev/null +++ b/indexer-reader/src/test/java/org/apache/maven/index/reader/IndexReaderTest.java @@ -0,0 +1,102 @@ +package org.apache.maven.index.reader; + +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.Writer; +import java.net.URL; +import java.util.Arrays; + +import org.junit.Ignore; +import org.junit.Test; + +import static org.hamcrest.core.IsEqual.equalTo; +import static org.junit.Assert.assertThat; + +/** + * UT for {@link IndexReader} + */ +public class IndexReaderTest +{ + @Test + public void simple() throws IOException { + final IndexReader indexReader = new IndexReader( + null, + new DirectoryResourceHandler(new File("src/test/resources/"))); + try { + assertThat(indexReader.getIndexId(), equalTo("apache-snapshots-local")); + assertThat(indexReader.getPublishedTimestamp().getTime(), equalTo(1243533418015L)); + assertThat(indexReader.isIncremental(), equalTo(false)); + assertThat(indexReader.getChunkNames(), equalTo(Arrays.asList("nexus-maven-repository-index.gz"))); + int chunks = 0; + int records = 0; + for (ChunkReader chunkReader : indexReader) { + chunks++; + assertThat(chunkReader.getName(), equalTo("nexus-maven-repository-index.gz")); + assertThat(chunkReader.getVersion(), equalTo(1)); + assertThat(chunkReader.getTimestamp().getTime(), equalTo(1243533418015L)); + for (Record record : chunkReader) { + records++; + } + } + + assertThat(chunks, equalTo(1)); + assertThat(records, equalTo(5)); + } + finally { + indexReader.close(); + } + } + + @Test + @Ignore("Here for example but test depending on external resource is not nice thing to have") + public void central() throws Exception { + final File tempDir = File.createTempFile("index-reader", "tmp"); + tempDir.mkdirs(); + final Writer writer = new OutputStreamWriter(System.out); + final IndexReader indexReader = new IndexReader( + new DirectoryResourceHandler(tempDir), + new HttpResourceHandler(new URL("http://repo1.maven.org/maven2/.index/";)) + ); + try { + writer.write("indexRepoId=" + indexReader.getIndexId() + "\n"); + writer.write("indexLastPublished=" + indexReader.getPublishedTimestamp() + "\n"); + writer.write("isIncremental=" + indexReader.isIncremental() + "\n"); + writer.write("indexRequiredChunkNames=" + indexReader.getChunkNames() + "\n"); + for (ChunkReader chunkReader : indexReader) { + writer.write("chunkName=" + chunkReader.getName() + "\n"); + writer.write("chunkVersion=" + chunkReader.getVersion() + "\n"); + writer.write("chunkPublished=" + chunkReader.getTimestamp() + "\n"); + writer.write("= = = = = = \n"); + for (Record record : chunkReader) { + writer.write(record.getExpanded() + "\n"); + writer.write("--------- \n"); + writer.write(record.getRaw() + "\n"); + } + } + } + finally { + indexReader.close(); + writer.close(); + } + } +} http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/resources/nexus-maven-repository-index.gz ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.gz b/indexer-reader/src/test/resources/nexus-maven-repository-index.gz new file mode 100644 index 0000000..490b21c Binary files /dev/null and b/indexer-reader/src/test/resources/nexus-maven-repository-index.gz differ http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/indexer-reader/src/test/resources/nexus-maven-repository-index.properties ---------------------------------------------------------------------- diff --git a/indexer-reader/src/test/resources/nexus-maven-repository-index.properties b/indexer-reader/src/test/resources/nexus-maven-repository-index.properties new file mode 100644 index 0000000..d1f1b48 --- /dev/null +++ b/indexer-reader/src/test/resources/nexus-maven-repository-index.properties @@ -0,0 +1,6 @@ +#Thu May 28 14:56:58 BRT 2009 +nexus.index.time=20090528175658.015 +0000 +nexus.index.chain-id=1243533418968 +nexus.index.id=apache-snapshots-local +nexus.index.timestamp=20090528175658.015 +0000 +nexus.index.last-incremental=0 http://git-wip-us.apache.org/repos/asf/maven-indexer/blob/af8783d8/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cbeeecc..369fe42 100644 --- a/pom.xml +++ b/pom.xml @@ -160,6 +160,7 @@ under the License. <module>indexer-core</module> <module>indexer-artifact</module> <module>indexer-cli</module> + <module>indexer-reader</module> </modules> <build>
