gitblit/src/main/java/com/gitblit/tickets/TicketIndexer.java

/*
 * Copyright 2014 gitblit.com.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package com.gitblit.tickets;

import java.io.File;
import java.io.IOException;
import java.text.MessageFormat;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.IntField;
import org.apache.lucene.document.LongField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.queryparser.classic.QueryParser;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.SortField.Type;
import org.apache.lucene.search.TopFieldDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import com.gitblit.Keys;
import com.gitblit.manager.IRuntimeManager;
import com.gitblit.models.RepositoryModel;
import com.gitblit.models.TicketModel;
import com.gitblit.models.TicketModel.Attachment;
import com.gitblit.models.TicketModel.Patchset;
import com.gitblit.models.TicketModel.Status;
import com.gitblit.utils.FileUtils;
import com.gitblit.utils.StringUtils;

/**
 * Indexes tickets in a Lucene database.
 *
 * @author James Moger
 *
 */
public class TicketIndexer {

	/**
	 * Fields in the Lucene index
	 */
	public static enum Lucene {

		rid(Type.STRING),
		did(Type.STRING),
		project(Type.STRING),
		repository(Type.STRING),
		number(Type.LONG),
		title(Type.STRING),
		body(Type.STRING),
		topic(Type.STRING),
		created(Type.LONG),
		createdby(Type.STRING),
		updated(Type.LONG),
		updatedby(Type.STRING),
		responsible(Type.STRING),
		milestone(Type.STRING),
		status(Type.STRING),
		type(Type.STRING),
		labels(Type.STRING),
		participants(Type.STRING),
		watchedby(Type.STRING),
		mentions(Type.STRING),
		attachments(Type.INT),
		content(Type.STRING),
		patchset(Type.STRING),
		comments(Type.INT),
		mergesha(Type.STRING),
		mergeto(Type.STRING),
		patchsets(Type.INT),
		votes(Type.INT),
		//NOTE: Indexing on the underlying value to allow flexibility on naming
		priority(Type.INT),
		severity(Type.INT);

		final Type fieldType;

		Lucene(Type fieldType) {
			this.fieldType = fieldType;
		}

		public String colon() {
			return name() + ":";
		}

		public String matches(String value) {
			if (StringUtils.isEmpty(value)) {
				return "";
			}
			boolean not = value.charAt(0) == '!';
			if (not) {
				return "!" + name() + ":" + escape(value.substring(1));
			}
			return name() + ":" + escape(value);
		}

		public String doesNotMatch(String value) {
			if (StringUtils.isEmpty(value)) {
				return "";
			}
			return "NOT " + name() + ":" + escape(value);
		}

		public String isNotNull() {
			return matches("[* TO *]");
		}

		public SortField asSortField(boolean descending) {
			return new SortField(name(), fieldType, descending);
		}

		private String escape(String value) {
			if (value.charAt(0) != '"') {
				for (char c : value.toCharArray()) {
					if (!Character.isLetterOrDigit(c)) {
						return "\"" + value + "\"";
					}
				}
			}
			return value;
		}

		public static Lucene fromString(String value) {
			for (Lucene field : values()) {
				if (field.name().equalsIgnoreCase(value)) {
					return field;
				}
			}
			return created;
		}
	}

	private final Logger log = LoggerFactory.getLogger(getClass());

	private final Version luceneVersion = Version.LUCENE_5_2_1;

	private final File luceneDir;

	private IndexWriter writer;

	private IndexSearcher searcher;

	public TicketIndexer(IRuntimeManager runtimeManager) {
		this.luceneDir = runtimeManager.getFileOrFolder(Keys.tickets.indexFolder, "${baseFolder}/tickets/lucene");
	}

	/**
	 * Close all writers and searchers used by the ticket indexer.
	 */
	public void close() {
		closeSearcher();
		closeWriter();
	}

	/**
	 * Deletes the entire ticket index for all repositories.
	 */
	public void deleteAll() {
		close();
		FileUtils.delete(luceneDir);
	}

	/**
	 * Deletes all tickets for the the repository from the index.
	 */
	public boolean deleteAll(RepositoryModel repository) {
		try {
			IndexWriter writer = getWriter();
			StandardAnalyzer analyzer = new StandardAnalyzer();
			QueryParser qp = new QueryParser(Lucene.rid.name(), analyzer);
			BooleanQuery query = new BooleanQuery();
			query.add(qp.parse(repository.getRID()), Occur.MUST);

			int numDocsBefore = writer.numDocs();
			writer.deleteDocuments(query);
			writer.commit();
			closeSearcher();
			int numDocsAfter = writer.numDocs();
			if (numDocsBefore == numDocsAfter) {
				log.debug(MessageFormat.format("no records found to delete in {0}", repository));
				return false;
			} else {
				log.debug(MessageFormat.format("deleted {0} records in {1}", numDocsBefore - numDocsAfter, repository));
				return true;
			}
		} catch (Exception e) {
			log.error("error", e);
		}
		return false;
	}

	/**
	 * Bulk Add/Update tickets in the Lucene index
	 *
	 * @param tickets
	 */
	public void index(List<TicketModel> tickets) {
		try {
			IndexWriter writer = getWriter();
			for (TicketModel ticket : tickets) {
				Document doc = ticketToDoc(ticket);
				writer.addDocument(doc);
			}
			writer.commit();
			closeSearcher();
		} catch (Exception e) {
			log.error("error", e);
		}
	}

	/**
	 * Add/Update a ticket in the Lucene index
	 *
	 * @param ticket
	 */
	public void index(TicketModel ticket) {
		try {
			IndexWriter writer = getWriter();
			delete(ticket.repository, ticket.number, writer);
			Document doc = ticketToDoc(ticket);
			writer.addDocument(doc);
			writer.commit();
			closeSearcher();
		} catch (Exception e) {
			log.error("error", e);
		}
	}

	/**
	 * Delete a ticket from the Lucene index.
	 *
	 * @param ticket
	 * @throws Exception
	 * @return true, if deleted, false if no record was deleted
	 */
	public boolean delete(TicketModel ticket) {
		try {
			IndexWriter writer = getWriter();
			return delete(ticket.repository, ticket.number, writer);
		} catch (Exception e) {
			log.error("Failed to delete ticket " + ticket.number, e);
		}
		return false;
	}

	/**
	 * Delete a ticket from the Lucene index.
	 *
	 * @param repository
	 * @param ticketId
	 * @throws Exception
	 * @return true, if deleted, false if no record was deleted
	 */
	private boolean delete(String repository, long ticketId, IndexWriter writer) throws Exception {
		StandardAnalyzer analyzer = new StandardAnalyzer();
		QueryParser qp = new QueryParser(Lucene.did.name(), analyzer);
		BooleanQuery query = new BooleanQuery();
		query.add(qp.parse(StringUtils.getSHA1(repository + ticketId)), Occur.MUST);

		int numDocsBefore = writer.numDocs();
		writer.deleteDocuments(query);
		writer.commit();
		closeSearcher();
		int numDocsAfter = writer.numDocs();
		if (numDocsBefore == numDocsAfter) {
			log.debug(MessageFormat.format("no records found to delete in {0}", repository));
			return false;
		} else {
			log.debug(MessageFormat.format("deleted {0} records in {1}", numDocsBefore - numDocsAfter, repository));
			return true;
		}
	}

	/**
	 * Returns true if the repository has tickets in the index.
	 *
	 * @param repository
	 * @return true if there are indexed tickets
	 */
	public boolean hasTickets(RepositoryModel repository) {
		return !queryFor(Lucene.rid.matches(repository.getRID()), 1, 0, null, true).isEmpty();
	}

	/**
	 * Search for tickets matching the query.  The returned tickets are
	 * shadows of the real ticket, but suitable for a results list.
	 *
	 * @param repository
	 * @param text
	 * @param page
	 * @param pageSize
	 * @return search results
	 */
	public List<QueryResult> searchFor(RepositoryModel repository, String text, int page, int pageSize) {
		if (StringUtils.isEmpty(text)) {
			return Collections.emptyList();
		}
		Set<QueryResult> results = new LinkedHashSet<QueryResult>();
		StandardAnalyzer analyzer = new StandardAnalyzer();
		try {
			// search the title, description and content
			BooleanQuery query = new BooleanQuery();
			QueryParser qp;

			qp = new QueryParser(Lucene.title.name(), analyzer);
			qp.setAllowLeadingWildcard(true);
			query.add(qp.parse(text), Occur.SHOULD);

			qp = new QueryParser(Lucene.body.name(), analyzer);
			qp.setAllowLeadingWildcard(true);
			query.add(qp.parse(text), Occur.SHOULD);

			qp = new QueryParser(Lucene.content.name(), analyzer);
			qp.setAllowLeadingWildcard(true);
			query.add(qp.parse(text), Occur.SHOULD);

			IndexSearcher searcher = getSearcher();
			Query rewrittenQuery = searcher.rewrite(query);

			log.debug(rewrittenQuery.toString());

			TopScoreDocCollector collector = TopScoreDocCollector.create(5000);
			searcher.search(rewrittenQuery, collector);
			int offset = Math.max(0, (page - 1) * pageSize);
			ScoreDoc[] hits = collector.topDocs(offset, pageSize).scoreDocs;
			for (int i = 0; i < hits.length; i++) {
				int docId = hits[i].doc;
				Document doc = searcher.doc(docId);
				QueryResult result = docToQueryResult(doc);
				if (repository != null) {
					if (!result.repository.equalsIgnoreCase(repository.name)) {
						continue;
					}
				}
				results.add(result);
			}
		} catch (Exception e) {
			log.error(MessageFormat.format("Exception while searching for {0}", text), e);
		}
		return new ArrayList<QueryResult>(results);
	}

	/**
	 * Search for tickets matching the query.  The returned tickets are
	 * shadows of the real ticket, but suitable for a results list.
	 *
	 * @param text
	 * @param page
	 * @param pageSize
	 * @param sortBy
	 * @param desc
	 * @return
	 */
	public List<QueryResult> queryFor(String queryText, int page, int pageSize, String sortBy, boolean desc) {
		if (StringUtils.isEmpty(queryText)) {
			return Collections.emptyList();
		}

		Set<QueryResult> results = new LinkedHashSet<QueryResult>();
		StandardAnalyzer analyzer = new StandardAnalyzer();
		try {
			QueryParser qp = new QueryParser(Lucene.content.name(), analyzer);
			Query query = qp.parse(queryText);

			IndexSearcher searcher = getSearcher();
			Query rewrittenQuery = searcher.rewrite(query);

			log.debug(rewrittenQuery.toString());

			Sort sort;
			if (sortBy == null) {
				sort = new Sort(Lucene.created.asSortField(desc));
			} else {
				sort = new Sort(Lucene.fromString(sortBy).asSortField(desc));
			}
			int maxSize = 5000;
			TopFieldDocs docs = searcher.search(rewrittenQuery, null, maxSize, sort, false, false);
			int size = (pageSize <= 0) ? maxSize : pageSize;
			int offset = Math.max(0, (page - 1) * size);
			ScoreDoc[] hits = subset(docs.scoreDocs, offset, size);
			for (int i = 0; i < hits.length; i++) {
				int docId = hits[i].doc;
				Document doc = searcher.doc(docId);
				QueryResult result = docToQueryResult(doc);
				result.docId = docId;
				result.totalResults = docs.totalHits;
				results.add(result);
			}
		} catch (Exception e) {
			log.error(MessageFormat.format("Exception while searching for {0}", queryText), e);
		}
		return new ArrayList<QueryResult>(results);
	}

	private ScoreDoc [] subset(ScoreDoc [] docs, int offset, int size) {
		if (docs.length >= (offset + size)) {
			ScoreDoc [] set = new ScoreDoc[size];
			System.arraycopy(docs, offset, set, 0, set.length);
			return set;
		} else if (docs.length >= offset) {
			ScoreDoc [] set = new ScoreDoc[docs.length - offset];
			System.arraycopy(docs, offset, set, 0, set.length);
			return set;
		} else {
			return new ScoreDoc[0];
		}
	}

	private IndexWriter getWriter() throws IOException {
		if (writer == null) {
			Directory directory = FSDirectory.open(luceneDir.toPath());

			if (!luceneDir.exists()) {
				luceneDir.mkdirs();
			}

			StandardAnalyzer analyzer = new StandardAnalyzer();
			IndexWriterConfig config = new IndexWriterConfig(analyzer);
			config.setOpenMode(OpenMode.CREATE_OR_APPEND);
			writer = new IndexWriter(directory, config);
		}
		return writer;
	}

	private synchronized void closeWriter() {
		try {
			if (writer != null) {
				writer.close();
			}
		} catch (Exception e) {
			log.error("failed to close writer!", e);
		} finally {
			writer = null;
		}
	}

	private IndexSearcher getSearcher() throws IOException {
		if (searcher == null) {
			searcher = new IndexSearcher(DirectoryReader.open(getWriter(), true));
		}
		return searcher;
	}

	private synchronized void closeSearcher() {
		try {
			if (searcher != null) {
				searcher.getIndexReader().close();
			}
		} catch (Exception e) {
			log.error("failed to close searcher!", e);
		} finally {
			searcher = null;
		}
	}

	/**
	 * Creates a Lucene document from a ticket.
	 *
	 * @param ticket
	 * @return a Lucene document
	 */
	private Document ticketToDoc(TicketModel ticket) {
		Document doc = new Document();
		// repository and document ids for Lucene querying
		toDocField(doc, Lucene.rid, StringUtils.getSHA1(ticket.repository));
		toDocField(doc, Lucene.did, StringUtils.getSHA1(ticket.repository + ticket.number));

		toDocField(doc, Lucene.project, ticket.project);
		toDocField(doc, Lucene.repository, ticket.repository);
		toDocField(doc, Lucene.number, ticket.number);
		toDocField(doc, Lucene.title, ticket.title);
		toDocField(doc, Lucene.body, ticket.body);
		toDocField(doc, Lucene.created, ticket.created);
		toDocField(doc, Lucene.createdby, ticket.createdBy);
		toDocField(doc, Lucene.updated, ticket.updated);
		toDocField(doc, Lucene.updatedby, ticket.updatedBy);
		toDocField(doc, Lucene.responsible, ticket.responsible);
		toDocField(doc, Lucene.milestone, ticket.milestone);
		toDocField(doc, Lucene.topic, ticket.topic);
		toDocField(doc, Lucene.status, ticket.status.name());
		toDocField(doc, Lucene.comments, ticket.getComments().size());
		toDocField(doc, Lucene.type, ticket.type == null ? null : ticket.type.name());
		toDocField(doc, Lucene.mergesha, ticket.mergeSha);
		toDocField(doc, Lucene.mergeto, ticket.mergeTo);
		toDocField(doc, Lucene.labels, StringUtils.flattenStrings(ticket.getLabels(), ";").toLowerCase());
		toDocField(doc, Lucene.participants, StringUtils.flattenStrings(ticket.getParticipants(), ";").toLowerCase());
		toDocField(doc, Lucene.watchedby, StringUtils.flattenStrings(ticket.getWatchers(), ";").toLowerCase());
		toDocField(doc, Lucene.mentions, StringUtils.flattenStrings(ticket.getMentions(), ";").toLowerCase());
		toDocField(doc, Lucene.votes, ticket.getVoters().size());
		toDocField(doc, Lucene.priority, ticket.priority.getValue());
		toDocField(doc, Lucene.severity, ticket.severity.getValue());

		List<String> attachments = new ArrayList<String>();
		for (Attachment attachment : ticket.getAttachments()) {
			attachments.add(attachment.name.toLowerCase());
		}
		toDocField(doc, Lucene.attachments, StringUtils.flattenStrings(attachments, ";"));

		List<Patchset> patches = ticket.getPatchsets();
		if (!patches.isEmpty()) {
			toDocField(doc, Lucene.patchsets, patches.size());
			Patchset patchset = patches.get(patches.size() - 1);
			String flat =
					patchset.number + ":" +
					patchset.rev + ":" +
					patchset.tip + ":" +
					patchset.base + ":" +
					patchset.commits;
			doc.add(new org.apache.lucene.document.Field(Lucene.patchset.name(), flat, TextField.TYPE_STORED));
		}

		doc.add(new TextField(Lucene.content.name(), ticket.toIndexableString(), Store.NO));

		return doc;
	}

	private void toDocField(Document doc, Lucene lucene, Date value) {
		if (value == null) {
			return;
		}
		doc.add(new LongField(lucene.name(), value.getTime(), Store.YES));
	}

	private void toDocField(Document doc, Lucene lucene, long value) {
		doc.add(new LongField(lucene.name(), value, Store.YES));
	}

	private void toDocField(Document doc, Lucene lucene, int value) {
		doc.add(new IntField(lucene.name(), value, Store.YES));
	}

	private void toDocField(Document doc, Lucene lucene, String value) {
		if (StringUtils.isEmpty(value)) {
			return;
		}
		doc.add(new org.apache.lucene.document.Field(lucene.name(), value, TextField.TYPE_STORED));
	}

	/**
	 * Creates a query result from the Lucene document.  This result is
	 * not a high-fidelity representation of the real ticket, but it is
	 * suitable for display in a table of search results.
	 *
	 * @param doc
	 * @return a query result
	 * @throws ParseException
	 */
	private QueryResult docToQueryResult(Document doc) throws ParseException {
		QueryResult result = new QueryResult();
		result.project = unpackString(doc, Lucene.project);
		result.repository = unpackString(doc, Lucene.repository);
		result.number = unpackLong(doc, Lucene.number);
		result.createdBy = unpackString(doc, Lucene.createdby);
		result.createdAt = unpackDate(doc, Lucene.created);
		result.updatedBy = unpackString(doc, Lucene.updatedby);
		result.updatedAt = unpackDate(doc, Lucene.updated);
		result.title = unpackString(doc, Lucene.title);
		result.body = unpackString(doc, Lucene.body);
		result.status = Status.fromObject(unpackString(doc, Lucene.status), Status.New);
		result.responsible = unpackString(doc, Lucene.responsible);
		result.milestone = unpackString(doc, Lucene.milestone);
		result.topic = unpackString(doc, Lucene.topic);
		result.type = TicketModel.Type.fromObject(unpackString(doc, Lucene.type), TicketModel.Type.defaultType);
		result.mergeSha = unpackString(doc, Lucene.mergesha);
		result.mergeTo = unpackString(doc, Lucene.mergeto);
		result.commentsCount = unpackInt(doc, Lucene.comments);
		result.votesCount = unpackInt(doc, Lucene.votes);
		result.attachments = unpackStrings(doc, Lucene.attachments);
		result.labels = unpackStrings(doc, Lucene.labels);
		result.participants = unpackStrings(doc, Lucene.participants);
		result.watchedby = unpackStrings(doc, Lucene.watchedby);
		result.mentions = unpackStrings(doc, Lucene.mentions);
		result.priority = TicketModel.Priority.fromObject(unpackInt(doc, Lucene.priority), TicketModel.Priority.defaultPriority);
		result.severity = TicketModel.Severity.fromObject(unpackInt(doc, Lucene.severity), TicketModel.Severity.defaultSeverity);

		if (!StringUtils.isEmpty(doc.get(Lucene.patchset.name()))) {
			// unpack most recent patchset
			String [] values = doc.get(Lucene.patchset.name()).split(":", 5);

			Patchset patchset = new Patchset();
			patchset.number = Integer.parseInt(values[0]);
			patchset.rev = Integer.parseInt(values[1]);
			patchset.tip = values[2];
			patchset.base = values[3];
			patchset.commits = Integer.parseInt(values[4]);

			result.patchset = patchset;
		}

		return result;
	}

	private String unpackString(Document doc, Lucene lucene) {
		return doc.get(lucene.name());
	}

	private List<String> unpackStrings(Document doc, Lucene lucene) {
		if (!StringUtils.isEmpty(doc.get(lucene.name()))) {
			return StringUtils.getStringsFromValue(doc.get(lucene.name()), ";");
		}
		return null;
	}

	private Date unpackDate(Document doc, Lucene lucene) {
		String val = doc.get(lucene.name());
		if (!StringUtils.isEmpty(val)) {
			long time = Long.parseLong(val);
			Date date = new Date(time);
			return date;
		}
		return null;
	}

	private long unpackLong(Document doc, Lucene lucene) {
		String val = doc.get(lucene.name());
		if (StringUtils.isEmpty(val)) {
			return 0;
		}
		long l = Long.parseLong(val);
		return l;
	}

	private int unpackInt(Document doc, Lucene lucene) {
		String val = doc.get(lucene.name());
		if (StringUtils.isEmpty(val)) {
			return 0;
		}
		int i = Integer.parseInt(val);
		return i;
	}
}