Attachment 192837 Details for Bug 334699 – Proposed Patch

Some Eclipse Foundation services are deprecated, or will be soon. Please ensure you've read this important communication.

[patch] Proposed Patch

bug334699.txt (text/plain), 7.49 KB, created by Jay Arthanareeswaran

on 2011-04-08 10:18:36 EDT

(hide)

Description:

Filename:

MIME Type:

Creator: Jay Arthanareeswaran

Created: 2011-04-08 10:18:36 EDT

Size: 7.49 KB

patch

obsolete

>From 2300dcdfa9f539ddffbfb6bf47e41e4a0025f649 Mon Sep 17 00:00:00 2001
>From: Jayaprakash A <jarthana@in.ibm.com>
>Date: Fri, 8 Apr 2011 19:42:47 +0530
>Subject: [PATCH] Fix to bug 334699
>
>---
> .../internal/server/search/IndexPurgeJob.java      |  153 ++++++++++++++++++++
> .../internal/server/search/SearchActivator.java    |    9 ++
> 2 files changed, 162 insertions(+), 0 deletions(-)
> create mode 100644 bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java
>
>diff --git a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java
>new file mode 100644
>index 0000000..b09c4dc
>--- /dev/null
>+++ b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java
>@@ -0,0 +1,153 @@
>+/*******************************************************************************
>+ * Copyright (c) 2010, 2011 IBM Corporation and others.
>+ * All rights reserved. This program and the accompanying materials
>+ * are made available under the terms of the Eclipse Public License v1.0
>+ * which accompanies this distribution, and is available at
>+ * http://www.eclipse.org/legal/epl-v10.html
>+ * 
>+ * Contributors:
>+ *     IBM Corporation - initial API and implementation
>+ *******************************************************************************/
>+package org.eclipse.orion.internal.server.search;
>+
>+import java.io.IOException;
>+import java.net.URI;
>+import java.util.ArrayList;
>+import java.util.Iterator;
>+import java.util.List;
>+
>+import org.apache.solr.client.solrj.SolrQuery;
>+import org.apache.solr.client.solrj.SolrServer;
>+import org.apache.solr.client.solrj.SolrServerException;
>+import org.apache.solr.client.solrj.response.QueryResponse;
>+import org.apache.solr.common.SolrDocument;
>+import org.apache.solr.common.SolrDocumentList;
>+import org.apache.solr.common.params.CommonParams;
>+import org.eclipse.core.filesystem.EFS;
>+import org.eclipse.core.filesystem.IFileStore;
>+import org.eclipse.core.filesystem.URIUtil;
>+import org.eclipse.core.runtime.IProgressMonitor;
>+import org.eclipse.core.runtime.IStatus;
>+import org.eclipse.core.runtime.OperationCanceledException;
>+import org.eclipse.core.runtime.Status;
>+import org.eclipse.core.runtime.jobs.Job;
>+import org.eclipse.orion.internal.server.servlets.ProtocolConstants;
>+import org.eclipse.orion.server.core.LogHelper;
>+import org.slf4j.Logger;
>+import org.slf4j.LoggerFactory;
>+
>+/**
>+ * The IndexPurgeJob is responsible for cleaning up the indexes that that are
>+ * no longer required. The job crawls the indexes and purges those whose corresponding
>+ * resources no longer present in the file system.
>+ * 
>+ */
>+public class IndexPurgeJob extends Job {
>+
>+	private static final long DEFAULT_DELAY = 180000;//3 minutes
>+	private static final long PAGE_SIZE = 1000;
>+	private final SolrServer server;
>+	private SolrQuery findAllQuery = null;
>+
>+	public IndexPurgeJob(SolrServer server) {
>+		super("Purging Index");
>+		this.server = server;
>+		this.findAllQuery = findAllQuery();
>+		setSystem(true);
>+	}
>+
>+	public void ensureUpdated() {
>+		schedule(DEFAULT_DELAY);
>+	}
>+
>+	@Override
>+	protected IStatus run(IProgressMonitor monitor) {
>+		Logger logger = LoggerFactory.getLogger(Indexer.class);
>+		if (logger.isDebugEnabled())
>+			logger.debug("Purging indexes"); //$NON-NLS-1$
>+		long start = System.currentTimeMillis();
>+		SolrQuery query = findAllQuery();
>+		try {
>+			QueryResponse solrResponse = this.server.query(findAllQuery);
>+			SolrDocumentList result = solrResponse.getResults();
>+			long numFound = result.getNumFound();
>+			long processed = 0;
>+			List<String> listIds = new ArrayList<String>();
>+			if (numFound > processed) {
>+				while (true) {
>+					checkCanceled(monitor);
>+					markStaleIndexes(result, listIds);
>+					processed += PAGE_SIZE;
>+					if (processed >= numFound)
>+						break;
>+					query.setParam(CommonParams.START, "" + processed);
>+					solrResponse = this.server.query(query);
>+					result = solrResponse.getResults();
>+					// New indexes may have been added, perhaps
>+					numFound = result.getNumFound();
>+				}
>+			}
>+
>+			checkCanceled(monitor);
>+			if (listIds.size() > 0) {
>+				this.server.deleteById(listIds);
>+				this.server.commit();
>+			}
>+			if (logger.isDebugEnabled())
>+				logger.debug("\tPurged: " + listIds.size()); //$NON-NLS-1$
>+
>+		} catch (Exception e) {
>+			handleIndexingFailure(e);
>+		}
>+		long duration = System.currentTimeMillis() - start;
>+		if (logger.isDebugEnabled())
>+			logger.debug("Purge job took " + duration + "ms"); //$NON-NLS-1$ //$NON-NLS-3$
>+
>+		long delay = Math.max(DEFAULT_DELAY, duration);
>+		schedule(delay);
>+		return Status.OK_STATUS;
>+	}
>+
>+	private void markStaleIndexes(SolrDocumentList list, List<String> listIds) throws IOException, SolrServerException {
>+		Iterator<SolrDocument> iterator = list.iterator();
>+		while (iterator.hasNext()) {
>+			SolrDocument doc = iterator.next();
>+			URI uri;
>+			try {
>+				uri = new URI((String) doc.getFieldValue(ProtocolConstants.KEY_ID));
>+				IFileStore file = null;
>+
>+				if (uri.isAbsolute()) {
>+					file = EFS.getLocalFileSystem().getStore(URIUtil.toPath(uri));
>+				} else {
>+					file = EFS.getStore(uri);
>+				}
>+
>+				if (!file.fetchInfo().exists())
>+					listIds.add((String) doc.getFieldValue(ProtocolConstants.KEY_ID));
>+
>+			} catch (Exception e) {
>+				handleIndexingFailure(e);
>+				continue;
>+			}
>+		}
>+	}
>+
>+	private SolrQuery findAllQuery() {
>+		SolrQuery query = new SolrQuery();
>+		query.setParam(CommonParams.ROWS, "" + PAGE_SIZE);
>+		String queryString = "*:*";
>+		query.setQuery(queryString);
>+		return query;
>+	}
>+
>+	private void checkCanceled(IProgressMonitor monitor) {
>+		if (monitor.isCanceled())
>+			throw new OperationCanceledException();
>+	}
>+
>+	private void handleIndexingFailure(Throwable t) {
>+		LogHelper.log(new Status(IStatus.ERROR, SearchActivator.PI_SEARCH, "Error during search index purge", t)); //$NON-NLS-1$
>+	}
>+
>+}
>diff --git a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java
>index b3741cf..650661a 100644
>--- a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java
>+++ b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java
>@@ -59,6 +59,7 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator {
> 	private static SearchActivator instance;
> 	public static final String PI_SEARCH = "org.eclipse.orion.server.core.search"; //$NON-NLS-1$
> 	private Indexer indexer;
>+	private IndexPurgeJob purgeJob;
> 	private ServiceRegistration<IWebResourceDecorator> searchDecoratorRegistration;
> 	private SolrServer server;
> 	private SolrCore solrCore;
>@@ -208,6 +209,9 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator {
> 		if (server != null) {
> 			indexer = new Indexer(server);
> 			indexer.schedule();
>+
>+			purgeJob = new IndexPurgeJob(server);
>+			purgeJob.schedule();
> 		}
> 		searchDecoratorRegistration = context.registerService(IWebResourceDecorator.class, this, null);
> 	}
>@@ -229,6 +233,11 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator {
> 			indexer.join();
> 			indexer = null;
> 		}
>+		if (purgeJob != null) {
>+			purgeJob.cancel();
>+			purgeJob.join();
>+			purgeJob = null;
>+		}
> 		SearchActivator.context = null;
> 	}
> 
>-- 
>1.7.3.1.msysgit.0
>

Flags:

jarthana: review?

Actions: View | Diff

Attachments on bug 334699: 192837