Download
Getting Started
Members
Projects
Community
Marketplace
Events
Planet Eclipse
Newsletter
Videos
Participate
Report a Bug
Forums
Mailing Lists
Wiki
IRC
How to Contribute
Working Groups
Automotive
Internet of Things
LocationTech
Long-Term Support
PolarSys
Science
OpenMDM
More
Community
Marketplace
Events
Planet Eclipse
Newsletter
Videos
Participate
Report a Bug
Forums
Mailing Lists
Wiki
IRC
How to Contribute
Working Groups
Automotive
Internet of Things
LocationTech
Long-Term Support
PolarSys
Science
OpenMDM
Toggle navigation
Bugzilla – Attachment 192837 Details for
Bug 334699
[search] index holds stale references to external locations that have been deleted
Home
|
New
|
Browse
|
Search
|
[?]
|
Reports
|
Requests
|
Help
|
Log In
[x]
|
Terms of Use
|
Copyright Agent
Some Eclipse Foundation services are deprecated, or will be soon. Please ensure you've read
this important communication.
[patch]
Proposed Patch
bug334699.txt (text/plain), 7.49 KB, created by
Jay Arthanareeswaran
on 2011-04-08 10:18:36 EDT
(
hide
)
Description:
Proposed Patch
Filename:
MIME Type:
Creator:
Jay Arthanareeswaran
Created:
2011-04-08 10:18:36 EDT
Size:
7.49 KB
patch
obsolete
>From 2300dcdfa9f539ddffbfb6bf47e41e4a0025f649 Mon Sep 17 00:00:00 2001 >From: Jayaprakash A <jarthana@in.ibm.com> >Date: Fri, 8 Apr 2011 19:42:47 +0530 >Subject: [PATCH] Fix to bug 334699 > >--- > .../internal/server/search/IndexPurgeJob.java | 153 ++++++++++++++++++++ > .../internal/server/search/SearchActivator.java | 9 ++ > 2 files changed, 162 insertions(+), 0 deletions(-) > create mode 100644 bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java > >diff --git a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java >new file mode 100644 >index 0000000..b09c4dc >--- /dev/null >+++ b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/IndexPurgeJob.java >@@ -0,0 +1,153 @@ >+/******************************************************************************* >+ * Copyright (c) 2010, 2011 IBM Corporation and others. >+ * All rights reserved. This program and the accompanying materials >+ * are made available under the terms of the Eclipse Public License v1.0 >+ * which accompanies this distribution, and is available at >+ * http://www.eclipse.org/legal/epl-v10.html >+ * >+ * Contributors: >+ * IBM Corporation - initial API and implementation >+ *******************************************************************************/ >+package org.eclipse.orion.internal.server.search; >+ >+import java.io.IOException; >+import java.net.URI; >+import java.util.ArrayList; >+import java.util.Iterator; >+import java.util.List; >+ >+import org.apache.solr.client.solrj.SolrQuery; >+import org.apache.solr.client.solrj.SolrServer; >+import org.apache.solr.client.solrj.SolrServerException; >+import org.apache.solr.client.solrj.response.QueryResponse; >+import org.apache.solr.common.SolrDocument; >+import org.apache.solr.common.SolrDocumentList; >+import org.apache.solr.common.params.CommonParams; >+import org.eclipse.core.filesystem.EFS; >+import org.eclipse.core.filesystem.IFileStore; >+import org.eclipse.core.filesystem.URIUtil; >+import org.eclipse.core.runtime.IProgressMonitor; >+import org.eclipse.core.runtime.IStatus; >+import org.eclipse.core.runtime.OperationCanceledException; >+import org.eclipse.core.runtime.Status; >+import org.eclipse.core.runtime.jobs.Job; >+import org.eclipse.orion.internal.server.servlets.ProtocolConstants; >+import org.eclipse.orion.server.core.LogHelper; >+import org.slf4j.Logger; >+import org.slf4j.LoggerFactory; >+ >+/** >+ * The IndexPurgeJob is responsible for cleaning up the indexes that that are >+ * no longer required. The job crawls the indexes and purges those whose corresponding >+ * resources no longer present in the file system. >+ * >+ */ >+public class IndexPurgeJob extends Job { >+ >+ private static final long DEFAULT_DELAY = 180000;//3 minutes >+ private static final long PAGE_SIZE = 1000; >+ private final SolrServer server; >+ private SolrQuery findAllQuery = null; >+ >+ public IndexPurgeJob(SolrServer server) { >+ super("Purging Index"); >+ this.server = server; >+ this.findAllQuery = findAllQuery(); >+ setSystem(true); >+ } >+ >+ public void ensureUpdated() { >+ schedule(DEFAULT_DELAY); >+ } >+ >+ @Override >+ protected IStatus run(IProgressMonitor monitor) { >+ Logger logger = LoggerFactory.getLogger(Indexer.class); >+ if (logger.isDebugEnabled()) >+ logger.debug("Purging indexes"); //$NON-NLS-1$ >+ long start = System.currentTimeMillis(); >+ SolrQuery query = findAllQuery(); >+ try { >+ QueryResponse solrResponse = this.server.query(findAllQuery); >+ SolrDocumentList result = solrResponse.getResults(); >+ long numFound = result.getNumFound(); >+ long processed = 0; >+ List<String> listIds = new ArrayList<String>(); >+ if (numFound > processed) { >+ while (true) { >+ checkCanceled(monitor); >+ markStaleIndexes(result, listIds); >+ processed += PAGE_SIZE; >+ if (processed >= numFound) >+ break; >+ query.setParam(CommonParams.START, "" + processed); >+ solrResponse = this.server.query(query); >+ result = solrResponse.getResults(); >+ // New indexes may have been added, perhaps >+ numFound = result.getNumFound(); >+ } >+ } >+ >+ checkCanceled(monitor); >+ if (listIds.size() > 0) { >+ this.server.deleteById(listIds); >+ this.server.commit(); >+ } >+ if (logger.isDebugEnabled()) >+ logger.debug("\tPurged: " + listIds.size()); //$NON-NLS-1$ >+ >+ } catch (Exception e) { >+ handleIndexingFailure(e); >+ } >+ long duration = System.currentTimeMillis() - start; >+ if (logger.isDebugEnabled()) >+ logger.debug("Purge job took " + duration + "ms"); //$NON-NLS-1$ //$NON-NLS-3$ >+ >+ long delay = Math.max(DEFAULT_DELAY, duration); >+ schedule(delay); >+ return Status.OK_STATUS; >+ } >+ >+ private void markStaleIndexes(SolrDocumentList list, List<String> listIds) throws IOException, SolrServerException { >+ Iterator<SolrDocument> iterator = list.iterator(); >+ while (iterator.hasNext()) { >+ SolrDocument doc = iterator.next(); >+ URI uri; >+ try { >+ uri = new URI((String) doc.getFieldValue(ProtocolConstants.KEY_ID)); >+ IFileStore file = null; >+ >+ if (uri.isAbsolute()) { >+ file = EFS.getLocalFileSystem().getStore(URIUtil.toPath(uri)); >+ } else { >+ file = EFS.getStore(uri); >+ } >+ >+ if (!file.fetchInfo().exists()) >+ listIds.add((String) doc.getFieldValue(ProtocolConstants.KEY_ID)); >+ >+ } catch (Exception e) { >+ handleIndexingFailure(e); >+ continue; >+ } >+ } >+ } >+ >+ private SolrQuery findAllQuery() { >+ SolrQuery query = new SolrQuery(); >+ query.setParam(CommonParams.ROWS, "" + PAGE_SIZE); >+ String queryString = "*:*"; >+ query.setQuery(queryString); >+ return query; >+ } >+ >+ private void checkCanceled(IProgressMonitor monitor) { >+ if (monitor.isCanceled()) >+ throw new OperationCanceledException(); >+ } >+ >+ private void handleIndexingFailure(Throwable t) { >+ LogHelper.log(new Status(IStatus.ERROR, SearchActivator.PI_SEARCH, "Error during search index purge", t)); //$NON-NLS-1$ >+ } >+ >+} >diff --git a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java >index b3741cf..650661a 100644 >--- a/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java >+++ b/bundles/org.eclipse.orion.server.search/src/org/eclipse/orion/internal/server/search/SearchActivator.java >@@ -59,6 +59,7 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator { > private static SearchActivator instance; > public static final String PI_SEARCH = "org.eclipse.orion.server.core.search"; //$NON-NLS-1$ > private Indexer indexer; >+ private IndexPurgeJob purgeJob; > private ServiceRegistration<IWebResourceDecorator> searchDecoratorRegistration; > private SolrServer server; > private SolrCore solrCore; >@@ -208,6 +209,9 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator { > if (server != null) { > indexer = new Indexer(server); > indexer.schedule(); >+ >+ purgeJob = new IndexPurgeJob(server); >+ purgeJob.schedule(); > } > searchDecoratorRegistration = context.registerService(IWebResourceDecorator.class, this, null); > } >@@ -229,6 +233,11 @@ public class SearchActivator implements BundleActivator, IWebResourceDecorator { > indexer.join(); > indexer = null; > } >+ if (purgeJob != null) { >+ purgeJob.cancel(); >+ purgeJob.join(); >+ purgeJob = null; >+ } > SearchActivator.context = null; > } > >-- >1.7.3.1.msysgit.0 >
You cannot view the attachment while viewing its details because your browser does not support IFRAMEs.
View the attachment on a separate page
.
View Attachment As Diff
View Attachment As Raw
Flags:
jarthana
:
review?
Actions:
View
|
Diff
Attachments on
bug 334699
: 192837