Community
Participate
Working Groups
Build Identifier: Helios release Hi folks! I've discovered a situation where a deadlock occurs when performing httpService.unregister(...) while there is another thread processing a request where the requestdispatcher is used. The situation causing the problem is: + a request is being processed by a servlet. the servlet uses requestdispatcher.include("a jsp file"). before processing of the jsp starts, the thread is paused (randomly; by the vm scheduler) + another thread starts working: a component calls httpService.unregister(...) for the servlet processing the request. thread runs up to Registration.destroy() where it waits until the current request is completed. The thread holds the monitor for its ProxyServlet instance because the unregister method is synchronized. the monitor of the Registration object is released by calling wait() while in the Registration object itself. + now the other thread continues processing the request. the requestdispatcher.include() call internally begins a new request which is being blocked because it cannot acquire the monitor of the ProxyServlet (line 96) because the thread performing the unregister is waiting for this thread to finish processing. Here is the stacktrace of the 2 involved threads: 1st thread: wants to enter crititcal section in ProxyServlet - waits for the 2nd Thread. Thread: Id=102 Name=30131099@qtp-10923886-4 State: BLOCKED org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:97) org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:60) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at.anext.equinox.http.jetty.internal.HttpServerManager$InternalHttpServiceServlet.service(HttpServerManager.java:373) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:390) org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765) org.mortbay.jetty.servlet.Dispatcher.include(Dispatcher.java:192) org.eclipse.equinox.http.servlet.internal.RequestDispatcherAdaptor.include(RequestDispatcherAdaptor.java:37) at.anext.os.p2.servlet.NewServlet.stage1(NewServlet.java:217) at.anext.os.p2.servlet.NewServlet.doGet(NewServlet.java:119) javax.servlet.http.HttpServlet.service(HttpServlet.java:707) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) org.eclipse.equinox.http.servlet.internal.ServletRegistration.service(ServletRegistration.java:61) org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:126) org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:60) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at.anext.equinox.http.jetty.internal.HttpServerManager$InternalHttpServiceServlet.service(HttpServerManager.java:373) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:390) org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765) org.mortbay.jetty.handler.HandlerList.handle(HandlerList.java:49) org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) org.mortbay.jetty.Server.handle(Server.java:326) org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:924) org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) 2nd thread: holds the monitor for the proxyservlet object and WAITS for the registration object-monitor to get released: Thread: Id=101 Name=Refresh Packages State: WAITING java.lang.Object.wait(Object.java:-2) java.lang.Object.wait(Object.java:485) org.eclipse.equinox.http.servlet.internal.Registration.destroy(Registration.java:34) org.eclipse.equinox.http.servlet.internal.ServletRegistration.destroy(ServletRegistration.java:37) org.eclipse.equinox.http.servlet.internal.ProxyServlet.unregister(ProxyServlet.java:151) org.eclipse.equinox.http.servlet.internal.HttpServiceImpl.unregister(HttpServiceImpl.java:88) at.anext.os.p2.servlet.ServiceComponent.unbind(ServiceComponent.java:220) at.anext.os.p2.servlet.ServiceComponent.rebind(ServiceComponent.java:62) at.anext.os.ext.updatesystemrebinder.UpdateSystemRebinder.removeUpdateSystemControl(UpdateSystemRebinder.java:16) sun.reflect.NativeMethodAccessorImpl.invoke0(NativeMethodAccessorImpl.java:-2) sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39) sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25) java.lang.reflect.Method.invoke(Method.java:597) org.eclipse.equinox.internal.ds.model.ComponentReference.unbind(ComponentReference.java:457) org.eclipse.equinox.internal.ds.model.ServiceComponentProp.unbindReference(ServiceComponentProp.java:597) org.eclipse.equinox.internal.ds.model.ServiceComponentProp.unbind(ServiceComponentProp.java:246) org.eclipse.equinox.internal.ds.model.ServiceComponentProp.dispose(ServiceComponentProp.java:372) org.eclipse.equinox.internal.ds.model.ServiceComponentProp.dispose(ServiceComponentProp.java:102) org.eclipse.equinox.internal.ds.InstanceProcess.disposeInstances(InstanceProcess.java:343) org.eclipse.equinox.internal.ds.InstanceProcess.disposeInstances(InstanceProcess.java:305) org.eclipse.equinox.internal.ds.Resolver.disposeComponentConfigs(Resolver.java:675) org.eclipse.equinox.internal.ds.Resolver.disableComponents(Resolver.java:651) org.eclipse.equinox.internal.ds.SCRManager.stoppingBundle(SCRManager.java:553) org.eclipse.equinox.internal.ds.SCRManager.bundleChanged(SCRManager.java:232) org.eclipse.osgi.framework.internal.core.BundleContextImpl.dispatchEvent(BundleContextImpl.java:919) org.eclipse.osgi.framework.eventmgr.EventManager.dispatchEvent(EventManager.java:227) org.eclipse.osgi.framework.eventmgr.ListenerQueue.dispatchEventSynchronous(ListenerQueue.java:149) org.eclipse.osgi.framework.internal.core.Framework.publishBundleEventPrivileged(Framework.java:1349) org.eclipse.osgi.framework.internal.core.Framework.publishBundleEvent(Framework.java:1300) org.eclipse.osgi.framework.internal.core.BundleHost.stopWorker(BundleHost.java:497) org.eclipse.osgi.framework.internal.core.AbstractBundle.suspend(AbstractBundle.java:550) org.eclipse.osgi.framework.internal.core.Framework.suspendBundle(Framework.java:1097) org.eclipse.osgi.framework.internal.core.PackageAdminImpl.suspendBundle(PackageAdminImpl.java:311) org.eclipse.osgi.framework.internal.core.PackageAdminImpl.processDelta(PackageAdminImpl.java:457) org.eclipse.osgi.framework.internal.core.PackageAdminImpl.doResolveBundles(PackageAdminImpl.java:239) org.eclipse.osgi.framework.internal.core.PackageAdminImpl$1.run(PackageAdminImpl.java:174) java.lang.Thread.run(Thread.java:637) i hope my investigations are correct. the problem now is that ALL requests targeted for this servlet are being hold and after a short time we're out of threads => application hangs => restart required. got any solutions? not yet :) greetings from austria, michael stacktrace of a new request performed while the deadlock take place: Thread: Id=103 Name=5365202@qtp-10923886-5 State: BLOCKED org.eclipse.equinox.http.servlet.internal.ProxyServlet.processAlias(ProxyServlet.java:97) org.eclipse.equinox.http.servlet.internal.ProxyServlet.service(ProxyServlet.java:60) javax.servlet.http.HttpServlet.service(HttpServlet.java:820) at.anext.equinox.http.jetty.internal.HttpServerManager$InternalHttpServiceServlet.service(HttpServerManager.java:373) org.mortbay.jetty.servlet.ServletHolder.handle(ServletHolder.java:511) org.mortbay.jetty.servlet.ServletHandler.handle(ServletHandler.java:390) org.mortbay.jetty.servlet.SessionHandler.handle(SessionHandler.java:182) org.mortbay.jetty.handler.ContextHandler.handle(ContextHandler.java:765) org.mortbay.jetty.handler.HandlerList.handle(HandlerList.java:49) org.mortbay.jetty.handler.HandlerWrapper.handle(HandlerWrapper.java:152) org.mortbay.jetty.Server.handle(Server.java:326) org.mortbay.jetty.HttpConnection.handleRequest(HttpConnection.java:542) org.mortbay.jetty.HttpConnection$RequestHandler.headerComplete(HttpConnection.java:924) org.mortbay.jetty.HttpParser.parseNext(HttpParser.java:549) org.mortbay.jetty.HttpParser.parseAvailable(HttpParser.java:212) org.mortbay.jetty.HttpConnection.handle(HttpConnection.java:404) org.mortbay.io.nio.SelectChannelEndPoint.run(SelectChannelEndPoint.java:409) org.mortbay.thread.QueuedThreadPool$PoolThread.run(QueuedThreadPool.java:582) Reproducible: Sometimes Steps to Reproduce: 1. perform multiple requests on a servlet which uses a requestdispatcher 2. unregister the servlet while requests being processed
Thanks! We need a different type of lock to take into consideration existing requests calling back out to the request dispatcher.
It is too late in 3.7 to change locking strategies. Will defer to 3.7.1. Hopefully we can get a fix early in 3.8 for testing before backporting a fix to 3.7.1.
It is too late for 3.7.1. Moving to 3.7.2.
Dropping milestone. Will need a patch to make progress here.
I'm very unsure about the synchronization mechanism of the whole system but i've created a patch which might work. Maybe somebody can do a small review and post some comments/hints. ### Eclipse Workspace Patch 1.0 #P org.eclipse.equinox.http.servlet Index: src/org/eclipse/equinox/http/servlet/internal/ProxyServlet.java =================================================================== RCS file: /cvsroot/rt/org.eclipse.equinox/compendium/bundles/org.eclipse.equinox.http.servlet/src/org/eclipse/equinox/http/servlet/internal/ProxyServlet.java,v retrieving revision 1.11 diff -u -r1.11 ProxyServlet.java --- src/org/eclipse/equinox/http/servlet/internal/ProxyServlet.java 29 Aug 2011 13:22:42 -0000 1.11 +++ src/org/eclipse/equinox/http/servlet/internal/ProxyServlet.java 31 May 2012 15:43:18 -0000 @@ -144,15 +144,22 @@ } //Effective unregistration of servlet and resources as defined in HttpService#unregister() - synchronized void unregister(String alias, boolean destroy) { - ServletRegistration removedRegistration = (ServletRegistration) servletRegistrations.remove(alias); - if (removedRegistration != null) { - registeredServlets.remove(removedRegistration.getServlet()); - try { - if (destroy) - removedRegistration.destroy(); - } finally { - proxyContext.destroyContextAttributes(removedRegistration.getHttpContext()); + void unregister(String alias, boolean destroy) { + ServletRegistration removedRegistration = null; + synchronized (this) { + removedRegistration = (ServletRegistration) servletRegistrations.remove(alias); + if (removedRegistration != null) { + registeredServlets.remove(removedRegistration.getServlet()); + } + } + try { + if (removedRegistration != null && destroy) + removedRegistration.destroy(); + } finally { + if (removedRegistration != null) { + synchronized (this) { + proxyContext.destroyContextAttributes(removedRegistration.getHttpContext()); + } } } } @@ -213,14 +220,20 @@ throw new IllegalArgumentException("Invalid alias '" + alias + "'"); //$NON-NLS-1$//$NON-NLS-2$ } - public synchronized void unregisterFilter(Filter filter, boolean destroy) { - FilterRegistration removedRegistration = (FilterRegistration) filterRegistrations.remove(filter); - if (removedRegistration != null) { - try { - if (destroy) - removedRegistration.destroy(); - } finally { - proxyContext.destroyContextAttributes(removedRegistration.getHttpContext()); + public void unregisterFilter(Filter filter, boolean destroy) { + FilterRegistration removedRegistration = null; + synchronized (this) { + removedRegistration = (FilterRegistration) filterRegistrations.remove(filter); + } + + try { + if (removedRegistration != null && destroy) + removedRegistration.destroy(); + } finally { + if (removedRegistration != null) { + synchronized (this) { + proxyContext.destroyContextAttributes(removedRegistration.getHttpContext()); + } } } }
Created attachment 216628 [details] proposal 1
Marking for Kepler to review patch proposal. Thanks!
I released a slightly different solution in commit: http://git.eclipse.org/c/equinox/rt.equinox.bundles.git/commit/?id=5647d938c346c6fab37ac351a043336b5aa6e962 The main difference is I saw no need to hold the ProxyServlet lock while calling destroy() on the registration objects.
