IC SunsetThe developerWorks Connections platform will be sunset on December 31, 2019. On January 1, 2020, this community and its apps will no longer be available. More details available on our FAQ.
Topic
  • No replies
TimK(USA)
TimK(USA)
21 Posts

Pinned topic Portal 8.5 Gets Hung up if 1 of our 4 DB instances are stopped

‏2019-08-19T18:07:59Z |

We have Portal 8.5 installed on 4 separate linux servers, there are 4 Oracle Load Balanced DBs housing the main portal schemas (release, likeminds, jcr, etc.)... If one of the four DBs are brought down, we would expect Portal to move to one of the other 3 available DBs, but it appears to get hung up in a state where is cannot resolve on it's own, not even after the DB server is back and fully online.  the logs constantly throw stack traces like this:

[8/19/19 5:29:46:249 CDT] 00006919 ContentModelI E com.ibm.wps.model.content.impl.ContentModelImpl findByID() EJPEG0111E: An unexpected exception has been caught.
                                 com.ibm.wps.model.exceptions.NlsDataException:EJPEG0111E: An unexpected exception has been caught.
    at com.ibm.wps.model.content.impl.ContentModelImpl.getParentID(ContentModelImpl.java:3032)
    at com.ibm.wps.model.content.impl.task.MountTeTask.createTEPath(MountTeTask.java:180)
    at com.ibm.wps.model.content.impl.task.MountTeTask.doItYourself(MountTeTask.java:106)
    at com.ibm.wps.model.content.impl.ConcurrentTaskSynchronizerImpl.synchronizeTask(ConcurrentTaskSynchronizerImpl.java:72)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.synchronizeTask(SmartTopologyValidator.java:933)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.mountTE(SmartTopologyValidator.java:852)
    at com.ibm.wps.model.content.impl.ContentModelImpl.findVisibleTE(ContentModelImpl.java:2297)
    at com.ibm.wps.model.content.impl.ContentModelImpl.findByID(ContentModelImpl.java:2359)
    at com.ibm.wps.model.content.impl.ContentModelImpl.findByID(ContentModelImpl.java:128)
    at com.ibm.wps.model.navigation.impl.NavigationModelImpl.findByID(NavigationModelImpl.java:296)
    at com.ibm.wps.model.navigation.impl.NavigationModelImpl.findByID(NavigationModelImpl.java:113)
    at com.ibm.wps.model.wrappers.NavigationModelWrapper.findByID(NavigationModelWrapper.java:136)
    at com.ibm.wps.model.navigation.impl.NavigationSelectionModel.makeValid(NavigationSelectionModel.java:289)
    at com.ibm.wps.model.navigation.impl.NavigationSelectionModel.getSelectedNode(NavigationSelectionModel.java:162)
    at com.ibm.wps.model.navigation.impl.NavigationSelectionModel.getSelectedNode(NavigationSelectionModel.java:56)
    at com.ibm.wps.model.wrappers.NavigationSelectionModelWrapper.getSelectedNode(NavigationSelectionModelWrapper.java:70)
    at com.ibm.wps.engine.phases.WPPhaseUtils.getSelectedContentNode(WPPhaseUtils.java:303)
    at com.ibm.wps.engine.phases.WPBaseRenderPhase.execute(WPBaseRenderPhase.java:225)
    at com.ibm.wps.state.phases.AbstractRenderPhase.next(AbstractRenderPhase.java:106)
    at com.ibm.wps.engine.phases.WPAbstractRenderPhase.next(WPAbstractRenderPhase.java:100)
    at com.ibm.wps.engine.Servlet.callPortal(Servlet.java:277)
    at com.ibm.wps.engine.Servlet.doGet(Servlet.java:579)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:575)
    at com.ibm.wps.engine.Servlet.doFilter(Servlet.java:359)
    at com.ibm.wps.resolver.servlet.ContentHandlerCleanup.doFilter(ContentHandlerCleanup.java:780)
    at com.ibm.wps.resolver.servlet.AbstractFilter.doFilter(AbstractFilter.java:103)
    at com.ibm.wps.engine.Servlet.service(Servlet.java:1279)
    at com.ibm.ws.cache.servlet.ServletWrapper.serviceProxied(ServletWrapper.java:307)
    at com.ibm.ws.cache.servlet.CacheHook.handleFragment(CacheHook.java:562)
    at com.ibm.ws.cache.servlet.CacheHook.handleServlet(CacheHook.java:255)
    at com.ibm.ws.cache.servlet.ServletWrapper.service(ServletWrapper.java:259)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1233)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:782)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:481)
    at com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:178)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:136)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:97)
    at com.ibm.wps.engine.ExtensionFilter$ExtendedFilterChain.doFilter(ExtensionFilter.java:110)
    at com.ibm.wps.engine.ExtensionFilter.doFilter(ExtensionFilter.java:197)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.SupportedLocalesFilter.doFilter(SupportedLocalesFilter.java:112)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.ExtendedLocaleFilter.doFilter(ExtendedLocaleFilter.java:132)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.resolver.friendly.servlet.FriendlySelectionFilter.doFilter(FriendlySelectionFilter.java:191)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.project.filter.ProjectIdFilter.doFilterWithoutProjectID(ProjectIdFilter.java:405)
    at com.ibm.wps.project.filter.ProjectIdFilter.doFilter(ProjectIdFilter.java:319)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.services.preview.PreviewFilterImpl.doFilter(PreviewFilterImpl.java:356)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.mappingurl.impl.URLAnalyzer.doFilter(URLAnalyzer.java:442)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.VirtualPortalFilter.doFilter(VirtualPortalFilter.java:89)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.resolver.servlet.ContentHandlerGzip.internalDoFilter(ContentHandlerGzip.java:758)
    at com.ibm.wps.resolver.servlet.ContentHandlerGzip.doFilter(ContentHandlerGzip.java:496)
    at com.ibm.wps.resolver.servlet.AbstractFilter.doFilter(AbstractFilter.java:103)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.state.filter.StateCleanup.doFilter(StateCleanup.java:103)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.devicesupport.WorklightFilter.doFilter(WorklightFilter.java:162)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:967)
    at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1107)
    at com.ibm.ws.webcontainer.servlet.CacheServletWrapper.handleRequest(CacheServletWrapper.java:87)
    at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:949)
    at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1817)
    at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:213)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:463)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:530)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:316)
    at com.ibm.ws.http.channel.inbound.impl.HttpICLReadCallback.complete(HttpICLReadCallback.java:88)
    at com.ibm.ws.ssl.channel.impl.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1833)
    at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
    at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
    at com.ibm.io.async.AsyncChannelFuture$1.run(AsyncChannelFuture.java:205)
    at com.ibm.ws.util.ThreadPool$Worker.run(ThreadPool.java:1892)
Caused by: com.ibm.wps.datastore.domains.DomainUnavailableException: EJPDB0101E: Database domain [Domain: rel] is currently unavailable.
    at com.ibm.wps.datastore.impl.DataStoreContext.init(DataStoreContext.java:140)
    at com.ibm.wps.datastore.impl.ResourcePersister.findObjectIDs(ResourcePersister.java:2647)
    at com.ibm.wps.datastore.impl.ResourcePersister.findObjectIDsAddProjectScope(ResourcePersister.java:2519)
    at com.ibm.wps.datastore.impl.ResourcePersister.findObjectIDsAddProjectScope(ResourcePersister.java:2498)
    at com.ibm.wps.datastore.impl.PageInstancePersister.getOIDsFilteredByInactive(PageInstancePersister.java:546)
    at com.ibm.wps.datastore.impl.PageInstanceHomeImpl.getOIDsFilteredByInactive(PageInstanceHomeImpl.java:875)
    at com.ibm.wps.datastore.impl.drafts.merger.PageInstanceMergerHomeImpl.getOIDsFilteredByInactive(PageInstanceMergerHomeImpl.java:626)
    at com.ibm.wps.datastore.impl.federation.PageInstanceHomeImpl.getOIDsFilteredByInactive(PageInstanceHomeImpl.java:1148)
    at com.ibm.wps.model.content.impl.ContentModelImpl.filterVisible(ContentModelImpl.java:3862)
    at com.ibm.wps.model.content.impl.task.FilterTeTask.doItYourself(FilterTeTask.java:125)
    at com.ibm.wps.model.content.impl.ConcurrentTaskSynchronizerImpl.synchronizeTask(ConcurrentTaskSynchronizerImpl.java:72)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.synchronizeTask(SmartTopologyValidator.java:953)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.filterTE(SmartTopologyValidator.java:885)
    at com.ibm.wps.model.content.impl.task.SortTeTask.doItYourself(SortTeTask.java:99)
    at com.ibm.wps.model.content.impl.ConcurrentTaskSynchronizerImpl.synchronizeTask(ConcurrentTaskSynchronizerImpl.java:72)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.synchronizeTask(SmartTopologyValidator.java:962)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.sortTE(SmartTopologyValidator.java:896)
    at com.ibm.wps.model.content.impl.task.FinalizeTeTask.doItYourself(FinalizeTeTask.java:104)
    at com.ibm.wps.model.content.impl.ConcurrentTaskSynchronizerImpl.synchronizeTask(ConcurrentTaskSynchronizerImpl.java:72)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.synchronizeTask(SmartTopologyValidator.java:972)
    at com.ibm.wps.model.content.impl.SmartTopologyValidator.finalizeTE(SmartTopologyValidator.java:874)
    at com.ibm.wps.model.content.impl.ContentModelImpl.getChildren(ContentModelImpl.java:2125)
    at com.ibm.wps.model.content.impl.ContentModelImpl.getChildren(ContentModelImpl.java:128)
    at com.ibm.wps.model.navigation.impl.NavigationModelImpl.getChildren(NavigationModelImpl.java:208)
    at com.ibm.wps.model.navigation.impl.NavigationModelImpl.getChildren(NavigationModelImpl.java:113)
    at com.ibm.wps.model.wrappers.NavigationModelWrapper.getChildren(NavigationModelWrapper.java:105)
    at com.ibm.wps.state.preprocessors.selection.StandardPortalSelectionResolver.findSelectableContent(StandardPortalSelectionResolver.java:229)
    at com.ibm.wps.state.preprocessors.selection.StandardPortalSelectionResolver.getDefaultSelection(StandardPortalSelectionResolver.java:335)
    at com.ibm.wps.state.preprocessors.selection.StandardPortalSelectionResolver.resolveSelection(StandardPortalSelectionResolver.java:526)
    at com.ibm.wps.state.preprocessors.selection.StandardPortalSelectionResolver.resolveSelection(StandardPortalSelectionResolver.java:472)
    at com.ibm.wps.state.preprocessors.selection.StandardPortalSelectionImpl.process(StandardPortalSelectionImpl.java:302)
    at com.ibm.wps.state.preprocessors.PreProcessorChain.process(PreProcessorChain.java:112)
    at com.ibm.wps.state.phases.PhaseManagerImpl.processState(PhaseManagerImpl.java:1050)
    at com.ibm.wps.state.phases.PhaseManagerImpl.decodeState(PhaseManagerImpl.java:372)
    at com.ibm.wps.state.phases.PhaseManagerImpl.next(PhaseManagerImpl.java:766)
    at com.ibm.wps.state.phases.AbstractPhaseManager.next(AbstractPhaseManager.java:125)
    at com.ibm.wps.engine.phases.WPInitPhase.next(WPInitPhase.java:535)
    at com.ibm.wps.engine.Servlet.doGet(Servlet.java:525)
    at javax.servlet.http.HttpServlet.service(HttpServlet.java:575)
    at com.ibm.wps.engine.Servlet.doFilter(Servlet.java:359)
    at com.ibm.wps.resolver.servlet.ContentHandlerCleanup.doFilter(ContentHandlerCleanup.java:780)
    at com.ibm.wps.resolver.servlet.AbstractFilter.doFilter(AbstractFilter.java:103)
    at com.ibm.wps.engine.Servlet.service(Servlet.java:1279)
    at com.ibm.ws.cache.servlet.ServletWrapper.serviceProxied(ServletWrapper.java:307)
    at com.ibm.ws.cache.servlet.CacheHook.handleFragment(CacheHook.java:562)
    at com.ibm.ws.cache.servlet.CacheHook.handleServlet(CacheHook.java:255)
    at com.ibm.ws.cache.servlet.ServletWrapper.service(ServletWrapper.java:259)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.service(ServletWrapper.java:1233)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:782)
    at com.ibm.ws.webcontainer.servlet.ServletWrapper.handleRequest(ServletWrapper.java:481)
    at com.ibm.ws.webcontainer.servlet.ServletWrapperImpl.handleRequest(ServletWrapperImpl.java:178)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.invokeTarget(WebAppFilterChain.java:136)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:97)
    at com.ibm.wps.engine.ExtensionFilter$ExtendedFilterChain.doFilter(ExtensionFilter.java:110)
    at com.ibm.wps.engine.ExtensionFilter.doFilter(ExtensionFilter.java:197)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.SupportedLocalesFilter.doFilter(SupportedLocalesFilter.java:112)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.ExtendedLocaleFilter.doFilter(ExtendedLocaleFilter.java:132)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.resolver.friendly.servlet.FriendlySelectionFilter.doFilter(FriendlySelectionFilter.java:191)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.project.filter.ProjectIdFilter.doFilterWithoutProjectID(ProjectIdFilter.java:405)
    at com.ibm.wps.project.filter.ProjectIdFilter.doFilter(ProjectIdFilter.java:319)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.services.preview.PreviewFilterImpl.doFilter(PreviewFilterImpl.java:356)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.mappingurl.impl.URLAnalyzer.doFilter(URLAnalyzer.java:442)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.engine.VirtualPortalFilter.doFilter(VirtualPortalFilter.java:89)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.resolver.servlet.ContentHandlerGzip.internalDoFilter(ContentHandlerGzip.java:758)
    at com.ibm.wps.resolver.servlet.ContentHandlerGzip.doFilter(ContentHandlerGzip.java:496)
    at com.ibm.wps.resolver.servlet.AbstractFilter.doFilter(AbstractFilter.java:103)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.state.filter.StateCleanup.doFilter(StateCleanup.java:103)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.wps.devicesupport.WorklightFilter.doFilter(WorklightFilter.java:162)
    at com.ibm.ws.webcontainer.filter.FilterInstanceWrapper.doFilter(FilterInstanceWrapper.java:195)
    at com.ibm.ws.webcontainer.filter.WebAppFilterChain.doFilter(WebAppFilterChain.java:91)
    at com.ibm.ws.webcontainer.filter.WebAppFilterManager.doFilter(WebAppFilterManager.java:967)
    at com.ibm.ws.webcontainer.filter.WebAppFilterManager.invokeFilters(WebAppFilterManager.java:1107)
    at com.ibm.ws.webcontainer.servlet.CacheServletWrapper.handleRequest(CacheServletWrapper.java:87)
    at com.ibm.ws.webcontainer.WebContainer.handleRequest(WebContainer.java:949)
    at com.ibm.ws.webcontainer.WSWebContainer.handleRequest(WSWebContainer.java:1817)
    at com.ibm.ws.webcontainer.channel.WCChannelLink.ready(WCChannelLink.java:213)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleDiscrimination(HttpInboundLink.java:463)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.handleNewRequest(HttpInboundLink.java:530)
    at com.ibm.ws.http.channel.inbound.impl.HttpInboundLink.processRequest(HttpInboundLink.java:316)
    at com.ibm.ws.http.channel.inbound.impl.HttpICLReadCallback.complete(HttpICLReadCallback.java:88)
    at com.ibm.ws.ssl.channel.impl.SSLReadServiceContext$SSLReadCompletedCallback.complete(SSLReadServiceContext.java:1833)
    at com.ibm.ws.tcp.channel.impl.AioReadCompletionListener.futureCompleted(AioReadCompletionListener.java:175)
    at com.ibm.io.async.AbstractAsyncFuture.invokeCallback(AbstractAsyncFuture.java:217)
    at com.ibm.io.async.AsyncChannelFuture.fireCompletionActions(AsyncChannelFuture.java:161)
    at com.ibm.io.async.AsyncFuture.completed(AsyncFuture.java:138)
    at com.ibm.io.async.ResultHandler.complete(ResultHandler.java:204)
    at com.ibm.io.async.ResultHandler.runEventProcessingLoop(ResultHandler.java:775)
    at com.ibm.io.async.ResultHandler$2.run(ResultHandler.java:905)
    ... 1 more

The only way to get that node back up is to manually restart portal.  Is this normal behavior?  Or are we missing something in the configuration where this failover is not occuring?  JDBC data source settings maybe?

Sometimes other nodes present this error, but still work some of the time, or they can recover on their own after some time...