diff --git a/src/java/net/brutex/xservices/util/cache/CacheExecutorService.java b/src/java/net/brutex/xservices/util/cache/CacheExecutorService.java new file mode 100644 index 0000000..0451897 --- /dev/null +++ b/src/java/net/brutex/xservices/util/cache/CacheExecutorService.java @@ -0,0 +1,60 @@ +/* + * Copyright 2013 Brian Rosenberger (Brutex Network) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.brutex.xservices.util.cache; + +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +import javax.servlet.ServletContext; +import javax.servlet.ServletContextEvent; +import javax.servlet.ServletContextListener; + +/** + * @author Brian Rosenberger, bru(at)brutex.de + * + */ + + + +public class CacheExecutorService + implements ServletContextListener +{ + static final String EXECUTOR_NAME = "CACHE_EXECUTOR"; + private ExecutorService executor; + + public void contextInitialized(ServletContextEvent arg0) + { + ServletContext context = arg0.getServletContext(); + int nr_executors = 5; + ThreadFactory daemonFactory = new DaemonThreadFactory(); + try { + nr_executors = Integer.parseInt(context.getInitParameter("cache:thread-count")); + } catch (NumberFormatException localNumberFormatException) { + } + if (nr_executors <= 1) + this.executor = Executors.newSingleThreadExecutor(daemonFactory); + else { + this.executor = Executors.newFixedThreadPool(nr_executors, daemonFactory); + } + context.setAttribute("CACHE_EXECUTOR", this.executor); + } + + public void contextDestroyed(ServletContextEvent arg0) { + ServletContext context = arg0.getServletContext(); + this.executor.shutdownNow(); + } +} \ No newline at end of file diff --git a/src/java/net/brutex/xservices/util/cache/CacheServlet.java b/src/java/net/brutex/xservices/util/cache/CacheServlet.java new file mode 100644 index 0000000..5fe7f38 --- /dev/null +++ b/src/java/net/brutex/xservices/util/cache/CacheServlet.java @@ -0,0 +1,70 @@ +/* + * Copyright 2013 Brian Rosenberger (Brutex Network) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.brutex.xservices.util.cache; + +import java.io.File; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.concurrent.ExecutorService; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import net.brutex.xservices.types.scm.ObjectFactory; +import org.apache.log4j.Logger; + +/** + * @author Brian Rosenberger, bru(at)brutex.de + * + */ + + +public class CacheServlet extends HttpServlet +{ + private final Logger logger = Logger.getLogger(CacheServlet.class); + List configfiles = new ArrayList(); + int cacheinterval; + private final ObjectFactory FACTORY = new ObjectFactory(); + + public void init() + throws ServletException + { + super.init(); + ExecutorService executor = (ExecutorService)getServletContext() + .getAttribute("CACHE_EXECUTOR"); + + Enumeration attributes = getServletContext() + .getInitParameterNames(); + while (attributes.hasMoreElements()) { + String name = (String)attributes.nextElement(); + if (name.startsWith("cvs-config-")) { + String configfile = getServletContext() + .getInitParameter(name); + this.logger.info("CVS configuration file: " + configfile); + this.configfiles.add(new File(configfile)); + } + } + this.cacheinterval = 15; + try { + this.cacheinterval = Integer.parseInt(getServletContext() + .getInitParameter("cvs-cache-interval")); + } catch (NumberFormatException e) { + this.logger.debug("Could not read parameter 'cvs-cache-interval' from web.xml. Using default value '" + this.cacheinterval + "' minutes"); + } + this.logger.info("CacheServlet set to " + this.cacheinterval + " minutes interval."); + } +} diff --git a/src/java/net/brutex/xservices/util/cache/DaemonThreadFactory.java b/src/java/net/brutex/xservices/util/cache/DaemonThreadFactory.java new file mode 100644 index 0000000..d664389 --- /dev/null +++ b/src/java/net/brutex/xservices/util/cache/DaemonThreadFactory.java @@ -0,0 +1,49 @@ +/* + * Copyright 2013 Brian Rosenberger (Brutex Network) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.brutex.xservices.util.cache; + +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadFactory; +/** + * @author Brian Rosenberger, bru(at)brutex.de + * + */ + + +public class DaemonThreadFactory + implements ThreadFactory +{ + private final ThreadFactory factory; + + public DaemonThreadFactory() + { + this(Executors.defaultThreadFactory()); + } + + public DaemonThreadFactory(ThreadFactory factory) + { + if (factory == null) + throw new NullPointerException("factory cannot be null"); + this.factory = factory; + } + + public Thread newThread(Runnable r) { + Thread t = this.factory.newThread(r); + t.setDaemon(true); + return t; + } +} \ No newline at end of file diff --git a/src/java/net/brutex/xservices/util/cache/FindingsCacheServlet.java b/src/java/net/brutex/xservices/util/cache/FindingsCacheServlet.java new file mode 100644 index 0000000..525ce96 --- /dev/null +++ b/src/java/net/brutex/xservices/util/cache/FindingsCacheServlet.java @@ -0,0 +1,346 @@ +/* + * Copyright 2013 Brian Rosenberger (Brutex Network) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.brutex.xservices.util.cache; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.ExecutorService; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import javax.servlet.ServletContext; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.ws.rs.core.Response; +import net.brutex.xservices.types.scm.ItemListType; +import net.brutex.xservices.types.scm.ItemType; +import net.brutex.xservices.types.scmfindings.FindingDetailsType; +import net.brutex.xservices.types.scmfindings.FindingType; +import net.brutex.xservices.types.scmfindings.FindingsListType; +import net.brutex.xservices.types.scmfindings.GroupMatchListType; +import net.brutex.xservices.types.scmfindings.ObjectFactory; +import net.brutex.xservices.ws.rs.CVSInfoImpl; +import org.apache.commons.configuration.ConfigurationException; +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.jcs.JCS; +import org.apache.jcs.access.exception.CacheException; +import org.apache.log4j.Logger; + +/** + * @author Brian Rosenberger, bru(at)brutex.de + * + */ + +public class FindingsCacheServlet extends HttpServlet { + + private static final long serialVersionUID = 4041338473949999960L; + private final static Logger logger = Logger + .getLogger(FindingsCacheServlet.class); + private final List configfiles = new ArrayList(); + private final ObjectFactory FACTORY = new ObjectFactory(); + + @Override + public void init() throws ServletException { + super.init(); + ExecutorService executor = (ExecutorService) getServletContext() + .getAttribute("CACHE_EXECUTOR"); + + if(! this.initConfigList()) return; + if(! this.initConfigFindings()) return; + + int i = 1; + for(File f: configfiles) { + //Initialise configuration bean using default values + FindingsConfigBean cbean = new FindingsConfigBean(i, Logger.getLogger("worker-"+i+ "." + this.getClass().getName())); + i++; + + + + //Read cvs-cache-interval parameter + try { + int cacheinterval = Integer.parseInt(getServletContext() + .getInitParameter("cvs-cache-interval")); + cbean.setCacheinterval(cacheinterval); + logger.info("FindingsCacheServlet set to "+ cacheinterval + " minutes interval."); + } catch (NumberFormatException e) { + logger.warn("Could not read parameter 'cvs-cache-interval' from web.xml. Using default value '" + + cbean.getCacheinterval()+ "' minutes"); + } + + PropertiesConfiguration config = null; + try { + config = new PropertiesConfiguration(f); + } catch (ConfigurationException e) { + logger.error("Could not read parameter file at '"+f.getAbsolutePath()+"'"); + return; + } + + + File cvsconfig = new File(config.getString("CVSROOTCONFIGFILE")); + cbean.setCvsconfig(cvsconfig); + FindingsCacheServlet.logger.debug("Fetching list of files using '" + + cvsconfig.getAbsolutePath() + "' config file"); + + + List filepatterns = config.getList("FILESEARCH"); + cbean.setFilepatterns(filepatterns); + FindingsCacheServlet.logger.debug("Checking '" + + filepatterns.size() + + "' patterns for file name and path matching."); + + + List contentpatterns = config.getList("CONTENTSEARCH"); + cbean.setContentpatterns(contentpatterns); + FindingsCacheServlet.logger.debug("Checking '" + + contentpatterns.size() + + "' patterns for content matching"); + + + + + executor.submit(new ThisRunnable(cbean)); + } + logger.info("FindingsCacheServlet has been initialized."); + + } + + /* + * Initialise CVS findings configuration + */ + private boolean initConfigFindings() { + String filename = getServletContext().getInitParameter( + "cvs-findings-configuration"); + if (filename == null) { + logger.warn("'cvs-findings-configuration' init parameter is not specified."); + return false; + } + final File findingsconfig = new File(filename); + logger.info("CVS findings configuration file found at '" + + findingsconfig.getAbsolutePath() + "'"); + if ((!findingsconfig.canRead()) || (findingsconfig.isDirectory())) { + logger.info("CVS findings configuration file '" + + findingsconfig.getAbsolutePath() + "' does not exist."); + return false; + } + return true; + } + + /* + * Add all specified CVS configuration files to the list. Parameter pattern + * is "cvs-config-XX". + */ + private boolean initConfigList() { + Enumeration attributes = getServletContext() + .getInitParameterNames(); + while (attributes.hasMoreElements()) { + String name = (String) attributes.nextElement(); + if (name.startsWith("cvs-config-")) { + String configfile = getServletContext().getInitParameter(name); + logger.info("Adding CVS configuration file: " + configfile); + this.configfiles.add(new File(configfile)); + } + } + /* + * Verify, that all configuration files do exists and are readable. + */ + List removelist = new ArrayList(); + for (File f : configfiles) { + if (!f.exists()) { + logger.warn("CVS configuration file '" + + f.getAbsolutePath() + + "' is specified, but does not exist. Removing from list."); + removelist.add(f); + } else if (!f.canRead()) { + logger.warn("CVS configuration file '" + + f.getAbsolutePath() + + "' does exist, but is not readable. Removing from list."); + removelist.add(f); + } + } + configfiles.removeAll(removelist); + return true; + } + + + + + class ThisRunnable implements Runnable { + boolean isInterrupted = false; + FindingsConfigBean configuration; + + public ThisRunnable(FindingsConfigBean configuration) { + this.configuration = configuration; + } + + public void run() { + CVSInfoImpl instance = new CVSInfoImpl(); + + ItemListType fileslist = (ItemListType) instance + .getRepositoryFiles(null, configuration.getCvsconfig(), "", true, true, true) + .getEntity(); + ObjectFactory FACTORY = new ObjectFactory(); + FindingsListType findingsList = FACTORY.createFindingsListType(); + + FindingsCacheServlet.logger.info("Processing '" + + fileslist.getItems().size() + "' files and directories."); + + while (!this.isInterrupted) { + Pattern p; + for (ItemType i : fileslist.getItems()) { + if (this.isInterrupted) + break; + Iterator iterF = configuration.getFilepatterns().iterator(); + while(iterF.hasNext()) { + Object o = iterF.next(); + if (this.isInterrupted) + break; + FindingsCacheServlet.logger.debug("Scanning filename '" + + i.getFullname() + "' for pattern '" + + (String) o + "'"); + p = Pattern.compile((String) o); + Matcher m = p.matcher(i.getFullname()); + if (m.find()) { + FindingType finding = FACTORY.createFindingType(); + finding.setFilesearch(p.toString()); + ItemType it = (ItemType) instance.getFileContent( + null, configuration.getCvsconfig(), i.getFullname(), true) + .getEntity(); + finding.setContent(it.getContent()); + finding.setData(it.getData()); + finding = copyDetails(finding, i); + findingsList.getFindings().add(finding); + FindingsCacheServlet.logger + .debug("Match found for '" + + i.getFullname() + "'"); + break; + } + FindingsCacheServlet.logger + .debug("No match found for '" + i.getFullname() + + "'"); + } + } + FindingsCacheServlet.logger + .debug("Processing file content for '" + + findingsList.getFindings().size() + + "' entries in the list."); + + for (FindingType t : findingsList.getFindings()) { + if (this.isInterrupted) + break; + boolean isFound = false; + Matcher m; + Iterator iter = configuration.getContentpatterns().iterator(); + while (iter.hasNext()) { + + Object o = iter.next(); + if (this.isInterrupted) + break; + FindingsCacheServlet.logger + .debug("Scanning file content for file '" + + t.getFullname() + "' for pattern '" + + (String) o + "'"); + + Pattern p1 = Pattern.compile((String) o); + m = p1.matcher(t.getContent()); + t.setContentsearch(p1.toString()); + + isFound = true; + int s = m.start(); + int e = m.end(); + String c = m.group(); + + FindingDetailsType fd = FACTORY + .createFindingDetailsType(); + GroupMatchListType gm = FACTORY + .createGroupMatchListType(); + gm.setMatchAtIndex(s); + gm.setMatchToIndex(e); + gm.setMatchString(c); + gm.setMatchGroup(0); + fd.setFullmatch(gm); + for (int i = 1; i <= m.groupCount(); i++) { + GroupMatchListType gmg = FACTORY + .createGroupMatchListType(); + s = m.start(i); + e = m.end(i); + c = m.group(i); + gmg.setMatchAtIndex(s); + gmg.setMatchToIndex(e); + gmg.setMatchString(c); + gmg.setMatchGroup(i); + fd.getMatchLists().add(gmg); + } + t.getFindingLists().add(fd); + FindingsCacheServlet.logger + .debug("Found matching content at index '" + s + + "' in file '" + t.getFullname() + + "' with pattern '" + p1.toString() + + "'"); + } + + if (!isFound) { + findingsList.getFindings().remove(t); + FindingsCacheServlet.logger + .debug("Found matching filename for '" + + t.getFullname() + + "' but content didn't match. Removing."); + } + + try { + instance.getCacheInstance().put( + "FINDINGS-" + t.getROOT(), findingsList); + FindingsCacheServlet.logger + .info("FINDINGS for CVSROOT '" + t.getROOT() + + "' have been updated in cache."); + } catch (CacheException e) { + FindingsCacheServlet.logger.error(e.getMessage(), e); + } + } + try { + int cacheinterval = configuration.getCacheinterval(); + FindingsCacheServlet.logger.debug("Now sleeping for '" + + cacheinterval + "' minutes"); + Thread.currentThread(); + Thread.sleep(cacheinterval * 60000); + FindingsCacheServlet.logger.debug("Waking up after '" + + cacheinterval + "' minutes of sleep"); + } catch (InterruptedException e) { + this.isInterrupted = true; + FindingsCacheServlet.logger + .warn("FindingsCacheServlet cache was interrupted. Shutting down."); + } + } + + } + + private FindingType copyDetails(FindingType finding, ItemType item) { + finding.setDescription(item.getDescription()); + finding.setFullname(item.getFullname()); + finding.setName(item.getName()); + finding.setPath(item.getPath()); + finding.setRemotefullname(item.getRemotefullname()); + finding.setRemotename(item.getRemotename()); + finding.setRemotepath(item.getRemotepath()); + finding.setROOT(item.getROOT()); + return finding; + } + + } +} \ No newline at end of file diff --git a/src/java/net/brutex/xservices/util/cache/FindingsConfigBean.java b/src/java/net/brutex/xservices/util/cache/FindingsConfigBean.java new file mode 100644 index 0000000..a91d841 --- /dev/null +++ b/src/java/net/brutex/xservices/util/cache/FindingsConfigBean.java @@ -0,0 +1,107 @@ +/* + * Copyright 2013 Brian Rosenberger (Brutex Network) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package net.brutex.xservices.util.cache; + +import java.io.File; +import java.util.List; + +import org.apache.commons.configuration.PropertiesConfiguration; +import org.apache.log4j.Logger; + +/** + * @author Brian Rosenberger, bru(at)brutex.de + * + */ +public class FindingsConfigBean { + + private final Logger logger; + + private final int instanceid; + + private File cvsconfig; + private int cacheinterval = 15; + private List filepatterns; + private List contentpatterns; + + public FindingsConfigBean(int instanceid, Logger logger) { + this.instanceid = instanceid; + this.logger = logger; + logger.debug("Initialise FindingsConfigBean instance '"+instanceid+"'"); + + } + + /** + * @return the cache interval + */ + public int getCacheinterval() { + return cacheinterval; + } + + /** + * @param cacheinterval the cache interval to set + */ + public void setCacheinterval(int cacheinterval) { + this.cacheinterval = cacheinterval; + } + + /** + * @return the cvsconfig + */ + public File getCvsconfig() { + return cvsconfig; + } + + /** + * @param cvsconfig the cvsconfig to set + */ + public void setCvsconfig(File cvsconfig) { + this.cvsconfig = cvsconfig; + } + + /** + * @return the filepatterns + */ + public List getFilepatterns() { + return filepatterns; + } + + /** + * @param filepatterns the filepatterns to set + */ + public void setFilepatterns(List filepatterns) { + this.filepatterns = filepatterns; + } + + /** + * @return the contentpatterns + */ + public List getContentpatterns() { + return contentpatterns; + } + + /** + * @param contentpatterns the contentpatterns to set + */ + public void setContentpatterns(List contentpatterns) { + this.contentpatterns = contentpatterns; + } + + /** + * @param instanceid + */ + +}