if (attempts == null)
serverCore.bfHost.put(clientIP, Integer.valueOf(1));
else
serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
final ResponseHeader responseHeader = getDefaultHeaders(path);
responseHeader.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"admin log-in\"");
final servletProperties tp=new servletProperties();
tp.put("returnto", path);
HTTPDemon.sendRespondError(conProp, out, 5, 401, "Wrong Authentication", "", new File("proxymsg/authfail.inc"), tp, null, responseHeader);
return;
}
// Authentication successful. remove brute-force flag
serverCore.bfHost.remove(conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
// parse arguments
serverObjects args = new serverObjects();
int argc = 0;
if (argsString == null) {
// no args here, maybe a POST with multipart extension
final int length = requestHeader.getContentLength();
//System.out.println("HEADER: " + requestHeader.toString()); // DEBUG
/* don't parse body in case of a POST CGI call since it has to be
* handed over to the CGI script unaltered and parsed by the script
*/
if (method.equals(HeaderFramework.METHOD_POST) &&
!(switchboard.getConfigBool("cgi.allow", false) &&
matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)))
) {
// if its a POST, it can be either multipart or as args in the body
if ((requestHeader.containsKey(HeaderFramework.CONTENT_TYPE)) &&
(requestHeader.get(HeaderFramework.CONTENT_TYPE).toLowerCase().startsWith("multipart"))) {
// parse multipart
final Map<String, byte[]> files = HTTPDemon.parseMultipart(requestHeader, args, body);
// integrate these files into the args
if (files != null) {
final Iterator<Map.Entry<String, byte[]>> fit = files.entrySet().iterator();
Map.Entry<String, byte[]> entry;
while (fit.hasNext()) {
entry = fit.next();
args.put(entry.getKey() + "$file", entry.getValue());
}
}
argc = Integer.parseInt(requestHeader.get("ARGC"));
} else {
// parse args in body
argc = HTTPDemon.parseArgs(args, body, length);
}
} else {
// no args
argsString = null;
args = null;
argc = 0;
}
} else {
// simple args in URL (stuff after the "?")
argc = HTTPDemon.parseArgs(args, argsString);
}
// check for cross site scripting - attacks in request arguments
if (args != null && argc > 0) {
// check all values for occurrences of script values
final Iterator<String> e = args.values().iterator(); // enumeration of values
String val;
while (e.hasNext()) {
val = e.next();
if ((val != null) && (val.indexOf("<script",0) >= 0) && !path.equals("/Crawler_p.html")) {
// deny request
HTTPDemon.sendRespondError(conProp,out,4,403,null,"bad post values",null);
return;
}
}
}
if (args != null) nocache = true;
// we are finished with parsing
// the result of value hand-over is in args and argc
if (path.length() == 0) {
HTTPDemon.sendRespondError(conProp,out,4,400,null,"Bad Request",null);
out.flush();
return;
}
File targetClass=null;
// locate the file
if (path.length() > 0 && path.charAt(0) != '/' && path.charAt(0) != '\\') path = "/" + path; // attach leading slash
if (path.endsWith("index.html")) path = path.substring(0, path.length() - 10);
// a different language can be desired (by i.e. ConfigBasic.html) than the one stored in the locale.language
String localeSelection = switchboard.getConfig("locale.language","default");
if (args != null && (args.containsKey("language"))) {
// TODO 9.11.06 Bost: a class with information about available languages is needed.
// the indexOf(".") is just a workaround because there from ConfigLanguage.html commes "de.lng" and
// from ConfigBasic.html comes just "de" in the "language" parameter
localeSelection = args.get("language", localeSelection);
if (localeSelection.indexOf('.') != -1)
localeSelection = localeSelection.substring(0, localeSelection.indexOf('.'));
}
File targetFile = getLocalizedFile(path, localeSelection);
String targetExt = (String) conProp.get("EXT"); if (targetExt == null) targetExt = "";
targetClass = rewriteClassFile(new File(htDefaultPath, path));
if (path.endsWith("/") || path.endsWith("\\")) {
String testpath;
// look for indexForward setting
if (indexForward.length() > 0 && (targetFile = getOverlayedFile(path + indexForward)).exists()) {
testpath = path + indexForward;
targetClass = getOverlayedClass(testpath);
path = testpath;
} else {
// attach default file name(s)
for (final String defaultFile : defaultFiles) {
testpath = path + defaultFile;
targetFile = getOverlayedFile(testpath);
targetClass = getOverlayedClass(testpath);
if (targetFile.exists()) {
path = testpath;
break;
}
}
}
targetFile = getLocalizedFile(path, localeSelection);
//no defaultfile, send a dirlisting
if (targetFile == null || !targetFile.exists() || (targetFile.exists() && targetFile.isDirectory())) {
final StringBuilder aBuffer = new StringBuilder();
aBuffer.append("<html>\n<head>\n</head>\n<body>\n<h1>Index of " + path + "</h1>\n <ul>\n");
String[] list = targetFile.list();
if (list == null) list = new String[0]; // should not occur!
File f;
String size;
long sz;
String headline, author, description, publisher;
int images, links;
ContentScraper scraper;
for (final String element : list) {
f = new File(targetFile, element);
if (f.isDirectory()) {
aBuffer.append(" <li><a href=\"" + path + element + "/\">" + element + "/</a><br/></li>\n");
} else {
if (element.endsWith("html") || (element.endsWith("htm"))) {
scraper = ContentScraper.parseResource(f);
headline = scraper.getTitle();
author = scraper.getAuthor();
publisher = scraper.getPublisher();
description = scraper.getDescription();
images = scraper.getImages().size();
links = scraper.getAnchors().size();
} else {
headline = null;
author = null;
publisher = null;
description = null;
images = 0;
links = 0;
}
sz = f.length();
if (sz < 1024) {
size = sz + " bytes";
} else if (sz < 1024 * 1024) {
size = (sz / 1024) + " KB";
} else {
size = (sz / 1024 / 1024) + " MB";
}
aBuffer.append(" <li>");
if (headline != null && headline.length() > 0) aBuffer.append("<a href=\"" + element + "\"><b>" + headline + "</b></a><br/>");
aBuffer.append("<a href=\"" + path + element + "\">" + element + "</a><br/>");
if (author != null && author.length() > 0) aBuffer.append("Author: " + author + "<br/>");
if (publisher != null && publisher.length() > 0) aBuffer.append("Publisher: " + publisher + "<br/>");
if (description != null && description.length() > 0) aBuffer.append("Description: " + description + "<br/>");
aBuffer.append(GenericFormatter.SHORT_DAY_FORMATTER.format(new Date(f.lastModified())) + ", " + size + ((images > 0) ? ", " + images + " images" : "") + ((links > 0) ? ", " + links + " links" : "") + "<br/></li>\n");
}
}
aBuffer.append(" </ul>\n</body>\n</html>\n");
// write the list to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, "text/html; charset=UTF-8", aBuffer.length(), new Date(targetFile.lastModified()), null, new ResponseHeader(), null, null, true);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
out.write(UTF8.getBytes(aBuffer.toString()));
}
return;
}
} else {
//XXX: you cannot share a .png/.gif file with a name like a class in htroot.
if ( !(targetFile.exists()) &&
!((path.endsWith("png")||path.endsWith("gif") ||
matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) ||
path.endsWith(".stream")) &&
targetClass!=null ) ){
targetFile = new File(htDocsPath, path);
targetClass = rewriteClassFile(new File(htDocsPath, path));
}
}
// implement proxy via url (not in servlet, because we need binary access on ouputStream)
if (path.equals("/proxy.html")) {
final List<Pattern> urlProxyAccess = Domains.makePatterns(sb.getConfig("proxyURL.access", "127.0.0.1"));
final UserDB.Entry user = sb.userDB.getUser(requestHeader);
final boolean user_may_see_proxyurl = Domains.matchesList(clientIP, urlProxyAccess) || (user!=null && user.hasRight(UserDB.AccessRight.PROXY_RIGHT));
if (sb.getConfigBool("proxyURL", false) && user_may_see_proxyurl) {
doURLProxy(args, conProp, requestHeader, out);
return;
}
else {
HTTPDemon.sendRespondError(conProp,out,3,403,"Access denied",null,null);
}
}
// track all files that had been accessed so far
if (targetFile != null && targetFile.exists()) {
if (args != null && args.size() > 0) sb.setConfig("server.servlets.submitted", appendPath(sb.getConfig("server.servlets.submitted", ""), path));
}
//File targetClass = rewriteClassFile(targetFile);
//We need tp here
servletProperties templatePatterns = null;
Date targetDate;
if ((targetClass != null) && (path.endsWith("png"))) {
// call an image-servlet to produce an on-the-fly - generated image
Object img = null;
try {
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "png");
// in case that there are no args given, args = null or empty hashmap
img = invokeServlet(targetClass, requestHeader, args);
} catch (final InvocationTargetException e) {
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() +
" target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" +
e.getTargetException().getMessage() +
"; java.awt.graphicsenv='" + System.getProperty("java.awt.graphicsenv","") + "'");
Log.logException(e);
Log.logException(e.getTargetException());
targetClass = null;
}
if (img == null) {
// error with image generation; send file-not-found
HTTPDemon.sendRespondError(conProp, out, 3, 404, "File not Found", null, null);
} else {
if (img instanceof RasterPlotter) {
final RasterPlotter yp = (RasterPlotter) img;
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = Classification.ext2mime(targetExt, "text/html");
final ByteBuffer result = RasterPlotter.exportImage(yp.getImage(), targetExt);
// write the array to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
result.writeTo(out);
}
}
if (img instanceof EncodedImage) {
final EncodedImage yp = (EncodedImage) img;
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = Classification.ext2mime(targetExt, "text/html");
final ByteBuffer result = yp.getImage();
// write the array to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
result.writeTo(out);
}
}
/*
if (img instanceof BufferedImage) {
final BufferedImage i = (BufferedImage) img;
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = MimeTable.ext2mime(targetExt, "text/html");
// generate an byte array from the generated image
int width = i.getWidth(); if (width < 0) width = 96; // bad hack
int height = i.getHeight(); if (height < 0) height = 96; // bad hack
final ByteBuffer result = RasterPlotter.exportImage(i, targetExt);
// write the array to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
result.writeTo(out);
}
}
*/
if (img instanceof Image) {
final Image i = (Image) img;
// send an image to client
targetDate = new Date(System.currentTimeMillis());
nocache = true;
final String mimeType = Classification.ext2mime(targetExt, "text/html");
// generate an byte array from the generated image
int width = i.getWidth(null); if (width < 0) width = 96; // bad hack
int height = i.getHeight(null); if (height < 0) height = 96; // bad hack
final BufferedImage bi = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
bi.createGraphics().drawImage(i, 0, 0, width, height, null);
final ByteBuffer result = RasterPlotter.exportImage(bi, targetExt);
// write the array to the client
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null, mimeType, result.length(), targetDate, null, null, null, null, nocache);
if (!method.equals(HeaderFramework.METHOD_HEAD)) {
result.writeTo(out);
}
}
}
// old-school CGI execution
} else if ((switchboard.getConfigBool("cgi.allow", false) // check if CGI execution is allowed in config
&& matchesSuffix(path, switchboard.getConfig("cgi.suffixes", null)) // "right" file extension?
&& path.substring(0, path.indexOf(targetFile.getName())).toUpperCase().contains("/CGI-BIN/") // file in right directory?
&& targetFile.exists())
) {
if (!targetFile.canExecute()) {
HTTPDemon.sendRespondError(
conProp,
out,
-1,
403,
null,
HeaderFramework.http1_1.get(
Integer.toString(403)),
null);
Log.logWarning(
"HTTPD",
"CGI script " + targetFile.getPath()
+ " could not be executed due to "
+ "insufficient access rights.");
} else {
String mimeType = "text/html";
int statusCode = 200;
final ProcessBuilder pb =
new ProcessBuilder(assembleCommandFromShebang(targetFile));
pb.directory(targetFile.getParentFile());
final String fileSeparator =
System.getProperty("file.separator", "/");
// set environment variables
final Map<String, String> env = pb.environment();
env.put(
"SERVER_SOFTWARE",
getDefaultHeaders(path).get(HeaderFramework.SERVER));
env.put("SERVER_NAME", sb.peers.mySeed().getName());
env.put("GATEWAY_INTERFACE", "CGI/1.1");
if (httpVersion != null) {
env.put("SERVER_PROTOCOL", httpVersion);
}
env.put("SERVER_PORT", switchboard.getConfig("port", "8090"));
env.put("REQUEST_METHOD", method);
// env.put("PATH_INFO", ""); // TODO: implement
// env.put("PATH_TRANSLATED", ""); // TODO: implement
env.put("SCRIPT_NAME", path);
if (argsString != null) {
env.put("QUERY_STRING", argsString);
}
env.put("REMOTE_ADDR", clientIP);
// env.put("AUTH_TYPE", ""); // TODO: implement
// env.put("REMOTE_USER", ""); // TODO: implement
// env.put("REMOTE_IDENT", ""); // I don't think we need this
env.put(
"DOCUMENT_ROOT",
switchboard.getAppPath().getAbsolutePath()
+ fileSeparator + switchboard.getConfig("htDocsPath", "DATA/HTDOCS"));
if (requestHeader.getContentType() != null) {
env.put("CONTENT_TYPE", requestHeader.getContentType());
}
if (method.equalsIgnoreCase(HeaderFramework.METHOD_POST)
&& body != null) {
env.put(
"CONTENT_LENGTH",
Integer.toString(requestHeader.getContentLength()));
}
/* add values from request header to environment
* (see: http://hoohoo.ncsa.uiuc.edu/cgi/env.html#headers) */
for (final Map.Entry<String, String> requestHeaderEntry
: requestHeader.entrySet()) {
env.put("HTTP_"
+ requestHeaderEntry.getKey().toUpperCase().replace("-", "_"),
requestHeaderEntry.getValue());
}
int exitValue = 0;
String cgiBody = null;
final StringBuilder error = new StringBuilder(256);
try {
// start execution of script
final Process p = pb.start();
final OutputStream os =
new BufferedOutputStream(p.getOutputStream());
if (method.equalsIgnoreCase(
HeaderFramework.METHOD_POST) && body != null) {
final byte[] buffer = new byte[1024];
int len = requestHeader.getContentLength();
while (len > 0) {
body.read(buffer);
len = len - buffer.length;
os.write(buffer);
}
}
os.close();
try {
p.waitFor();
} catch (final InterruptedException ex) {
}
exitValue = p.exitValue();
final InputStream is =
new BufferedInputStream(p.getInputStream());
final InputStream es =
new BufferedInputStream(p.getErrorStream());
final StringBuilder processOutput =
new StringBuilder(1024);
while (is.available() > 0) {
processOutput.append((char) is.read());
}
while (es.available() > 0) {
error.append((char) es.read());
}
int indexOfDelimiter = processOutput.indexOf("\n\n", 0);
final String[] cgiHeader;
if (indexOfDelimiter > -1) {
cgiHeader =
processOutput.substring(
0, indexOfDelimiter).split("\n");
} else {
cgiHeader = new String[0];
}
cgiBody = processOutput.substring(indexOfDelimiter + 1);
String key;
String value;
for (final String element : cgiHeader) {
indexOfDelimiter = element.indexOf(':');
key = element.substring(0, indexOfDelimiter).trim();
value = element.substring(indexOfDelimiter + 1).trim();
conProp.put(key, value);
if ("Cache-Control".equals(key)
&& "no-cache".equals(value)) {
nocache = true;
} else if ("Content-type".equals(key)) {
mimeType = value;
} else if ("Status".equals(key)) {
if (key.length() > 2) {
try {
statusCode =
Integer.parseInt(
value.substring(0, 3));
} catch (final NumberFormatException ex) {
Log.logWarning(
"HTTPD",
"CGI script " + targetFile.getPath()
+ " returned illegal status code \""
+ value + "\".");
}
}
}
}
} catch (final IOException ex) {
exitValue = -1;
}
/* did the script return an exit value != 0
* and still there is supposed to be
* everything right with the HTTP status?
* -> change status to 500 since 200 would
* be a lie
*/
if (exitValue != 0 && statusCode == 200) {
statusCode = 500;
}
targetDate = new Date(System.currentTimeMillis());
if (exitValue == 0
&& cgiBody != null
&& !cgiBody.isEmpty()) {
HTTPDemon.sendRespondHeader(
conProp,
out,
httpVersion,
statusCode,
null,
mimeType,
cgiBody.length(),
targetDate,
null,
null,
null,
null,
nocache);
out.write(UTF8.getBytes(cgiBody));
} else {
HTTPDemon.sendRespondError(
conProp,
out,
exitValue,
statusCode,
null,
HeaderFramework.http1_1.get(
Integer.toString(statusCode)),
null);
Log.logWarning(
"HTTPD",
"CGI script " + targetFile.getPath()
+ " returned exit value " + exitValue
+ ", body empty: "
+ (cgiBody == null || cgiBody.isEmpty()));
if (error.length() > 0) {
Log.logWarning("HTTPD", "Reported error: " + error);
}
}
}
} else if ((targetClass != null) && (path.endsWith(".stream"))) {
// call rewrite-class
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, "stream");
//requestHeader.put(httpHeader.CONNECTION_PROP_INPUTSTREAM, body);
//requestHeader.put(httpHeader.CONNECTION_PROP_OUTPUTSTREAM, out);
HTTPDemon.sendRespondHeader(conProp, out, httpVersion, 200, null);
// in case that there are no args given, args = null or empty hashmap
/* servletProperties tp = (servlerObjects) */ invokeServlet(targetClass, requestHeader, args);
forceConnectionClose(conProp);
return;
} else if (targetFile.exists() && targetFile.isFile() && targetFile.canRead()) {
// we have found a file that can be written to the client
// if this file uses templates, then we use the template
// re-write - method to create an result
String mimeType = Classification.ext2mime(targetExt, "text/html");
String ext = (String) conProp.get("EXT"); if (ext == null) ext = "";
final boolean zipContent = requestHeader.acceptGzip() && HTTPDemon.shallTransportZipped("." + ext);
if (path.endsWith("html") ||
path.endsWith("htm") ||
path.endsWith("xml") ||
path.endsWith("json") ||
path.endsWith("rdf") ||
path.endsWith("rss") ||
path.endsWith("csv") ||
path.endsWith("pac") ||
path.endsWith("src") ||
path.endsWith("vcf") ||
path.endsWith("kml") ||
path.endsWith("gpx") ||
path.endsWith("css") ||
path.endsWith("/") ||
path.equals("/robots.txt")) {
/*targetFile = getLocalizedFile(path);
if (!(targetFile.exists())) {
// try to find that file in the htDocsPath
File trialFile = new File(htDocsPath, path);
if (trialFile.exists()) targetFile = trialFile;
}*/
// call rewrite-class
if (targetClass != null) {
// CGI-class: call the class to create a property for rewriting
try {
requestHeader.put(HeaderFramework.CONNECTION_PROP_CLIENTIP, (String) conProp.get(HeaderFramework.CONNECTION_PROP_CLIENTIP));
requestHeader.put(HeaderFramework.CONNECTION_PROP_PATH, path);
final int ep = path.lastIndexOf(".");
requestHeader.put(HeaderFramework.CONNECTION_PROP_EXT, path.substring(ep + 1));
// in case that there are no args given, args = null or empty hashmap
final Object tmp = invokeServlet(targetClass, requestHeader, args);
if (tmp == null) {
// if no args given, then tp will be an empty Hashtable object (not null)
templatePatterns = new servletProperties();
} else if (tmp instanceof servletProperties) {
templatePatterns = (servletProperties) tmp;
} else {
templatePatterns = new servletProperties((serverObjects) tmp);
}
// check if the servlets requests authentication
if (templatePatterns.containsKey(servletProperties.ACTION_AUTHENTICATE)) {
// handle brute-force protection
if (realmProp != null) {
Log.logInfo("HTTPD", "dynamic log-in for account 'admin' in http file handler for path '" + path + "' from host '" + clientIP + "'");
final Integer attempts = serverCore.bfHost.get(clientIP);
if (attempts == null)
serverCore.bfHost.put(clientIP, Integer.valueOf(1));
else
serverCore.bfHost.put(clientIP, Integer.valueOf(attempts.intValue() + 1));
}
// send authentication request to browser
final ResponseHeader headers = getDefaultHeaders(path);
headers.put(RequestHeader.WWW_AUTHENTICATE,"Basic realm=\"" + templatePatterns.get(servletProperties.ACTION_AUTHENTICATE, "") + "\"");
HTTPDemon.sendRespondHeader(conProp,out,httpVersion,401,headers);
return;
} else if (templatePatterns.containsKey(servletProperties.ACTION_LOCATION)) {
String location = templatePatterns.get(servletProperties.ACTION_LOCATION, "");
if (location.length() == 0) location = path;
final ResponseHeader headers = getDefaultHeaders(path);
headers.setAdditionalHeaderProperties(templatePatterns.getOutgoingHeader().getAdditionalHeaderProperties()); //put the cookies into the new header TODO: can we put all headerlines, without trouble?
headers.put(HeaderFramework.LOCATION,location);
HTTPDemon.sendRespondHeader(conProp,out,httpVersion,302,headers);
return;
}
// add the application version, the uptime and the client name to every rewrite table
templatePatterns.put(servletProperties.PEER_STAT_VERSION, yacyBuildProperties.getVersion());
templatePatterns.put(servletProperties.PEER_STAT_UPTIME, ((System.currentTimeMillis() - serverCore.startupTime) / 1000) / 60); // uptime in minutes
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTNAME, sb.peers.mySeed().getName());
templatePatterns.putHTML(servletProperties.PEER_STAT_CLIENTID, ((Switchboard) switchboard).peers.myID());
templatePatterns.put(servletProperties.PEER_STAT_MYTIME, GenericFormatter.SHORT_SECOND_FORMATTER.format());
final Seed myPeer = sb.peers.mySeed();
templatePatterns.put("newpeer", myPeer.getAge() >= 1 ? 0 : 1);
templatePatterns.putHTML("newpeer_peerhash", myPeer.hash);
//System.out.println("respond props: " + ((tp == null) ? "null" : tp.toString())); // debug
} catch (final InvocationTargetException e) {
if (e.getCause() instanceof InterruptedException) {
throw new InterruptedException(e.getCause().getMessage());
}
theLogger.logSevere("INTERNAL ERROR: " + e.toString() + ":" +
e.getMessage() +
" target exception at " + targetClass + ": " +
e.getTargetException().toString() + ":" +
e.getTargetException().getMessage(),e);
targetClass = null;
throw e;
}
nocache = true;
}
targetDate = new Date(targetFile.lastModified());
Date expireDate = null;
if (templatePatterns == null) {
// if the file will not be changed, cache it in the browser
expireDate = new Date(new Date().getTime() + (31l * 24 * 60 * 60 * 1000));
}
// rewrite the file
InputStream fis = null;
// read the file/template
TemplateCacheEntry templateCacheEntry = null;
final long fileSize = targetFile.length();
if (useTemplateCache && fileSize <= 512 * 1024) {
// read from cache
SoftReference<TemplateCacheEntry> ref = templateCache.get(targetFile);
if (ref != null) {
templateCacheEntry = ref.get();
if (templateCacheEntry == null) templateCache.remove(targetFile);
}
final Date targetFileDate = new Date(targetFile.lastModified());
if (templateCacheEntry == null || targetFileDate.after(templateCacheEntry.lastModified)) {
// loading the content of the template file into
// a byte array
templateCacheEntry = new TemplateCacheEntry();
templateCacheEntry.lastModified = targetFileDate;
templateCacheEntry.content = FileUtils.read(targetFile);
// storing the content into the cache
ref = new SoftReference<TemplateCacheEntry>(templateCacheEntry);
if (MemoryControl.shortStatus()) templateCache.clear();
templateCache.put(targetFile, ref);
if (theLogger.isFinest()) theLogger.logFinest("Cache MISS for file " + targetFile);
} else {
if (theLogger.isFinest()) theLogger.logFinest("Cache HIT for file " + targetFile);
}
// creating an inputstream needed by the template
// rewrite function
fis = new ByteArrayInputStream(templateCacheEntry.content);
templateCacheEntry = null;
} else if (fileSize <= Math.min(4 * 1024 * 1204, MemoryControl.available() / 100)) {
// read file completely into ram, avoid that too many files are open at the same time
fis = new ByteArrayInputStream(FileUtils.read(targetFile));
} else {
fis = new BufferedInputStream(new FileInputStream(targetFile));
}
if (mimeType.startsWith("text")) {
// every text-file distributed by yacy is UTF-8
if(!path.startsWith("/repository")) {
mimeType = mimeType + "; charset=UTF-8";
} else {
// detect charset of html-files
if((path.endsWith("html") || path.endsWith("htm"))) {
// save position
fis.mark(1000);
// scrape document to look up charset
final ScraperInputStream htmlFilter = new ScraperInputStream(fis,"UTF-8",new DigestURI("http://localhost"),null,false);
final String charset = htmlParser.patchCharsetEncoding(htmlFilter.detectCharset());
if(charset != null)
mimeType = mimeType + "; charset="+charset;
// reset position
fis.reset();
}
}
}
// write the array to the client
// we can do that either in standard mode (whole thing completely) or in chunked mode
// since yacy clients do not understand chunked mode (yet), we use this only for communication with the administrator
final boolean yacyClient = requestHeader.userAgent().startsWith("yacy");
final boolean chunked = !method.equals(HeaderFramework.METHOD_HEAD) && !yacyClient && httpVersion.equals(HeaderFramework.HTTP_VERSION_1_1);
if (chunked) {
// send page in chunks and parse SSIs
final ByteBuffer o = new ByteBuffer();
// apply templates
TemplateEngine.writeTemplate(fis, o, templatePatterns, UNRESOLVED_PATTERN);
fis.close();
HTTPDemon.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, -1,
targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
null, "chunked", nocache);
// send the content in chunked parts, see RFC 2616 section 3.6.1
final ChunkedOutputStream chos = new ChunkedOutputStream(out);
// GZIPOutputStream does not implement flush (this is a bug IMHO)
// so we can't compress this stuff, without loosing the cool SSI trickle feature
ServerSideIncludes.writeSSI(o, chos, realmProp, clientIP, requestHeader);
//chos.write(result);
chos.finish();
} else {
// send page as whole thing, SSIs are not possible
final String contentEncoding = (zipContent) ? "gzip" : null;
// apply templates
final ByteBuffer o1 = new ByteBuffer();
TemplateEngine.writeTemplate(fis, o1, templatePatterns, ASCII.getBytes("-UNRESOLVED_PATTERN-"));
fis.close();
final ByteBuffer o = new ByteBuffer();
if (zipContent) {
GZIPOutputStream zippedOut = new GZIPOutputStream(o);
ServerSideIncludes.writeSSI(o1, zippedOut, realmProp, clientIP, requestHeader);
//httpTemplate.writeTemplate(fis, zippedOut, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
zippedOut.finish();
zippedOut.flush();
zippedOut.close();
zippedOut = null;
} else {
ServerSideIncludes.writeSSI(o1, o, realmProp, clientIP, requestHeader);
//httpTemplate.writeTemplate(fis, o, tp, "-UNRESOLVED_PATTERN-".getBytes("UTF-8"));
}
if (method.equals(HeaderFramework.METHOD_HEAD)) {
HTTPDemon.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, o.length(),
targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
contentEncoding, null, nocache);
} else {
final byte[] result = o.getBytes(); // this interrupts streaming (bad idea!)
HTTPDemon.sendRespondHeader(conProp, out,
httpVersion, 200, null, mimeType, result.length,
targetDate, expireDate, (templatePatterns == null) ? new ResponseHeader() : templatePatterns.getOutgoingHeader(),
contentEncoding, null, nocache);
FileUtils.copy(result, out);
}
}
} else { // no html
int statusCode = 200;
int rangeStartOffset = 0;
final ResponseHeader header = new ResponseHeader();
// adding the accept ranges header
header.put(HeaderFramework.ACCEPT_RANGES, "bytes");
// reading the files md5 hash if availabe and use it as ETAG of the resource
String targetMD5 = null;
final File targetMd5File = new File(targetFile + ".md5");
try {
if (targetMd5File.exists()) {
//String description = null;
targetMD5 = UTF8.String(FileUtils.read(targetMd5File));
final int pos = targetMD5.indexOf('\n');
if (pos >= 0) {
//description = targetMD5.substring(pos + 1);
targetMD5 = targetMD5.substring(0, pos);
}
// using the checksum as ETAG header
header.put(HeaderFramework.ETAG, targetMD5);
}
} catch (final IOException e) {
Log.logException(e);
}
if (requestHeader.containsKey(HeaderFramework.RANGE)) {
final Object ifRange = requestHeader.ifRange();
if ((ifRange == null)||
(ifRange instanceof Date && targetFile.lastModified() == ((Date)ifRange).getTime()) ||
(ifRange instanceof String && ifRange.equals(targetMD5))) {
final String rangeHeaderVal = requestHeader.get(HeaderFramework.RANGE).trim();
if (rangeHeaderVal.startsWith("bytes=")) {
final String rangesVal = rangeHeaderVal.substring("bytes=".length());
final String[] ranges = rangesVal.split(",");
if ((ranges.length == 1)&&(ranges[0].endsWith("-"))) {
rangeStartOffset = Integer.parseInt(ranges[0].substring(0,ranges[0].length()-1));
statusCode = 206;
header.put(HeaderFramework.CONTENT_RANGE, "bytes " + rangeStartOffset + "-" + (targetFile.length()-1) + "/" + targetFile.length());
}
}
}
}