package net.matuschek.jobo;
import java.awt.*;
import java.net.MalformedURLException;
import java.net.URL;
import java.text.DecimalFormat;
import java.util.Date;
import javax.swing.*;
import net.matuschek.http.HttpTool;
import net.matuschek.http.HttpToolCallback;
import net.matuschek.http.cookie.Cookie;
import net.matuschek.http.cookie.CookieException;
import net.matuschek.spider.WebRobotCallback;
import net.matuschek.swing.OptionPanel;
import net.matuschek.swing.VerticalAlignPanel;
import org.apache.log4j.BasicConfigurator;
/**
* The JoBo interface with Swing GUI.
*
* @author Daniel Matuschek
* @version $Revision: 1.52 $
*/
public class JoBoSwing
extends JFrame
implements HttpToolCallback,
WebRobotCallback
{
/** JoBo version */
final static String VERSION="1.4";
//private static Category log = Category.getInstance("");
private static LogFrame logFrame;
private static LogFrameAppender lfAppend=null;
private JoBoBase jobobase = null;
private Thread robotThread=null;
private Date docDownloadStarted=new Date();
private long httpToolDocSize = 0;
private int robotCount = 0;
private long robotSize = 0;
private int robotQueueSize = 0;
// Swing variables declaration
private JTextField urlField;
private JTextField directoryField;
private JProgressBar progressBar;
private JTextField currentUrlField;
private JTextField queuedField;
private JTextField retrievedField;
private JButton runStopButton;
private JButton sleepButton;
// End of Swing variables
// Other frames
private RobotConfigFrame robotConfigFrame = null;
private URLCheckConfigFrame urlCheckConfigFrame = null;
private FilterConfigFrame filterConfigFrame = null;
private AllowedListFrame allowedURLsFrame = null;
// End of other frames
/** Creates new form JoBoSwing */
public JoBoSwing() {
try {
jobobase = JoBoBase.createFromXML();
} catch (ClassNotFoundException e) {
System.out.println("Could not initialize WebRobot: "+e);
System.exit(1);
}
jobobase.registerHttpToolCallback(this);
jobobase.registerWebRobotCallback(this);
initComponents ();
pack ();
updateDialogFromRobot();
}
private void initComponents() {
getContentPane().setLayout(new BorderLayout());
setTitle("JoBo/Swing: http://www.matuschek.net/jobo/");
addWindowListener(new java.awt.event.WindowAdapter() {
public void windowClosing(java.awt.event.WindowEvent evt) {
exitApp();
}
}
);
/** Heading **/
VerticalAlignPanel headPanel = new VerticalAlignPanel();
JLabel heading1 = new JLabel();
heading1.setText("JoBo "+VERSION);
heading1.setHorizontalAlignment(JTextField.CENTER);
heading1.setFont(new java.awt.Font ("Dialog", 0, 24));
JLabel heading2 = new JLabel();
heading2.setText("look at http://www.matuschek.net/jobo/");
heading2.setHorizontalAlignment(JTextField.CENTER);
heading2.setFont(new java.awt.Font ("Dialog", 0, 12));
JLabel heading3 = new JLabel();
heading3.setText("for more details");
heading3.setHorizontalAlignment(JTextField.CENTER);
heading3.setFont(new java.awt.Font ("Dialog", 0, 12));
headPanel.add(heading1);
headPanel.add(heading2);
headPanel.add(heading3);
getContentPane().add(headPanel,BorderLayout.NORTH);
/* fields */
OptionPanel optPanel = new OptionPanel(2);
urlField = new JTextField();
urlField.setColumns(40);
urlField.setText("http://");
optPanel.add("URL:",urlField);
directoryField = new JTextField();
directoryField.setText(jobobase.getStorageDirectory());
directoryField.setColumns(40);
optPanel.add("Storage directory:",directoryField);
currentUrlField = new JTextField();
currentUrlField.setEditable(false);
currentUrlField.setColumns(40);
currentUrlField.setText("-");
optPanel.add("Current URL:",currentUrlField);
retrievedField = new JTextField();
retrievedField.setEditable(false);
retrievedField.setColumns(40);
optPanel.add("Retrieved:",retrievedField);
queuedField = new JTextField();
queuedField.setEditable(false);
queuedField.setColumns(40);
optPanel.add("Queued:",queuedField);
progressBar = new JProgressBar();
progressBar.setStringPainted(true);
optPanel.add("Progress:",progressBar,GridBagConstraints.HORIZONTAL);
getContentPane().add(optPanel);
/** Buttons **/
JPanel buttonPanel = new JPanel();
JButton butt = null;
runStopButton = new JButton();
runStopButton.setText("Run");
runStopButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
runStopButtonActionPerformed();
}
}
);
buttonPanel.add(runStopButton);
sleepButton = new JButton();
sleepButton.setText("Sleep");
sleepButton.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
sleepButtonActionPerformed();
}
}
);
buttonPanel.add(sleepButton);
butt = new JButton();
butt.setText("Robot settings");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
doRobotSettingsDialog();
}
}
);
buttonPanel.add(butt);
butt = new JButton();
butt.setText("Save");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
saveSettings();
}
}
);
// buttonPanel.add(butt);
butt = new JButton();
butt.setText("URLCheck");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
configureURLCheck();
}
}
);
// buttonPanel.add(butt);
butt = new JButton();
butt.setText("Filter configuration");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
configureFilters();
}
}
);
// buttonPanel.add(butt);
butt = new JButton();
butt.setText("Allowed URLs");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
configureAllowedURLs();
}
});
buttonPanel.add(butt);
butt = new JButton();
butt.setText("Add cookie");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
addCookie();
}
});
buttonPanel.add(butt);
butt = new JButton();
butt.setText("Log");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
logButtonActionPerformed();
}
});
buttonPanel.add(butt);
butt = new JButton();
butt.setText("Exit");
butt.addActionListener(new java.awt.event.ActionListener() {
public void actionPerformed(java.awt.event.ActionEvent evt) {
exitApp();
}
});
buttonPanel.add(butt);
getContentPane().add(buttonPanel,BorderLayout.SOUTH);
}
private void sleepButtonActionPerformed() {
// is there a robotThread running ?
if ((robotThread != null) &&
(robotThread.isAlive())) {
if (jobobase.getRobot().isSleeping()) {
jobobase.getRobot().setSleep(false);
sleepButton.setText("Sleep");
} else {
jobobase.getRobot().setSleep(true);
sleepButton.setText("Wake up");
}
}
}
private void logButtonActionPerformed() {
logFrame.setVisible(true);
}
private void runStopButtonActionPerformed() {
// is there another robotThread running ?
if ((robotThread != null) &&
(robotThread.isAlive())) {
jobobase.getRobot().stopRobot();
} else {
if (updateRobotFromDialog()) {
runStopButton.setText("Stop");
// clear cookies from last run
//jobobase.getRobot().clearCookies();
robotThread = new Thread(jobobase.getRobot());
robotThread.start();
}
}
}
/**
* open the robot settings dialog
*/
private void doRobotSettingsDialog() {
if (robotConfigFrame == null) {
robotConfigFrame = new RobotConfigFrame(jobobase);
}
robotConfigFrame.setVisible(true);
}
/**
* configure URLCheck
*/
private void configureURLCheck() {
if (urlCheckConfigFrame == null) {
urlCheckConfigFrame = new URLCheckConfigFrame(jobobase.getURLCheck());
}
urlCheckConfigFrame.setVisible(true);
}
/**
* configure URLCheck
*/
private void addCookie() {
String cookieStr = JOptionPane.showInputDialog(this, "Cookie string:");
String domain = JOptionPane.showInputDialog(this, "Domain:");
URL url;
try {
url = new URL("http://"+domain);
} catch (MalformedURLException e1) {
JOptionPane.showMessageDialog(this, "Domain invalid: "+e1.getMessage());
return;
}
try {
if (cookieStr.startsWith("Set-Cookie")) {
Cookie cookie;
cookie = new Cookie(cookieStr,url);
jobobase.getRobot().getCookieManager().add(cookie);
JOptionPane.showMessageDialog(this, "Cookie added");
} else {
Cookie[] cookies = Cookie.cookieStringToCookies(cookieStr, domain);
for (int i=0; i<cookies.length; i++) {
System.out.println(cookies[i]);
jobobase.getRobot().getCookieManager().add(cookies[i]);
}
JOptionPane.showMessageDialog(this, cookies.length+" cookies added");
}
} catch (CookieException e) {
JOptionPane.showMessageDialog(this, "Cookie string invalid: "+e.getMessage());
}
}
/**
* configure document filters
*/
private void configureFilters() {
if (filterConfigFrame == null) {
filterConfigFrame = new FilterConfigFrame();
}
filterConfigFrame.setVisible(true);
}
/**
* configure allowed URls
*/
private void configureAllowedURLs() {
if (allowedURLsFrame == null) {
allowedURLsFrame =
new AllowedListFrame(jobobase.getRobot().getAllowedURLs());
}
allowedURLsFrame.setVisible(true);
}
/**
* store current settings
*/
private void saveSettings() {
jobobase.saveConfig("test.xml");
}
/**
* Exit the Application
*/
private void exitApp() {
System.exit(0);
}
/**
* @param args the command line arguments
*/
public static void main (String[] args) {
// create a new frame for logging
logFrame = new LogFrame();
logFrame.addMsg("Starting ...");
// configure Log4J subsystem
lfAppend = new LogFrameAppender(logFrame);
BasicConfigurator.configure(lfAppend);
// other command line arguments
for (int i=0; i<args.length; i++) {
if (args[i].equals("-debug")) {
// send all log output to standard out (including level debug)
BasicConfigurator.configure();
}
}
new JoBoSwing().setVisible(true);
}
/**
* update the WebRobot setting using the values of the input fields
* @return true if everything is ok, false otherwise
*/
private boolean updateRobotFromDialog() {
// start URL
String startUrl = urlField.getText();
try {
URL u = new URL(startUrl);
jobobase.getRobot().setStartURL(u);
} catch (MalformedURLException e) {
JOptionPane.showMessageDialog(this,"URL "+startUrl+" is invalid");
return false;
}
// storage directory
String storageDir = directoryField.getText();
jobobase.setStorageDirectory(storageDir);
return true;
}
/**
* update the dialog field from the current setting in the webrobot
* will only be called on startup !
*/
private void updateDialogFromRobot() {
// start URL
URL u = jobobase.getRobot().getStartURL();
if (u != null) {
urlField.setText(u.toString());
} else {
urlField.setText("");
}
}
/***********************************************************************
***********************************************************************
Callback interface for httpTool
***********************************************************************
***********************************************************************/
public void setHttpToolDocUrl(String url) {
try {
currentUrlField.setText(urlToString(new URL(url)));
} catch (MalformedURLException e) {}
}
public void setHttpToolDocSize(int size) {
this.httpToolDocSize = size;
progressBar.setMaximum(size);
}
public void setHttpToolDocCurrentSize(int size) {
StringBuffer progressStr = new StringBuffer();
progressBar.setValue(size);
int kB = size / 1024;
long longsize = size;
Date current = new Date();
long millisecs = (current.getTime()-docDownloadStarted.getTime());
long ratio = 0;
if (millisecs > 0) {
ratio = ((longsize*1000)/millisecs);
} else {
ratio = 0;
}
progressStr.append(kB);
progressStr.append(" kB");
// document size known ?
if ( this.httpToolDocSize > 0) {
progressStr.append(" of ");
progressStr.append(httpToolDocSize / 1024);
progressStr.append(" kB");
}
// ratio
DecimalFormat myFormat = new DecimalFormat("####0.000");
progressStr.append(", ");
progressStr.append(myFormat.format((float)(ratio)/1000));
progressStr.append(" kB/s");
// time left
if (this.httpToolDocSize > 0) {
long secleft;
if (ratio > 0)
secleft = (this.httpToolDocSize - longsize)/ratio;
else
secleft=100000;
long min = secleft/60;
int sec = (int)(secleft%60);
String secStr = Integer.toString(sec);
if (secStr.length()<2) {
secStr = "0"+secStr;
}
progressStr.append(", ");
progressStr.append(min);
progressStr.append(":");
progressStr.append(secStr);
progressStr.append(" min left");
}
progressBar.setString(progressStr.toString());
}
public void setHttpToolStatus(int status) {
if (status == HttpTool.STATUS_CONNECTING) {
progressBar.setString("Connecting");
this.setHttpToolDocSize(0);
} else if (status == HttpTool.STATUS_CONNECTED) {
progressBar.setString("Connected");
docDownloadStarted=new Date();
} else if (status == HttpTool.STATUS_RETRIEVING) {
progressBar.setString("Retrieving");
} else if (status == HttpTool.STATUS_DONE) {
progressBar.setString("Finished");
} else if (status == HttpTool.STATUS_DENIEDBYRULE) {
progressBar.setString("Denied by rule");
} else {
progressBar.setString("Unknown status ("+status+")");
}
}
/**
* converts a URL to a textual representation where the authentication
* info is hidden
* @param u
* @return a textual representation
*/
protected String urlToString(URL u) {
if (u==null) {
return null;
}
String userInfo=u.getUserInfo();
if ((userInfo != null) &&
(! userInfo.equals(""))) {
userInfo="authenticated@";
} else {
userInfo="";
}
return u.getProtocol()+"://"+userInfo+u.getHost()+u.getPath();
}
/***********************************************************************
***********************************************************************
End of callback interface for httpTool
***********************************************************************
***********************************************************************/
public void webRobotRetrievedDoc(String url, int size) {
robotCount++;
robotSize += size;
updateControls();
}
public void webRobotUpdateQueueStatus(int length) {
robotQueueSize=length;
updateControls();
}
public void webRobotDone() {
progressBar.setString("Download completed");
robotCount=0;
robotSize=0;
runStopButton.setText("Run");
}
public void webRobotSleeping(boolean sleeping) {
if (sleeping) {
progressBar.setString("sleeping");
} else {
progressBar.setString("");
}
}
/***********************************************************************
***********************************************************************
End of callback interface for webRobot
***********************************************************************
***********************************************************************/
/**
* update status of dynamic elements
*/
protected void updateControls() {
DecimalFormat myFormat = new DecimalFormat("###,###,###.0");
String retrievedContent=
robotCount+" files with "
+myFormat.format((float)robotSize/1024)
+" kB";
String queuedContent=robotQueueSize+"";
retrievedField.setText(retrievedContent);
queuedField.setText(queuedContent);
}
}