1 package net.sourceforge.phpeclipse.wiki.actions.mediawiki.connect;
3 //Parts of this sources are copied and modified from the jEdit Wikipedia plugin:
4 //http://www.djini.de/software/wikipedia/index.html
6 //The modified sources are available under the "Common Public License"
7 //with permission from the original author: Daniel Wunsch
9 import java.io.IOException;
10 import java.io.StringReader;
11 import java.io.UnsupportedEncodingException;
12 import java.net.URLDecoder;
13 import java.util.ArrayList;
14 import java.util.regex.Matcher;
15 import java.util.regex.Pattern;
17 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.config.IWikipedia;
18 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.MethodException;
19 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.PageNotEditableException;
20 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.UnexpectedAnswerException;
21 import net.sourceforge.phpeclipse.wiki.editor.WikiEditorPlugin;
23 import org.apache.commons.httpclient.ConnectMethod;
24 import org.apache.commons.httpclient.HostConfiguration;
25 import org.apache.commons.httpclient.HttpClient;
26 import org.apache.commons.httpclient.HttpConnection;
27 import org.apache.commons.httpclient.HttpException;
28 import org.apache.commons.httpclient.HttpMethod;
29 import org.apache.commons.httpclient.HttpState;
30 import org.apache.commons.httpclient.HttpStatus;
31 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
32 import org.apache.commons.httpclient.NameValuePair;
33 import org.apache.commons.httpclient.URI;
34 import org.apache.commons.httpclient.UsernamePasswordCredentials;
35 import org.apache.commons.httpclient.methods.GetMethod;
36 import org.apache.commons.httpclient.methods.PostMethod;
37 import org.apache.commons.httpclient.protocol.Protocol;
38 import org.apache.commons.httpclient.util.EncodingUtil;
39 import org.eclipse.core.runtime.CoreException;
40 import org.eclipse.core.runtime.Preferences;
43 * This class gets the wikitext from a wikipedia edit page
45 * The basic coding was copied from the commons-httpclient example <code>MediaWikiConnector.java</code>
47 public class MediaWikiConnector {
48 //pattern used to scarp an edit page
49 private static final Pattern BODY_PATTERN = Pattern.compile(
51 * action=".*?title=(.*?)(&|\") <form id="editform" name="editform" method="post"
52 * action="/w/wiki.phtml?title=Ammersee&action=submit" locked pages: <textarea cols='80' rows='25' readonly>
54 ".*<form[^>]*\\sid=\"editform\"[^>]*title=(.*?)&[^>]*>" + ".*<textarea[^>]*\\sname=\"wpTextbox1\"[^>]*>(.*?)</textarea>"
55 + ".*<input[^>]*\\svalue=\"(\\d*)\"[^>]*\\sname=\"wpEdittime\"[^>]*>" + ".*", Pattern.DOTALL);
57 //setup default user agent
58 final static public String userAgent = "plog4u.org/0.0";
60 // create a ConnectionManager
61 private MultiThreadedHttpConnectionManager manager;
63 private HttpClient client;
66 * Delay a new store to 1 second
68 private Throttle storeThrottle = new Throttle(1000);
71 private long nextTime = 0;
73 private final long minimumDelay;
75 public Throttle(long minimumDelay) {
76 this.minimumDelay = minimumDelay;
79 /** this is called from the client */
80 public synchronized void delay() throws InterruptedException {
81 long delay = nextTime - System.currentTimeMillis();
84 nextTime = System.currentTimeMillis() + minimumDelay;
88 public MediaWikiConnector() {
89 // <a href="javascript:window.location.href='http://127.0.0.1:8009/open/?' + window.location.href">bookmarklet</a>
90 manager = new MultiThreadedHttpConnectionManager();
91 manager.setMaxConnectionsPerHost(6);
92 manager.setMaxTotalConnections(18);
93 manager.setConnectionStaleCheckingEnabled(true);
94 // open the conversation
95 client = new HttpClient(manager);
96 setHTTPClientParameters(client);
97 //client.State.CookiePolicy = CookiePolicy.COMPATIBILITY;
98 //client.HostConfiguration.setHost(LOGON_SITE, LOGON_PORT, "http");
101 /** destructor freeing all resources. the Connection is not usable any more after calling this method */
102 public void destroy() {
106 /** log in - returns success */
107 public boolean login(IWikipedia config, String actionUrl, String user, String password, boolean remember)
108 throws UnexpectedAnswerException, MethodException {
109 PostMethod method = new PostMethod(actionUrl);
110 method.setFollowRedirects(false);
111 method.addRequestHeader("User-Agent", userAgent);
112 NameValuePair[] params = new NameValuePair[] {
113 new NameValuePair("title", config.getLoginTitle()),
114 new NameValuePair("action", "submit"),
115 new NameValuePair("wpName", user),
116 new NameValuePair("wpPassword", password),
117 new NameValuePair("wpRemember", remember ? "1" : "0"),
118 new NameValuePair("wpLoginattempt", "submit") };
119 method.addParameters(params);
123 int responseCode = client.executeMethod(method);
124 String responseBody = method.getResponseBodyAsString();
130 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
131 && responseBody.matches(config.getLoginSuccess())) {
133 } else if (responseCode == 200 && responseBody.matches(config.getLoginWrongPw()) || responseCode == 200
134 && responseBody.matches(config.getLoginNoUser())) {
136 if (responseBody.matches(config.getLoginNoUser())) {
137 throw new UnexpectedAnswerException("login not successful: wrong user name: " + user);
138 } else if (responseBody.matches(config.getLoginWrongPw())) {
139 throw new UnexpectedAnswerException("login not successful: wrong password for user: " + user);
141 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
144 throw new UnexpectedAnswerException("login not successful: " + method.getStatusLine());
146 } catch (HttpException e) {
147 throw new MethodException("method failed", e);
148 } catch (IOException e) {
149 throw new MethodException("method failed", e);
151 method.releaseConnection();
154 * // display cookies System.err.println("login: " + result); for (var cookie : client.State.Cookies) {
155 * System.err.println("cookie: " + cookie); }
159 SiteState state = SiteState.siteState(config);
160 state.loggedIn = result;
161 state.userName = user;
166 /** log out - return success */
167 public boolean logout(IWikipedia config, String actionUrl) throws UnexpectedAnswerException, MethodException {
168 GetMethod method = new GetMethod(actionUrl);
169 method.setFollowRedirects(false);
170 method.addRequestHeader("User-Agent", userAgent);
171 NameValuePair[] params = new NameValuePair[] {
172 new NameValuePair("title", config.getLogoutTitle()),
173 new NameValuePair("action", "submit") };
174 method.setQueryString(EncodingUtil.formUrlEncode(params, config.getCharSet()));
178 int responseCode = client.executeMethod(method);
179 String responseBody = method.getResponseBodyAsString();
182 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
183 && responseBody.matches(config.getLoginSuccess())) {
184 // config.getloggedIn = false;
186 } else if (responseCode == 200) {
187 //### should check for a failure message
189 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
191 throw new UnexpectedAnswerException("logout not successful: " + method.getStatusLine());
193 } catch (HttpException e) {
194 throw new MethodException("method failed", e);
195 } catch (IOException e) {
196 throw new MethodException("method failed", e);
198 method.releaseConnection();
202 SiteState state = SiteState.siteState(config);
203 state.loggedIn = false;
208 /** parses a returned editform into a Content object with UNIX-EOLs ("\n") */
209 private Parsed parseBody(String charSet, String responseBody) throws PageNotEditableException, UnsupportedEncodingException {
210 Matcher matcher = BODY_PATTERN.matcher(responseBody);
211 if (!matcher.matches())
212 throw new PageNotEditableException("cannot find editform form");
214 String title = matcher.group(1);
215 String body = matcher.group(2);
216 String timestamp = matcher.group(3);
218 title = URLDecoder.decode(title, charSet);
219 body = body.replaceAll(""", "\"").replaceAll("'", "'").replaceAll("<", "<").replaceAll(">", ">").replaceAll(
220 "&", "&").replaceAll("\r\n", "\n").replace('\r', '\n');
222 return new Parsed(timestamp, title, body);
225 /** load a Page Version - returns a Loaded Object */
226 public Loaded load(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
227 PageNotEditableException {
228 GetMethod method = new GetMethod(actionURL);
229 method.setFollowRedirects(false);
230 method.addRequestHeader("User-Agent", userAgent);
231 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
232 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
236 int responseCode = client.executeMethod(method);
237 String responseBody = method.getResponseBodyAsString();
240 if (responseCode == 200) {
241 Parsed parsed = parseBody(charSet, responseBody);
242 Content content = new Content(parsed.timestamp, parsed.body);
243 result = new Loaded(actionURL, charSet, parsed.title, content);
245 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
247 } catch (HttpException e) {
248 throw new MethodException("method failed", e);
249 } catch (IOException e) {
250 throw new MethodException("method failed", e);
252 method.releaseConnection();
257 public ArrayList loadXML(IWikipedia config, String actionURL, String pages) throws UnexpectedAnswerException, MethodException,
258 InterruptedException {
259 storeThrottle.delay();
260 PostMethod method = new PostMethod(actionURL);
261 method.setFollowRedirects(false);
262 method.addRequestHeader("User-Agent", userAgent);
263 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
265 NameValuePair[] params = new NameValuePair[] {
266 new NameValuePair("pages", pages),
267 new NameValuePair("curonly", "X"),
268 new NameValuePair("action", "submit") };
269 method.addParameters(params);
271 int responseCode = client.executeMethod(method);
272 String responseBody = method.getResponseBodyAsString();
274 if (responseCode == 200) {
275 StringReader reader = new StringReader(responseBody);
276 return XMLReader.readFromStream(reader);
278 throw new UnexpectedAnswerException("XML load not successful: expected 200 OK, got " + method.getStatusLine());
280 } catch (CoreException e) {
281 throw new UnexpectedAnswerException("XML load method failed" + e.getMessage());
282 } catch (HttpException e) {
283 throw new MethodException("XML load method failed", e);
284 } catch (IOException e) {
285 throw new MethodException("XML load method failed", e);
287 method.releaseConnection();
292 * store a Page Version - returns a Stored object
295 * WiKipedia predefined properties
303 * @throws UnexpectedAnswerException
304 * @throws MethodException
305 * @throws PageNotEditableException
306 * @throws InterruptedException
308 public Stored store(IWikipedia config, String actionUrl, String title, Content content, String summary, boolean minorEdit,
309 boolean watchThis) throws UnexpectedAnswerException, MethodException, PageNotEditableException, InterruptedException {
310 //### workaround: prevent too many stores at a time
311 storeThrottle.delay();
313 PostMethod method = new PostMethod(actionUrl);
315 method.setFollowRedirects(false);
316 method.addRequestHeader("User-Agent", userAgent);
317 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
318 NameValuePair[] params = new NameValuePair[] {
319 // new NameValuePair("wpSection", ""),
320 // new NameValuePair("wpPreview", "Vorschau zeigen"),
321 // new NameValuePair("wpSave", "Artikel speichern"),
322 new NameValuePair("title", title),
323 new NameValuePair("wpTextbox1", content.body),
324 new NameValuePair("wpEdittime", content.timestamp),
325 new NameValuePair("wpSummary", summary),
326 new NameValuePair("wpSave", "yes"),
327 new NameValuePair("action", "submit") };
328 method.addParameters(params);
330 method.addParameter("wpMinoredit", "1");
332 method.addParameter("wpWatchthis", "1");
336 int responseCode = client.executeMethod(method);
337 String responseBody = method.getResponseBodyAsString();
340 // since 11dec04 there is a single linefeed instead of an empty page.. trim() helps.
341 if (responseCode == 302 && responseBody.trim().length() == 0) {
342 // log("store successful, reloading");
343 Loaded loaded = load(actionUrl, config.getCharSet(), title);
344 result = new Stored(actionUrl, config.getCharSet(), loaded.title, loaded.content, false);
345 } else if (responseCode == 200) {
346 // log("store not successful, conflict detected");
347 Parsed parsed = parseBody(config.getCharSet(), responseBody);
348 Content cont = new Content(parsed.timestamp, parsed.body);
349 result = new Stored(actionUrl, config.getCharSet(), parsed.title, cont, true);
351 throw new UnexpectedAnswerException("store not successful: expected 200 OK, got " + method.getStatusLine());
353 } catch (HttpException e) {
354 throw new MethodException("method failed", e);
355 } catch (IOException e) {
356 throw new MethodException("method failed", e);
358 method.releaseConnection();
364 * Get the text of a wikimedia article
367 public String getWikiRawText(String wikiname, String urlStr) {
369 // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=raw
370 // http://en.wikibooks.org/w/index.php?title=Programming:PHP:SQL_Injection&action=raw
371 // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=raw
372 HttpMethod method = null;
374 if (urlStr == null) {
375 WikiEditorPlugin.getDefault().reportError("No Wikipedia URL configured", "URL-String == null");
376 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=raw";
378 URI uri = new URI(urlStr.toCharArray());
380 String schema = uri.getScheme();
381 if ((schema == null) || (schema.equals(""))) {
384 Protocol protocol = Protocol.getProtocol(schema);
386 method = new GetMethod(uri.toString());
387 String host = uri.getHost();
388 int port = uri.getPort();
390 HttpConnection connection = new HttpConnection(host, port, protocol);
391 HttpState state = setHTTPParameters(connection);
393 if (connection.isProxied() && connection.isSecure()) {
394 method = new ConnectMethod(method);
397 method.execute(state, connection);
398 // client.executeMethod(method);
400 if (method.getStatusCode() == HttpStatus.SC_OK) {
401 // get the wiki text now:
402 String wikiText = method.getResponseBodyAsString();
405 } catch (Throwable e) {
406 WikiEditorPlugin.log(e);
407 WikiEditorPlugin.getDefault().reportError("Exception occured", e.getMessage() + "\nSee stacktrace in /.metadata/.log file.");
409 if (method != null) {
410 method.releaseConnection();
413 return null; // no success in getting wiki text
416 // public static String getWikiEditTextarea(String wikiname, String urlStr) {
418 // // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=edit
419 // // http://en.wikibooks.org/w/wiki.phtml?title=Programming:PHP:SQL_Injection&action=edit
420 // // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=edit
421 // HttpMethod method = null;
423 // if (urlStr == null) {
424 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=edit";
427 // // urlStr = urlStr + "?title=" + wikiname + "&action=edit";
429 // URI uri = new URI(urlStr.toCharArray());
431 // String schema = uri.getScheme();
432 // if ((schema == null) || (schema.equals(""))) {
435 // Protocol protocol = Protocol.getProtocol(schema);
437 // HttpState state = new HttpState();
439 // method = new GetMethod(uri.toString());
440 // String host = uri.getHost();
441 // int port = uri.getPort();
443 // HttpConnection connection = new HttpConnection(host, port, protocol);
445 // connection.setProxyHost(System.getProperty("http.proxyHost"));
446 // connection.setProxyPort(Integer.parseInt(System.getProperty("http.proxyPort", "80")));
448 // if (System.getProperty("http.proxyUserName") != null) {
449 // state.setProxyCredentials(null, null, new UsernamePasswordCredentials(System.getProperty("http.proxyUserName"), System
450 // .getProperty("http.proxyPassword")));
453 // if (connection.isProxied() && connection.isSecure()) {
454 // method = new ConnectMethod(method);
457 // method.execute(state, connection);
459 // if (method.getStatusCode() == HttpStatus.SC_OK) {
460 // // get the textareas wiki text now:
461 // InputStream stream = method.getResponseBodyAsStream();
462 // int byteLen = stream.available();
464 // byte[] buffer = new byte[byteLen];
465 // stream.read(buffer, 0, byteLen);
466 // String wikiText = new String(buffer);
467 // // String wikiText = method.getResponseBodyAsString();
468 // int start = wikiText.indexOf("<textarea");
469 // if (start != (-1)) {
470 // start = wikiText.indexOf(">", start + 1);
471 // if (start != (-1)) {
472 // int end = wikiText.indexOf("</textarea>");
473 // wikiText = wikiText.substring(start + 1, end);
477 // // System.out.println(wikiText);
480 // } catch (Exception e) {
481 // e.printStackTrace();
483 // if (method != null) {
484 // method.releaseConnection();
487 // return null; // no success in getting wiki text
494 private HttpState setHTTPParameters(HttpConnection connection) {
495 HttpState state = new HttpState();
496 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
497 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
498 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
501 // timeout after xx seconds
502 connection.setConnectionTimeout(Integer.parseInt(timeout));
504 if (proxyHost.length() > 0) {
505 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
506 connection.setProxyHost(proxyHost);
507 connection.setProxyPort(Integer.parseInt(proxyPort));
509 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
510 if (proxyUserName.length() > 0) {
511 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
512 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
516 } catch (Exception e) {
522 private void setHTTPClientParameters(HttpClient client) {
524 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
525 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
526 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
529 // timeout after xx seconds
530 client.setConnectionTimeout(Integer.parseInt(timeout));
532 if (proxyHost.length() > 0) {
533 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
534 HostConfiguration conf = new HostConfiguration();
535 client.setHostConfiguration(conf);
536 conf.setProxy(proxyHost, Integer.parseInt(proxyPort));
538 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
539 if (proxyUserName.length() > 0) {
540 HttpState state = new HttpState();
541 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
542 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
543 client.setState(state);
547 } catch (Exception e) {
552 public static void main(String[] args) {
553 MediaWikiConnector mwc = new MediaWikiConnector();
555 IWikipedia wp = null;
556 ArrayList list = mwc.loadXML(wp, "http://www.plog4u.de/wiki/index.php/Spezial:Export", "Mechanisches Fernsehen\nSynästhesie");
557 for (int i = 0; i < list.size(); i++) {
558 System.out.println(list.get(i).toString());
560 } catch (UnexpectedAnswerException e) {
561 // TODO Auto-generated catch block
563 } catch (Exception e) {
564 // TODO Auto-generated catch block