1 package net.sourceforge.phpeclipse.wiki.actions.mediawiki.connect;
3 //Parts of this sources are copied and modified from the jEdit Wikipedia plugin:
4 //http://www.djini.de/software/wikipedia/index.html
6 //The modified sources are available under the "Common Public License"
7 //with permission from the original author: Daniel Wunsch
9 import java.io.IOException;
10 import java.io.StringReader;
11 import java.io.UnsupportedEncodingException;
12 import java.net.URLDecoder;
13 import java.util.ArrayList;
14 import java.util.regex.Matcher;
15 import java.util.regex.Pattern;
17 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.config.IWikipedia;
18 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.MethodException;
19 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.PageNotEditableException;
20 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.UnexpectedAnswerException;
21 import net.sourceforge.phpeclipse.wiki.editor.WikiEditorPlugin;
23 import org.apache.commons.httpclient.ConnectMethod;
24 import org.apache.commons.httpclient.HostConfiguration;
25 import org.apache.commons.httpclient.HttpClient;
26 import org.apache.commons.httpclient.HttpConnection;
27 import org.apache.commons.httpclient.HttpException;
28 import org.apache.commons.httpclient.HttpMethod;
29 import org.apache.commons.httpclient.HttpState;
30 import org.apache.commons.httpclient.HttpStatus;
31 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
32 import org.apache.commons.httpclient.NameValuePair;
33 import org.apache.commons.httpclient.URI;
34 import org.apache.commons.httpclient.UsernamePasswordCredentials;
35 import org.apache.commons.httpclient.methods.GetMethod;
36 import org.apache.commons.httpclient.methods.PostMethod;
37 import org.apache.commons.httpclient.protocol.Protocol;
38 import org.apache.commons.httpclient.util.EncodingUtil;
39 import org.eclipse.core.runtime.CoreException;
40 import org.eclipse.core.runtime.Preferences;
43 * This class gets the wikitext from a wikipedia edit page
45 * The basic coding was copied from the commons-httpclient example <code>MediaWikiConnector.java</code>
47 public class MediaWikiConnector {
48 //pattern used to scarp an edit page
49 private static final Pattern BODY_PATTERN = Pattern.compile(
51 * action=".*?title=(.*?)(&|\") <form id="editform" name="editform" method="post"
52 * action="/w/wiki.phtml?title=Ammersee&action=submit" locked pages: <textarea cols='80' rows='25' readonly>
54 ".*<form[^>]*\\sid=\"editform\"[^>]*title=(.*?)&[^>]*>" + ".*<textarea[^>]*\\sname=\"wpTextbox1\"[^>]*>(.*?)</textarea>"
55 + ".*<input[^>]*\\svalue=\"(\\d*)\"[^>]*\\sname=\"wpEdittime\"[^>]*>" + ".*", Pattern.DOTALL);
57 // <input type='hidden' value="53ee6d8b42ff9b7d" name="wpEditToken" />
58 private static final Pattern EDIT_TOKEN = Pattern.compile(".*<input\\stype='hidden'\\svalue=\"(.*?)\"\\sname=\"wpEditToken\"\\s/>.*",
61 //setup default user agent
62 final static public String userAgent = "plog4u.org/0.0";
64 // create a ConnectionManager
65 private MultiThreadedHttpConnectionManager manager;
67 private HttpClient client;
70 * Delay a new store to 1 second
72 private Throttle storeThrottle = new Throttle(1000);
75 private long nextTime = 0;
77 private final long minimumDelay;
79 public Throttle(long minimumDelay) {
80 this.minimumDelay = minimumDelay;
83 /** this is called from the client */
84 public synchronized void delay() throws InterruptedException {
85 long delay = nextTime - System.currentTimeMillis();
88 nextTime = System.currentTimeMillis() + minimumDelay;
92 public MediaWikiConnector() {
93 // <a href="javascript:window.location.href='http://127.0.0.1:8009/open/?' + window.location.href">bookmarklet</a>
94 manager = new MultiThreadedHttpConnectionManager();
95 manager.setMaxConnectionsPerHost(6);
96 manager.setMaxTotalConnections(18);
97 manager.setConnectionStaleCheckingEnabled(true);
98 // open the conversation
99 client = new HttpClient(manager);
100 setHTTPClientParameters(client);
101 //client.State.CookiePolicy = CookiePolicy.COMPATIBILITY;
102 //client.HostConfiguration.setHost(LOGON_SITE, LOGON_PORT, "http");
105 /** destructor freeing all resources. the Connection is not usable any more after calling this method */
106 public void destroy() {
110 /** log in - returns success */
111 public boolean login(IWikipedia config, String actionUrl, String user, String password, boolean remember)
112 throws UnexpectedAnswerException, MethodException {
113 PostMethod method = new PostMethod(actionUrl);
114 method.setFollowRedirects(false);
115 method.addRequestHeader("User-Agent", userAgent);
116 NameValuePair[] params = new NameValuePair[] {
117 new NameValuePair("title", config.getLoginTitle()),
118 new NameValuePair("action", "submit"),
119 new NameValuePair("wpName", user),
120 new NameValuePair("wpPassword", password),
121 new NameValuePair("wpRemember", remember ? "1" : "0"),
122 new NameValuePair("wpLoginattempt", "submit") };
123 method.addParameters(params);
127 int responseCode = client.executeMethod(method);
128 String responseBody = method.getResponseBodyAsString();
134 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
135 && responseBody.matches(config.getLoginSuccess())) {
137 } else if (responseCode == 200 && responseBody.matches(config.getLoginWrongPw()) || responseCode == 200
138 && responseBody.matches(config.getLoginNoUser())) {
140 if (responseBody.matches(config.getLoginNoUser())) {
141 throw new UnexpectedAnswerException("login not successful: wrong user name: " + user);
142 } else if (responseBody.matches(config.getLoginWrongPw())) {
143 throw new UnexpectedAnswerException("login not successful: wrong password for user: " + user);
145 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
148 throw new UnexpectedAnswerException("login not successful: " + method.getStatusLine());
150 } catch (HttpException e) {
151 throw new MethodException("method failed", e);
152 } catch (IOException e) {
153 throw new MethodException("method failed", e);
155 method.releaseConnection();
158 * // display cookies System.err.println("login: " + result); for (var cookie : client.State.Cookies) {
159 * System.err.println("cookie: " + cookie); }
163 SiteState state = SiteState.siteState(config);
164 state.loggedIn = result;
165 state.userName = user;
170 /** log out - return success */
171 public boolean logout(IWikipedia config, String actionUrl) throws UnexpectedAnswerException, MethodException {
172 GetMethod method = new GetMethod(actionUrl);
173 method.setFollowRedirects(false);
174 method.addRequestHeader("User-Agent", userAgent);
175 NameValuePair[] params = new NameValuePair[] {
176 new NameValuePair("title", config.getLogoutTitle()),
177 new NameValuePair("action", "submit") };
178 method.setQueryString(EncodingUtil.formUrlEncode(params, config.getCharSet()));
182 int responseCode = client.executeMethod(method);
183 String responseBody = method.getResponseBodyAsString();
186 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
187 && responseBody.matches(config.getLogoutSuccess())) {
188 // config.getloggedIn = false;
190 } else if (responseCode == 200) {
191 //### should check for a failure message
193 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
195 throw new UnexpectedAnswerException("logout not successful: " + method.getStatusLine());
197 } catch (HttpException e) {
198 throw new MethodException("method failed", e);
199 } catch (IOException e) {
200 throw new MethodException("method failed", e);
202 method.releaseConnection();
206 SiteState state = SiteState.siteState(config);
207 state.loggedIn = false;
212 /** parses a returned editform for the sessions wpEditToken */
213 private String parseEditToken(String charSet, String responseBody) throws PageNotEditableException {
214 Matcher matcher = EDIT_TOKEN.matcher(responseBody);
215 if (!matcher.matches()) {
219 return matcher.group(1);
223 * returns the edit token or <code>null</code> if no token is available
229 * @throws UnexpectedAnswerException
230 * @throws MethodException
231 * @throws PageNotEditableException
233 public String loadEditToken(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
234 PageNotEditableException {
235 GetMethod method = new GetMethod(actionURL);
236 method.setFollowRedirects(false);
237 method.addRequestHeader("User-Agent", userAgent);
238 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
239 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
242 int responseCode = client.executeMethod(method);
243 String responseBody = method.getResponseBodyAsString();
246 if (responseCode == 200) {
247 String parsed = parseEditToken(charSet, responseBody);
248 if (parsed != null && parsed.length() == 0) {
253 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
255 } catch (HttpException e) {
256 throw new MethodException("method failed", e);
257 } catch (IOException e) {
258 throw new MethodException("method failed", e);
260 method.releaseConnection();
264 /** parses a returned editform into a Content object with UNIX-EOLs ("\n") */
265 private Parsed parseBody(String charSet, String responseBody) throws PageNotEditableException, UnsupportedEncodingException {
266 Matcher matcher = BODY_PATTERN.matcher(responseBody);
267 if (!matcher.matches())
268 throw new PageNotEditableException("cannot find editform form");
270 String title = matcher.group(1);
271 String body = matcher.group(2);
272 String timestamp = matcher.group(3);
273 String tokenEdit = null;
274 // String tokenEdit = matcher.group(4);
275 title = URLDecoder.decode(title, charSet);
276 body = body.replaceAll(""", "\"").replaceAll("'", "'").replaceAll("<", "<").replaceAll(">", ">").replaceAll(
277 "&", "&").replaceAll("\r\n", "\n").replace('\r', '\n');
279 return new Parsed(timestamp, title, body, tokenEdit);
282 /** load a Page Version - returns a Loaded Object */
283 public Loaded load(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
284 PageNotEditableException {
285 GetMethod method = new GetMethod(actionURL);
286 method.setFollowRedirects(false);
287 method.addRequestHeader("User-Agent", userAgent);
288 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
289 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
293 int responseCode = client.executeMethod(method);
294 String responseBody = method.getResponseBodyAsString();
297 if (responseCode == 200) {
298 Parsed parsed = parseBody(charSet, responseBody);
299 Content content = new Content(parsed.timestamp, parsed.body);
300 result = new Loaded(actionURL, charSet, parsed.title, content);
302 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
304 } catch (HttpException e) {
305 throw new MethodException("method failed", e);
306 } catch (IOException e) {
307 throw new MethodException("method failed", e);
309 method.releaseConnection();
314 public ArrayList loadXML(IWikipedia config, String actionURL, String pages) throws UnexpectedAnswerException, MethodException,
315 InterruptedException {
316 storeThrottle.delay();
317 PostMethod method = new PostMethod(actionURL);
318 method.setFollowRedirects(false);
319 method.addRequestHeader("User-Agent", userAgent);
320 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
322 NameValuePair[] params = new NameValuePair[] {
323 new NameValuePair("pages", pages),
324 new NameValuePair("curonly", "X"),
325 new NameValuePair("action", "submit") };
326 method.addParameters(params);
328 int responseCode = client.executeMethod(method);
329 String responseBody = method.getResponseBodyAsString();
331 if (responseCode == 200) {
332 StringReader reader = new StringReader(responseBody);
333 return XMLReader.readFromStream(reader);
335 throw new UnexpectedAnswerException("XML load not successful: expected 200 OK, got " + method.getStatusLine());
337 } catch (CoreException e) {
338 throw new UnexpectedAnswerException("XML load method failed" + e.getMessage());
339 } catch (HttpException e) {
340 throw new MethodException("XML load method failed", e);
341 } catch (IOException e) {
342 throw new MethodException("XML load method failed", e);
344 method.releaseConnection();
349 * store a Page Version - returns a Stored object
352 * WiKipedia predefined properties
360 * @throws UnexpectedAnswerException
361 * @throws MethodException
362 * @throws PageNotEditableException
363 * @throws InterruptedException
365 public Stored store(IWikipedia config, String editToken, String actionUrl, String title, Content content, String summary,
366 boolean minorEdit, boolean watchThis) throws UnexpectedAnswerException, MethodException, PageNotEditableException,
367 InterruptedException {
368 //### workaround: prevent too many stores at a time
369 storeThrottle.delay();
371 PostMethod method = new PostMethod(actionUrl);
373 method.setFollowRedirects(false);
374 method.addRequestHeader("User-Agent", userAgent);
375 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
376 if (editToken == null) {
377 // in some versions editToken isn't supported
380 NameValuePair[] params = new NameValuePair[] {
381 // new NameValuePair("wpSection", ""),
382 // new NameValuePair("wpPreview", "Vorschau zeigen"),
383 // new NameValuePair("wpSave", "Artikel speichern"),
384 new NameValuePair("title", title),
385 new NameValuePair("wpTextbox1", content.body),
386 new NameValuePair("wpEdittime", content.timestamp),
387 new NameValuePair("wpSummary", summary),
388 new NameValuePair("wpEditToken", editToken),
389 new NameValuePair("wpSave", "yes"),
390 new NameValuePair("action", "submit") };
391 method.addParameters(params);
393 method.addParameter("wpMinoredit", "1");
395 method.addParameter("wpWatchthis", "1");
399 int responseCode = client.executeMethod(method);
400 String responseBody = method.getResponseBodyAsString();
403 // since 11dec04 there is a single linefeed instead of an empty page.. trim() helps.
404 if (responseCode == 302 && responseBody.trim().length() == 0) {
405 // log("store successful, reloading");
406 Loaded loaded = load(actionUrl, config.getCharSet(), title);
407 result = new Stored(actionUrl, config.getCharSet(), loaded.title, loaded.content, false);
408 } else if (responseCode == 200) {
409 // log("store not successful, conflict detected");
410 Parsed parsed = parseBody(config.getCharSet(), responseBody);
411 Content cont = new Content(parsed.timestamp, parsed.body);
412 result = new Stored(actionUrl, config.getCharSet(), parsed.title, cont, true);
414 throw new UnexpectedAnswerException("store not successful: expected 200 OK, got " + method.getStatusLine());
416 } catch (HttpException e) {
417 throw new MethodException("method failed", e);
418 } catch (IOException e) {
419 throw new MethodException("method failed", e);
421 method.releaseConnection();
427 * Get the text of a wikimedia article
430 public String getWikiRawText(String wikiname, String urlStr) {
432 // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=raw
433 // http://en.wikibooks.org/w/index.php?title=Programming:PHP:SQL_Injection&action=raw
434 // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=raw
435 HttpMethod method = null;
437 if (urlStr == null) {
438 WikiEditorPlugin.getDefault().reportError("No Wikipedia URL configured", "URL-String == null");
439 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=raw";
441 URI uri = new URI(urlStr.toCharArray());
443 String schema = uri.getScheme();
444 if ((schema == null) || (schema.equals(""))) {
447 Protocol protocol = Protocol.getProtocol(schema);
449 method = new GetMethod(uri.toString());
450 String host = uri.getHost();
451 int port = uri.getPort();
453 HttpConnection connection = new HttpConnection(host, port, protocol);
454 HttpState state = setHTTPParameters(connection);
456 if (connection.isProxied() && connection.isSecure()) {
457 method = new ConnectMethod(method);
460 method.execute(state, connection);
461 // client.executeMethod(method);
463 if (method.getStatusCode() == HttpStatus.SC_OK) {
464 // get the wiki text now:
465 String wikiText = method.getResponseBodyAsString();
468 } catch (Throwable e) {
469 WikiEditorPlugin.log(e);
470 WikiEditorPlugin.getDefault().reportError("Exception occured", e.getMessage() + "\nSee stacktrace in /.metadata/.log file.");
472 if (method != null) {
473 method.releaseConnection();
476 return null; // no success in getting wiki text
479 // public static String getWikiEditTextarea(String wikiname, String urlStr) {
481 // // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=edit
482 // // http://en.wikibooks.org/w/wiki.phtml?title=Programming:PHP:SQL_Injection&action=edit
483 // // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=edit
484 // HttpMethod method = null;
486 // if (urlStr == null) {
487 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=edit";
490 // // urlStr = urlStr + "?title=" + wikiname + "&action=edit";
492 // URI uri = new URI(urlStr.toCharArray());
494 // String schema = uri.getScheme();
495 // if ((schema == null) || (schema.equals(""))) {
498 // Protocol protocol = Protocol.getProtocol(schema);
500 // HttpState state = new HttpState();
502 // method = new GetMethod(uri.toString());
503 // String host = uri.getHost();
504 // int port = uri.getPort();
506 // HttpConnection connection = new HttpConnection(host, port, protocol);
508 // connection.setProxyHost(System.getProperty("http.proxyHost"));
509 // connection.setProxyPort(Integer.parseInt(System.getProperty("http.proxyPort", "80")));
511 // if (System.getProperty("http.proxyUserName") != null) {
512 // state.setProxyCredentials(null, null, new UsernamePasswordCredentials(System.getProperty("http.proxyUserName"), System
513 // .getProperty("http.proxyPassword")));
516 // if (connection.isProxied() && connection.isSecure()) {
517 // method = new ConnectMethod(method);
520 // method.execute(state, connection);
522 // if (method.getStatusCode() == HttpStatus.SC_OK) {
523 // // get the textareas wiki text now:
524 // InputStream stream = method.getResponseBodyAsStream();
525 // int byteLen = stream.available();
527 // byte[] buffer = new byte[byteLen];
528 // stream.read(buffer, 0, byteLen);
529 // String wikiText = new String(buffer);
530 // // String wikiText = method.getResponseBodyAsString();
531 // int start = wikiText.indexOf("<textarea");
532 // if (start != (-1)) {
533 // start = wikiText.indexOf(">", start + 1);
534 // if (start != (-1)) {
535 // int end = wikiText.indexOf("</textarea>");
536 // wikiText = wikiText.substring(start + 1, end);
540 // // System.out.println(wikiText);
543 // } catch (Exception e) {
544 // e.printStackTrace();
546 // if (method != null) {
547 // method.releaseConnection();
550 // return null; // no success in getting wiki text
557 private HttpState setHTTPParameters(HttpConnection connection) {
558 HttpState state = new HttpState();
559 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
560 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
561 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
564 // timeout after xx seconds
565 connection.setConnectionTimeout(Integer.parseInt(timeout));
567 if (proxyHost.length() > 0) {
568 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
569 connection.setProxyHost(proxyHost);
570 connection.setProxyPort(Integer.parseInt(proxyPort));
572 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
573 if (proxyUserName.length() > 0) {
574 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
575 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
579 } catch (Exception e) {
585 private void setHTTPClientParameters(HttpClient client) {
587 Preferences prefs = WikiEditorPlugin.getDefault().getPluginPreferences();
588 String timeout = prefs.getString(WikiEditorPlugin.HTTP_TIMEOUT);
589 String proxyHost = prefs.getString(WikiEditorPlugin.HTTP_PROXYHOST);
592 // timeout after xx seconds
593 client.setConnectionTimeout(Integer.parseInt(timeout));
595 if (proxyHost.length() > 0) {
596 String proxyPort = prefs.getString(WikiEditorPlugin.HTTP_PROXYPORT);
597 HostConfiguration conf = new HostConfiguration();
598 client.setHostConfiguration(conf);
599 conf.setProxy(proxyHost, Integer.parseInt(proxyPort));
601 String proxyUserName = prefs.getString(WikiEditorPlugin.HTTP_PROXYUSERNAME);
602 if (proxyUserName.length() > 0) {
603 HttpState state = new HttpState();
604 String proxyPassWord = prefs.getString(WikiEditorPlugin.HTTP_PROXYPASSWORD);
605 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(proxyUserName, proxyPassWord));
606 client.setState(state);
610 } catch (Exception e) {
616 public static void main(String[] args) {
617 MediaWikiConnector mwc = new MediaWikiConnector();
619 IWikipedia wp = null;
620 ArrayList list = mwc.loadXML(wp, "http://www.plog4u.de/wiki/index.php/Spezial:Export", "Mechanisches Fernsehen\nSynästhesie");
621 for (int i = 0; i < list.size(); i++) {
622 System.out.println(list.get(i).toString());
624 } catch (UnexpectedAnswerException e) {
625 // TODO Auto-generated catch block
627 } catch (Exception e) {
628 // TODO Auto-generated catch block