1 package net.sourceforge.phpeclipse.wiki.actions.mediawiki.connect;
3 //Parts of this sources are copied and modified from the jEdit Wikipedia plugin:
4 //http://www.djini.de/software/wikipedia/index.html
6 //The modified sources are available under the "Common Public License"
7 //with permission from the original author: Daniel Wunsch
9 import java.io.IOException;
10 import java.io.StringReader;
11 import java.io.UnsupportedEncodingException;
12 import java.net.URLDecoder;
13 import java.util.ArrayList;
14 import java.util.regex.Matcher;
15 import java.util.regex.Pattern;
17 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.config.IWikipedia;
18 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.MethodException;
19 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.PageNotEditableException;
20 import net.sourceforge.phpeclipse.wiki.actions.mediawiki.exceptions.UnexpectedAnswerException;
21 import net.sourceforge.phpeclipse.wiki.editor.WikiEditorPlugin;
23 import org.apache.commons.httpclient.ConnectMethod;
24 import org.apache.commons.httpclient.HttpClient;
25 import org.apache.commons.httpclient.HttpConnection;
26 import org.apache.commons.httpclient.HttpException;
27 import org.apache.commons.httpclient.HttpMethod;
28 import org.apache.commons.httpclient.HttpState;
29 import org.apache.commons.httpclient.HttpStatus;
30 import org.apache.commons.httpclient.MultiThreadedHttpConnectionManager;
31 import org.apache.commons.httpclient.NameValuePair;
32 import org.apache.commons.httpclient.URI;
33 import org.apache.commons.httpclient.UsernamePasswordCredentials;
34 import org.apache.commons.httpclient.methods.GetMethod;
35 import org.apache.commons.httpclient.methods.PostMethod;
36 import org.apache.commons.httpclient.protocol.Protocol;
37 import org.apache.commons.httpclient.util.EncodingUtil;
38 import org.eclipse.core.runtime.CoreException;
41 * This class gets the wikitext from a wikipedia edit page
43 * The basic coding was copied from the commons-httpclient example <code>MediaWikiConnector.java</code>
45 public class MediaWikiConnector {
46 //pattern used to scarp an edit page
47 private static final Pattern BODY_PATTERN = Pattern.compile(
49 * action=".*?title=(.*?)(&|\") <form id="editform" name="editform" method="post"
50 * action="/w/wiki.phtml?title=Ammersee&action=submit" locked pages: <textarea cols='80' rows='25' readonly>
52 ".*<form[^>]*\\sid=\"editform\"[^>]*title=(.*?)&[^>]*>" + ".*<textarea[^>]*\\sname=\"wpTextbox1\"[^>]*>(.*?)</textarea>"
53 + ".*<input[^>]*\\svalue=\"(\\d*)\"[^>]*\\sname=\"wpEdittime\"[^>]*>" + ".*", Pattern.DOTALL);
55 //setup default user agent
56 final static public String userAgent = "plog4u.org/0.0";
58 // create a ConnectionManager
59 private MultiThreadedHttpConnectionManager manager;
61 private HttpClient client;
64 * Delay a new store to 1 second
66 private Throttle storeThrottle = new Throttle(1000);
69 private long nextTime = 0;
71 private final long minimumDelay;
73 public Throttle(long minimumDelay) {
74 this.minimumDelay = minimumDelay;
77 /** this is called from the client */
78 public synchronized void delay() throws InterruptedException {
79 long delay = nextTime - System.currentTimeMillis();
82 nextTime = System.currentTimeMillis() + minimumDelay;
86 public MediaWikiConnector() {
87 // <a href="javascript:window.location.href='http://127.0.0.1:8009/open/?' + window.location.href">bookmarklet</a>
88 manager = new MultiThreadedHttpConnectionManager();
89 manager.setMaxConnectionsPerHost(6);
90 manager.setMaxTotalConnections(18);
91 manager.setConnectionStaleCheckingEnabled(true);
92 // open the conversation
93 client = new HttpClient(manager);
94 //client.State.CookiePolicy = CookiePolicy.COMPATIBILITY;
95 //client.HostConfiguration.setHost(LOGON_SITE, LOGON_PORT, "http");
98 /** destructor freeing all resources. the Connection is not usable any more after calling this method */
99 public void destroy() {
103 /** log in - returns success */
104 public boolean login(IWikipedia config, String actionUrl, String user, String password, boolean remember)
105 throws UnexpectedAnswerException, MethodException {
106 PostMethod method = new PostMethod(actionUrl);
107 method.setFollowRedirects(false);
108 method.addRequestHeader("User-Agent", userAgent);
109 NameValuePair[] params = new NameValuePair[] {
110 new NameValuePair("title", config.getLoginTitle()),
111 new NameValuePair("action", "submit"),
112 new NameValuePair("wpName", user),
113 new NameValuePair("wpPassword", password),
114 new NameValuePair("wpRemember", remember ? "1" : "0"),
115 new NameValuePair("wpLoginattempt", "submit") };
116 method.addParameters(params);
120 int responseCode = client.executeMethod(method);
121 String responseBody = method.getResponseBodyAsString();
127 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
128 && responseBody.matches(config.getLoginSuccess())) {
130 } else if (responseCode == 200 && responseBody.matches(config.getLoginWrongPw()) || responseCode == 200
131 && responseBody.matches(config.getLoginNoUser())) {
133 if (responseBody.matches(config.getLoginNoUser())) {
134 throw new UnexpectedAnswerException("login not successful: wrong user name: "+user);
135 } else if (responseBody.matches(config.getLoginWrongPw())) {
136 throw new UnexpectedAnswerException("login not successful: wrong password for user: "+user);
138 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
141 throw new UnexpectedAnswerException("login not successful: " + method.getStatusLine());
143 } catch (HttpException e) {
144 throw new MethodException("method failed", e);
145 } catch (IOException e) {
146 throw new MethodException("method failed", e);
148 method.releaseConnection();
151 * // display cookies System.err.println("login: " + result); for (var cookie : client.State.Cookies) {
152 * System.err.println("cookie: " + cookie); }
156 SiteState state = SiteState.siteState(config);
157 state.loggedIn = result;
158 state.userName = user;
163 /** log out - return success */
164 public boolean logout(IWikipedia config, String actionUrl) throws UnexpectedAnswerException, MethodException {
165 GetMethod method = new GetMethod(actionUrl);
166 method.setFollowRedirects(false);
167 method.addRequestHeader("User-Agent", userAgent);
168 NameValuePair[] params = new NameValuePair[] {
169 new NameValuePair("title", config.getLogoutTitle()),
170 new NameValuePair("action", "submit") };
171 method.setQueryString(EncodingUtil.formUrlEncode(params, config.getCharSet()));
175 int responseCode = client.executeMethod(method);
176 String responseBody = method.getResponseBodyAsString();
179 if (responseCode == 302 && responseBody.length() == 0 || responseCode == 200
180 && responseBody.matches(config.getLoginSuccess())) {
181 // config.getloggedIn = false;
183 } else if (responseCode == 200) {
184 //### should check for a failure message
186 throw new UnexpectedAnswerException("logout not successful: responseCode == 200");
188 throw new UnexpectedAnswerException("logout not successful: " + method.getStatusLine());
190 } catch (HttpException e) {
191 throw new MethodException("method failed", e);
192 } catch (IOException e) {
193 throw new MethodException("method failed", e);
195 method.releaseConnection();
199 SiteState state = SiteState.siteState(config);
200 state.loggedIn = false;
205 /** parses a returned editform into a Content object with UNIX-EOLs ("\n") */
206 private Parsed parseBody(String charSet, String responseBody) throws PageNotEditableException, UnsupportedEncodingException {
207 Matcher matcher = BODY_PATTERN.matcher(responseBody);
208 if (!matcher.matches())
209 throw new PageNotEditableException("cannot find editform form");
211 String title = matcher.group(1);
212 String body = matcher.group(2);
213 String timestamp = matcher.group(3);
215 title = URLDecoder.decode(title, charSet);
216 body = body.replaceAll(""", "\"").replaceAll("'", "'").replaceAll("<", "<").replaceAll(">", ">").replaceAll(
217 "&", "&").replaceAll("\r\n", "\n").replace('\r', '\n');
219 return new Parsed(timestamp, title, body);
222 /** load a Page Version - returns a Loaded Object */
223 public Loaded load(String actionURL, String charSet, String title) throws UnexpectedAnswerException, MethodException,
224 PageNotEditableException {
225 GetMethod method = new GetMethod(actionURL);
226 method.setFollowRedirects(false);
227 method.addRequestHeader("User-Agent", userAgent);
228 NameValuePair[] params = new NameValuePair[] { new NameValuePair("title", title), new NameValuePair("action", "edit") };
229 method.setQueryString(EncodingUtil.formUrlEncode(params, charSet));
233 int responseCode = client.executeMethod(method);
234 String responseBody = method.getResponseBodyAsString();
237 if (responseCode == 200) {
238 Parsed parsed = parseBody(charSet, responseBody);
239 Content content = new Content(parsed.timestamp, parsed.body);
240 result = new Loaded(actionURL, charSet, parsed.title, content);
242 throw new UnexpectedAnswerException("load not successful: expected 200 OK, got " + method.getStatusLine());
244 } catch (HttpException e) {
245 throw new MethodException("method failed", e);
246 } catch (IOException e) {
247 throw new MethodException("method failed", e);
249 method.releaseConnection();
254 public ArrayList loadXML(IWikipedia config, String actionURL, String pages) throws UnexpectedAnswerException, MethodException , InterruptedException{
255 storeThrottle.delay();
256 PostMethod method = new PostMethod(actionURL);
257 method.setFollowRedirects(false);
258 method.addRequestHeader("User-Agent", userAgent);
259 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
261 NameValuePair[] params = new NameValuePair[] {
262 new NameValuePair("pages", pages),
263 new NameValuePair("curonly", "X"),
264 new NameValuePair("action", "submit") };
265 method.addParameters(params);
267 int responseCode = client.executeMethod(method);
268 String responseBody = method.getResponseBodyAsString();
270 if (responseCode == 200) {
271 StringReader reader = new StringReader(responseBody);
272 return XMLReader.readFromStream(reader);
274 throw new UnexpectedAnswerException("XML load not successful: expected 200 OK, got " + method.getStatusLine());
276 } catch(CoreException e) {
277 throw new UnexpectedAnswerException("XML load method failed" + e.getMessage());
278 } catch (HttpException e) {
279 throw new MethodException("XML load method failed", e);
280 } catch (IOException e) {
281 throw new MethodException("XML load method failed", e);
283 method.releaseConnection();
288 * store a Page Version - returns a Stored object
291 * WiKipedia predefined properties
299 * @throws UnexpectedAnswerException
300 * @throws MethodException
301 * @throws PageNotEditableException
302 * @throws InterruptedException
304 public Stored store(IWikipedia config, String actionUrl, String title, Content content, String summary, boolean minorEdit,
305 boolean watchThis) throws UnexpectedAnswerException, MethodException, PageNotEditableException, InterruptedException {
306 //### workaround: prevent too many stores at a time
307 storeThrottle.delay();
309 PostMethod method = new PostMethod(actionUrl);
310 method.setFollowRedirects(false);
311 method.addRequestHeader("User-Agent", userAgent);
312 method.addRequestHeader("Content-Type", PostMethod.FORM_URL_ENCODED_CONTENT_TYPE + "; charset=" + config.getCharSet());
313 NameValuePair[] params = new NameValuePair[] {
314 // new NameValuePair("wpSection", ""),
315 // new NameValuePair("wpPreview", "Vorschau zeigen"),
316 // new NameValuePair("wpSave", "Artikel speichern"),
317 new NameValuePair("title", title),
318 new NameValuePair("wpTextbox1", content.body),
319 new NameValuePair("wpEdittime", content.timestamp),
320 new NameValuePair("wpSummary", summary),
321 new NameValuePair("wpSave", "yes"),
322 new NameValuePair("action", "submit") };
323 method.addParameters(params);
325 method.addParameter("wpMinoredit", "1");
327 method.addParameter("wpWatchthis", "1");
331 int responseCode = client.executeMethod(method);
332 String responseBody = method.getResponseBodyAsString();
335 // since 11dec04 there is a single linefeed instead of an empty page.. trim() helps.
336 if (responseCode == 302 && responseBody.trim().length() == 0) {
337 // log("store successful, reloading");
338 Loaded loaded = load(actionUrl, config.getCharSet(), title);
339 result = new Stored(actionUrl, config.getCharSet(), loaded.title, loaded.content, false);
340 } else if (responseCode == 200) {
341 // log("store not successful, conflict detected");
342 Parsed parsed = parseBody(config.getCharSet(), responseBody);
343 Content cont = new Content(parsed.timestamp, parsed.body);
344 result = new Stored(actionUrl, config.getCharSet(), parsed.title, cont, true);
346 throw new UnexpectedAnswerException("store not successful: expected 200 OK, got " + method.getStatusLine());
348 } catch (HttpException e) {
349 throw new MethodException("method failed", e);
350 } catch (IOException e) {
351 throw new MethodException("method failed", e);
353 method.releaseConnection();
359 * Get the text of a wikimedia article
362 public String getWikiRawText(String wikiname, String urlStr) {
364 // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=raw
365 // http://en.wikibooks.org/w/index.php?title=Programming:PHP:SQL_Injection&action=raw
366 // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=raw
367 HttpMethod method = null;
369 if (urlStr == null) {
370 WikiEditorPlugin.getDefault().reportError("No Wikipedia URL configured", "URL-String == null");
371 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=raw";
373 URI uri = new URI(urlStr.toCharArray());
375 String schema = uri.getScheme();
376 if ((schema == null) || (schema.equals(""))) {
379 Protocol protocol = Protocol.getProtocol(schema);
381 HttpState state = new HttpState();
383 method = new GetMethod(uri.toString());
384 String host = uri.getHost();
385 int port = uri.getPort();
387 HttpConnection connection = new HttpConnection(host, port, protocol);
388 // timeout after 30 seconds
389 connection.setConnectionTimeout(30000);
390 connection.setProxyHost(System.getProperty("http.proxyHost"));
391 connection.setProxyPort(Integer.parseInt(System.getProperty("http.proxyPort", "80")));
393 if (System.getProperty("http.proxyUserName") != null) {
394 state.setProxyCredentials(null, null, new UsernamePasswordCredentials(System.getProperty("http.proxyUserName"), System
395 .getProperty("http.proxyPassword")));
398 if (connection.isProxied() && connection.isSecure()) {
399 method = new ConnectMethod(method);
402 method.execute(state, connection);
403 // client.executeMethod(method);
405 if (method.getStatusCode() == HttpStatus.SC_OK) {
406 // get the wiki text now:
407 String wikiText = method.getResponseBodyAsString();
410 } catch (Throwable e) {
411 WikiEditorPlugin.log(e);
412 WikiEditorPlugin.getDefault().reportError("Exception occured", e.getMessage() + "\nSee stacktrace in /.metadata/.log file.");
414 if (method != null) {
415 method.releaseConnection();
418 return null; // no success in getting wiki text
421 // public static String getWikiEditTextarea(String wikiname, String urlStr) {
423 // // http://en.wikipedia.org/w/wiki.phtml?title=Main_Page&action=edit
424 // // http://en.wikibooks.org/w/wiki.phtml?title=Programming:PHP:SQL_Injection&action=edit
425 // // http://en.wikipedia.org/w/wiki.phtml?title=Talk:Division_by_zero&action=edit
426 // HttpMethod method = null;
428 // if (urlStr == null) {
429 // urlStr = "http://en.wikipedia.org/w/wiki.phtml?title=" + wikiname + "&action=edit";
432 // // urlStr = urlStr + "?title=" + wikiname + "&action=edit";
434 // URI uri = new URI(urlStr.toCharArray());
436 // String schema = uri.getScheme();
437 // if ((schema == null) || (schema.equals(""))) {
440 // Protocol protocol = Protocol.getProtocol(schema);
442 // HttpState state = new HttpState();
444 // method = new GetMethod(uri.toString());
445 // String host = uri.getHost();
446 // int port = uri.getPort();
448 // HttpConnection connection = new HttpConnection(host, port, protocol);
450 // connection.setProxyHost(System.getProperty("http.proxyHost"));
451 // connection.setProxyPort(Integer.parseInt(System.getProperty("http.proxyPort", "80")));
453 // if (System.getProperty("http.proxyUserName") != null) {
454 // state.setProxyCredentials(null, null, new UsernamePasswordCredentials(System.getProperty("http.proxyUserName"), System
455 // .getProperty("http.proxyPassword")));
458 // if (connection.isProxied() && connection.isSecure()) {
459 // method = new ConnectMethod(method);
462 // method.execute(state, connection);
464 // if (method.getStatusCode() == HttpStatus.SC_OK) {
465 // // get the textareas wiki text now:
466 // InputStream stream = method.getResponseBodyAsStream();
467 // int byteLen = stream.available();
469 // byte[] buffer = new byte[byteLen];
470 // stream.read(buffer, 0, byteLen);
471 // String wikiText = new String(buffer);
472 // // String wikiText = method.getResponseBodyAsString();
473 // int start = wikiText.indexOf("<textarea");
474 // if (start != (-1)) {
475 // start = wikiText.indexOf(">", start + 1);
476 // if (start != (-1)) {
477 // int end = wikiText.indexOf("</textarea>");
478 // wikiText = wikiText.substring(start + 1, end);
482 // // System.out.println(wikiText);
485 // } catch (Exception e) {
486 // e.printStackTrace();
488 // if (method != null) {
489 // method.releaseConnection();
492 // return null; // no success in getting wiki text
495 public static void main(String[] args) {
496 MediaWikiConnector mwc = new MediaWikiConnector();
498 IWikipedia wp = null;
499 ArrayList list = mwc.loadXML(wp, "http://www.plog4u.de/wiki/index.php/Spezial:Export", "Mechanisches Fernsehen\nSynästhesie");
500 for (int i = 0; i < list.size(); i++) {
501 System.out.println(list.get(i).toString());
503 } catch (UnexpectedAnswerException e) {
504 // TODO Auto-generated catch block
506 } catch (Exception e) {
507 // TODO Auto-generated catch block