diff options
| -rw-r--r-- | wsj.com.txt | 2 |
1 files changed, 2 insertions, 0 deletions
diff --git a/wsj.com.txt b/wsj.com.txt index 467c39c..4b63433 100644 --- a/wsj.com.txt +++ b/wsj.com.txt @@ -18,6 +18,8 @@ strip: //*[contains(@style, 'visibility: hidden;')] strip: //div[contains(@class, 'insetContent') and not(contains(@class, 'image'))] strip: //div[contains(@class, 'carousel')] +http_header(user-agent): Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + prune: no tidy: no |