From b94a698e7353a752baf2db55cef1bc796532d9c5 Mon Sep 17 00:00:00 2001 From: "FiveFilters.org" Date: Fri, 19 Feb 2016 22:49:31 +0100 Subject: Update wsj.com.txt --- wsj.com.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/wsj.com.txt b/wsj.com.txt index 467c39c..4b63433 100644 --- a/wsj.com.txt +++ b/wsj.com.txt @@ -18,6 +18,8 @@ strip: //*[contains(@style, 'visibility: hidden;')] strip: //div[contains(@class, 'insetContent') and not(contains(@class, 'image'))] strip: //div[contains(@class, 'carousel')] +http_header(user-agent): Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + prune: no tidy: no -- cgit v1.3.1