diff options
67 files changed, 367 insertions, 278 deletions
diff --git a/.estadao.com.br.txt b/.estadao.com.br.txt deleted file mode 100644 index 00290c1..0000000 --- a/.estadao.com.br.txt +++ /dev/null @@ -1,15 +0,0 @@ -title: //span[@id='ctl00_ctl00_MainContent_MainContent_RecipeImage1_lblRecipeTitle'] -body: //div[@class='img_article'] | //div[@class='article']//div[@class='article_header' or @class='article_content'] -body: //div[@class='texto-noticia'] - -author: //div[@class='autor']//em -author: //div[@class='bb-md-noticia-autor'] - -strip_id_or_class: divulgar -strip_id_or_class: innerRight -strip: //div[@class='size' or @class='imprimir'] - -prune: no - -test_url: http://revistapiaui.estadao.com.br/edicao-62/carta-de-havana/la-vida-por-la-izquierda -test_url: http://economia.estadao.com.br/noticias/economia,pf-panamericano-tambem-irrigou-contas-de-executivos-do-grupo-silvio-santos,94648,0.htm
\ No newline at end of file diff --git a/.orf.at.txt b/.orf.at.txt new file mode 100644 index 0000000..ad3d7ce --- /dev/null +++ b/.orf.at.txt @@ -0,0 +1,25 @@ +single_page_link: //div[@id='ss-storyText']//p[@class='readMore']/a + +title: substring-before(//title,' - ') +body: //div[@id="ss-storyText"] +author: substring-before(//div[@id="ss-storyText"]//p[contains(text(), ', ORF.at')], ', ORF.at') +strip: //div[@id="ss-storyText"]//p[contains(text(), ', ORF.at')] +date: substring-after(//div[@class='storyMeta socialshare']//p[@class='date'],'Publiziert am') +strip: //p[@class='date'] + +strip: //p[@class='credit'] +strip: //p[@class='toplink'] +strip: //div[@id="ss-storyText"]/h1 +strip: //div[@class='socialButtons'] +strip: //div[@class='storyMeta socialshare'] +strip: //div[@class='socialShareWrapper'] +strip: //div[@id='socialshareprivacy'] + +prune: no +tidy: no + +test_url: http://orf.at/stories/2317355/ +test_url: http://orf.at/stories/2084731/ +test_url: http://orf.at/stories/2317313/2317311/ +test_url: http://wien.orf.at/news/stories/2746414/ +test_url: http://ooe.orf.at/news/stories/2750613/ diff --git a/applature.com.txt b/applature.com.txt deleted file mode 100644 index a820bba..0000000 --- a/applature.com.txt +++ /dev/null @@ -1,18 +0,0 @@ -title: //h1[contains(@class, 'title')# -body: //div[@id='mainContent']//div[contains(@class, 'section_content')] | //ul[@class='section_footer'] -date: //div[@class='date'] - -strip_id_or_class: sharethis -strip_id_or_class: stats -strip_id_or_class: apply_form -strip_id_or_class: job_map -strip_id_or_class: respond -strip: //h1//span[@class='type'] -strip: //li[@class='print' or @class='map'] - -replace_string(<ul class="section_footer" style="display): <ul class="section_footer" style="display-bla - -prune: no -tidy: no - -test_url: http://applature.com/mining-jobs/jobs/nickel-west-leinster-analytical-laboratory-technician/
\ No newline at end of file diff --git a/bbc.co.uk.txt b/bbc.co.uk.txt index 7bef73a..8abacde 100644 --- a/bbc.co.uk.txt +++ b/bbc.co.uk.txt @@ -1,4 +1,4 @@ -body: //div[@class="story-body"] +body: //div[@id="story-body"] # for video entries body: //div[contains(@class, "videoInStory") or @id="meta-information"] title: //h1[@class="story-header"] @@ -30,18 +30,27 @@ strip: //div[contains(@class, 'comment-introduction')] strip: //div[contains(@class, 'share-tools')] strip: //div[@id='also-related-links'] +strip: //figcaption +strip_id_or_class: image-and-copyright-container + +strip: //aside[contains(@class, 'sp-pullout')] + +strip_id_or_class: share-help +strip_id_or_class: comments_module + find_string: http://ichef.bbci.co.uk/news/200/ replace_string: http://ichef.bbci.co.uk/news/624/ find_string: http://ichef.bbci.co.uk/news/304/ replace_string: http://ichef.bbci.co.uk/news/624/ -strip_id_or_class: share-help -strip_id_or_class: comments_module +replace_string({width}{hidpi}): 624 replace_string(<noscript>): <div> replace_string(</noscript>): </div> +native_ad_clue: //meta[@property="og:url" and contains(@content, '/sponsored/')] + tidy: no prune: no diff --git a/bbc.com.txt b/bbc.com.txt index 200dba6..baf12b9 100644 --- a/bbc.com.txt +++ b/bbc.com.txt @@ -1,4 +1,4 @@ -body: //div[@class="story-body"] +body: //div[@id="story-body"] # for video entries body: //div[contains(@class, "videoInStory") or @id="meta-information"] title: //h1[@class="story-header"] @@ -30,6 +30,11 @@ strip: //div[contains(@class, 'comment-introduction')] strip: //div[contains(@class, 'share-tools')] strip: //div[@id='also-related-links'] +strip: //figcaption +strip_id_or_class: image-and-copyright-container + +strip: //aside[contains(@class, 'sp-pullout')] + strip_id_or_class: share-help strip_id_or_class: comments_module @@ -39,6 +44,8 @@ replace_string: http://ichef.bbci.co.uk/news/624/ find_string: http://ichef.bbci.co.uk/news/304/ replace_string: http://ichef.bbci.co.uk/news/624/ +replace_string({width}{hidpi}): 624 + replace_string(<noscript>): <div> replace_string(</noscript>): </div> diff --git a/blog.renren.com.txt b/blog.renren.com.txt index 401d31e..0193842 100644 --- a/blog.renren.com.txt +++ b/blog.renren.com.txt @@ -8,4 +8,4 @@ author://span[contains(@class, 'name')] body://div[contains(@class, 'content-body')] convert_double_br_tags:yes -test_url: http://blog.renren.com/share/230305019/1502806705
\ No newline at end of file +test_url: http://blog.renren.com/share/224959024/14260739544 diff --git a/bloomberg.com.txt b/bloomberg.com.txt new file mode 100644 index 0000000..c8715c4 --- /dev/null +++ b/bloomberg.com.txt @@ -0,0 +1,6 @@ +date: //*[@itemprop='datePublished']/@datetime +author: //meta[@name="sailthru.author"]/@content +body: //*[@itemprop='articleBody' or @itemprop='description'] + +test_url: http://www.bloomberg.com/news/articles/2015-12-30/will-your-job-disappear-by-2024- +test_contains: Every week, hosts Tori Stilwell
\ No newline at end of file diff --git a/caffereggio.net.txt b/caffereggio.net.txt new file mode 100644 index 0000000..08e1f05 --- /dev/null +++ b/caffereggio.net.txt @@ -0,0 +1,3 @@ +body: //div[@class="pf-content"] + +test_url: http://www.caffereggio.net/2016/01/16/115327/ diff --git a/chomsky.info.txt b/chomsky.info.txt index 2645f11..21c945e 100644 --- a/chomsky.info.txt +++ b/chomsky.info.txt @@ -1,6 +1,6 @@ -title: //div[@class='title'] -author: //div[@class='author'] +title: //*[@class='title'] +author: //*[@class='author'] prune: no -test_url: http://www.chomsky.info/onchomsky/2002----.htm -test_contains: The propaganda model argues +test_url: https://chomsky.info/20150820-2/ +test_contains: The first prime-time Republican primary debate diff --git a/christianpf.com.txt b/christianpf.com.txt deleted file mode 100644 index fb5f342..0000000 --- a/christianpf.com.txt +++ /dev/null @@ -1,5 +0,0 @@ -title: //h1[@class="entry-title"] -author: //*[@class="author vcard fn"] -date: //*[@class="published"] -body: //div[(@class = "dd_content_wrap")] -test_url: http://christianpf.com/do-ibuys-lead-to-more-buying/
\ No newline at end of file diff --git a/ciaosamin.com.txt b/ciaosamin.com.txt index 02fd343..ee09060 100644 --- a/ciaosamin.com.txt +++ b/ciaosamin.com.txt @@ -1,4 +1,4 @@ body://div[contains(@class, 'entry-content')] date://h2[contains(@class, 'date-header')] title://h3[contains(@class, 'post-title')] -test_url: http://www.ciaosamin.com/2013/04/how-this-happened.html
\ No newline at end of file +test_url: http://ciaosamin.com/ciao/2015/12/28/recipe-million-dollar-caramels diff --git a/counterpunch.org.txt b/counterpunch.org.txt index b6bd8be..12ca19f 100644 --- a/counterpunch.org.txt +++ b/counterpunch.org.txt @@ -1,6 +1,7 @@ title: //div[@class='main']//h1[contains(@class, 'article-title')] author: //div[@class='mainauthorstyle'] -body: //div[@class='main']//div[@class='main-text'] -strip: //td[@width='140'] +body: //div[@itemprop="articleBody"] +date: //meta[@itemprop="datePublished"]/@content +#strip: //td[@width='140'] -test_url: http://www.counterpunch.org/johnstone05172011.html
\ No newline at end of file +test_url: http://www.counterpunch.org/2011/09/28/the-dangerous-cult-of-the-guardian/ diff --git a/derstandard.at.txt b/derstandard.at.txt index 07db352..0bfeec0 100644 --- a/derstandard.at.txt +++ b/derstandard.at.txt @@ -1,13 +1,12 @@ -title: //div[@id='artikelHeader']/h1 +title: //div[@id='content-header']/h1 author: //span[@class='author'] -date: //span[@class='date'] body: //div[@class='copytext'] strip: //ul[@class='lookupLinksArtikel'] strip: //div[@id='pageTop'] strip: //div[@id='toolbar'] strip: //div[@id='articleTools'] -strip: //div[@id='weiterlesen'] +strip: //div[@id='weiterLesen'] strip: //div[@id='communityCanvas'] -test_url: http://derstandard.at/1318726018343/Breitband-LTE-Was-bringt-die-neue-Mobilfunk-Generation
\ No newline at end of file +test_url: http://derstandard.at/1318726018343/Breitband-LTE-Was-bringt-die-neue-Mobilfunk-Generation diff --git a/desitvforum.net.txt b/desitvforum.net.txt deleted file mode 100644 index c77007b..0000000 --- a/desitvforum.net.txt +++ /dev/null @@ -1,7 +0,0 @@ -body: (//blockquote[contains(@class, 'postcontent')])[1] -body: (//div[starts-with(@id, 'post_message')])[1] - -prune: no -tidy: no - -test_url: http://www.desitvforum.net/forum/watch-online/431739-creature-3d-2014-watch-online-download-dvd-rip.html @@ -1,5 +1,6 @@ -# Since this element has class="clear", the Instapaper stylesheets (at least this text parser preview), will render it unreadable, with a 1px font size and line height. +title: //h1[contains(@class, 'article__headline')] +body: //div[contains(@class, 'article__body-content') or contains(@class, 'article__lead') or contains(@class, 'image-box__container')] body: //div[@id="article-content"] @@ -12,7 +13,7 @@ strip_id_or_class: recommend strip_id_or_class: article-readers strip_id_or_class: article-addons strip_id_or_class: hook -strip_id_or_class: right +#strip_id_or_class: right strip_id_or_class: footer strip_id_or_class: ad-head @@ -24,6 +25,15 @@ strip: //div[@id="mirrors"] # Author author: //div[@id="byline"]/div/p/strong +find_string: <noscript> +replace_string: <!-- removed --> +find_string: </noscript> +replace_string: <!-- removed --> + +#parser: html5php +prune: no +tidy: no + # Date date: substring(substring-after(//p[@class="published"], 'Publicerad '), 0, 11) diff --git a/elpais.com.txt b/elpais.com.txt index c6f9787..7552afa 100644 --- a/elpais.com.txt +++ b/elpais.com.txt @@ -19,4 +19,4 @@ strip: //div[starts-with(@id, 'sumario') and contains(., 'más información')] strip: //div[@id='coment' or @id='foros_not'] test_url: http://elpais.com/elpais/2012/02/06/gente/1328526783_491687.html -test_url: http://www.elpais.com/articulo/cultura/mano/retrato/materia/elpepicul/20120207elpepicul_2/Tes
\ No newline at end of file +test_url: http://elpais.com/diario/2012/02/07/cultura/1328569202_850215.html diff --git a/estadao.com.br.txt b/estadao.com.br.txt new file mode 100644 index 0000000..5ed8f7e --- /dev/null +++ b/estadao.com.br.txt @@ -0,0 +1,4 @@ +title: //h1[contains(concat(' ',normalize-space(@class),' '),' titulo ')] +body: //article[contains(concat(' ',normalize-space(@class),' '),' texto ')] + +test_url: http://ciencia.estadao.com.br/noticias/geral,22-mil-toneladas-de-fosforo-do-saara-fertilizam-a-amazonia,1640532 diff --git a/facebook.com.txt b/facebook.com.txt index 2641a0b..f100990 100644 --- a/facebook.com.txt +++ b/facebook.com.txt @@ -7,8 +7,8 @@ strip: //div[contains(@data-sigil, 'm-mentions-expand')] prune: no tidy: no -single_page_link: concat("https://m.", substring-after(//link[@rel="alternate" and @media="handheld"]/@href, "//www.")) -if_page_contains: //link[@rel="alternate" and @media="handheld"] +#single_page_link: concat("https://m.", substring-after(//link[@rel="alternate" and @media="handheld"]/@href, "//www.")) +#if_page_contains: //link[@rel="alternate" and @media="handheld"] test_url: https://www.facebook.com/permalink.php?story_fbid=10154584776550183&id=294468630182 test_contains: holding an extraordinary session in Brussels this month diff --git a/facta.co.jp.txt b/facta.co.jp.txt index 4c96a1a..f130568 100644 --- a/facta.co.jp.txt +++ b/facta.co.jp.txt @@ -1,3 +1,3 @@ -bosdy: //div[@class='content'] +body: //div[@class='content'] -test_url: http://facta.co.jp/blog/archives/20111026001026.html
\ No newline at end of file +test_url: http://facta.co.jp/blog/archives/20111026001026.html diff --git a/fm4.orf.at.txt b/fm4.orf.at.txt index 5db3e58..0c8bf57 100644 --- a/fm4.orf.at.txt +++ b/fm4.orf.at.txt @@ -1,7 +1,13 @@ author: //div[@class='authorDescription']/h2 body: //div[@id='story'] -date: substring-before(substring-after(//p[@class='date'],'Erstellt am:'), '-') title: //h1[@class='detail'] -strip: //div[@class='fact'] -test_url: http://fm4.orf.at/stories/1689156/
\ No newline at end of file +date: translate(translate(substring-before(substring-after(//p[@class='date'],'Erstellt am:'), ' Uhr'), '. ', '.'), '-', ' ') +strip: //p[@class='date'] + +strip: //p[@class='credit'] + +tidy: no +prune: no + +test_url: http://fm4.orf.at/stories/1689156/ diff --git a/forbes.com.txt b/forbes.com.txt index 9e1d04c..36d4757 100644 --- a/forbes.com.txt +++ b/forbes.com.txt @@ -23,5 +23,7 @@ strip_id_or_class: engagement_block single_page_link: //a[contains(@href, '/print/')] +http_header(user-agent): Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + test_url: http://www.forbes.com/forbes/2011/0509/technology-frog-design-jan-chipchase-ethnographer-birth-cool_print.html -test_url: http://www.forbes.com/sites/bruceupbin/2012/09/11/the-iphone-5-winners-and-losers/
\ No newline at end of file +test_url: http://www.forbes.com/sites/bruceupbin/2012/09/11/the-iphone-5-winners-and-losers/ diff --git a/friendskorner.com.txt b/friendskorner.com.txt deleted file mode 100644 index b067d88..0000000 --- a/friendskorner.com.txt +++ /dev/null @@ -1,11 +0,0 @@ -#body: (//div[@class='ftr-yt-vid'])[1] -body: (//blockquote[contains(@class, 'postcontent')])[1] -body: (//div[starts-with(@id, 'post_message')])[1] - -prune: no -tidy: no - -#replace_string(<iframe title="YouTube video player"): <div class="ftr-yt-vid"><iframe title="YouTube video player" -#replace_string(</iframe>): </iframe> </div> - -test_url: http://www.friendskorner.com/forum/f137/debate-personal-lives-leaders-west-vs-pakistan-must-read-297989/
\ No newline at end of file @@ -1,5 +1,6 @@ -body: //div[contains(@class, 'ft-story-body')] +body: //div[@id='storyContent'] -author: substring-after(//div[contains(@class, 'ft-story-header')]/p[1], 'By ') -date: substring-before(substring-after(//div[contains(@class, 'ft-story-header')]/p[2], 'Published:'), '|') -test_url: http://www.ft.com/cms/s/2/e1be4b5a-620c-11e0-8ee4-00144feab49a.html
\ No newline at end of file +strip_id_or_class: shareArt +strip_id_or_class: promobox + +test_url: http://www.ft.com/cms/s/2/19fe32bc-d6db-11e5-8887-98e7feb46f27.html diff --git a/fubiz.net.txt b/fubiz.net.txt deleted file mode 100644 index 0dc3047..0000000 --- a/fubiz.net.txt +++ /dev/null @@ -1,3 +0,0 @@ -body: //div[@class = 'entry'] - -test_url: http://www.fubiz.net/2011/05/31/world-press-photo-2011/
\ No newline at end of file diff --git a/gizmodo.com.au.txt b/gizmodo.com.au.txt index 9dbfc15..d8e8c6c 100644 --- a/gizmodo.com.au.txt +++ b/gizmodo.com.au.txt @@ -5,4 +5,5 @@ date: //time prune: no -test_url: http://gizmodo.com/5880147/kuhn-rikon-improves-their-spice-grinder-with-grade-school-science +test_url: http://www.gizmodo.com.au/2016/01/these-hateful-eightfigures-are-delightfully-retro/ +test_contains: NECA have fully unveiled diff --git a/gov.ky.txt b/gov.ky.txt deleted file mode 100644 index 294ece3..0000000 --- a/gov.ky.txt +++ /dev/null @@ -1,4 +0,0 @@ -strip: //body//title - -test_url: http://www.gov.ky/pls/portal/PORTAL.wwv_media.show?p_id=7593947&p_settingssetid=1&p_settingssiteid=0&p_siteid=2425&p_type=basetext&p_textid=7593948 -test_url: http://www.rcips.ky/pls/portal/wlacomp.wlafeed.show_cignewsfeed_agency?p_sitecode=POL&p_agency=Police
\ No newline at end of file diff --git a/heise.de.txt b/heise.de.txt index 2c33208..9a472ae 100755 --- a/heise.de.txt +++ b/heise.de.txt @@ -9,6 +9,8 @@ author: //h4[@class='author'] body: //article | //div[@class='meldung_wrapper'] +strip: //nav + # General cleanup strip: //time strip: //header @@ -27,7 +29,7 @@ strip: //span[@class='source'] #strip: //div[@class='gallery compact']/figcaption strip_id_or_class: comments strip_id_or_class: ISI_IGNORE -strip_id_or_class: clear +#strip_id_or_class: clear strip_id_or_class: linkurl_grossbild strip_id_or_class: image-num @@ -44,6 +46,7 @@ replace_string(</h5>): </h2> replace_string(<span class="bild_rechts" style="width:): <p " replace_string(<div class="heisebox">): <blockquote> +single_page_link: //footer//a[contains(@href, '?view=print')] next_page_link: //a[@class='next'] next_page_link: //a[@title='vor'] diff --git a/imasters.com.br.txt b/imasters.com.br.txt new file mode 100644 index 0000000..f967d9c --- /dev/null +++ b/imasters.com.br.txt @@ -0,0 +1,5 @@ +title://h1[contains(concat(' ',normalize-space(@class),' '),' title ')] +body: //div[@id='content-font'] +strip://div[contains(concat(' ',normalize-space(@class),' '),' single-content ')]//div[contains(concat(' ',normalize-space(@class),' '),' single-info-post ')] + +test_url: http://imasters.com.br/desenvolvimento/como-se-destacar-em-programacao-e-banco-de-dados/?utm_source=iMasters+List&utm_campaign=ec7fb0575d-News_semanal_2014_07_297_28_2014&utm_medium=email&utm_term=0_c1528e6ab3-ec7fb0575d-359620857 diff --git a/iolanguage.com.txt b/iolanguage.com.txt deleted file mode 100644 index 231875a..0000000 --- a/iolanguage.com.txt +++ /dev/null @@ -1,2 +0,0 @@ -body: //center/table -test_url: http://www.iolanguage.com/scm/io/docs/IoGuide.html
\ No newline at end of file diff --git a/jetzt.de.txt b/jetzt.de.txt new file mode 100644 index 0000000..3b138f5 --- /dev/null +++ b/jetzt.de.txt @@ -0,0 +1,16 @@ +title: //header/h2 +strip: //header/h2 + +body: //div[@class='article__header-teaser'] | //div[@class='article__content']//p | //div[@class='article__content']//img | //div[@class='article__content']//a +strip: //ul[contains(@class, 'sharingbar')] +strip: //ol[@class='teaser__widget'] + +author: substring-after(//div[@class='article__header-author'], 'Von ') +author: substring-after(//div[@class='article__header-author'], 'Interview: ') +author: //div[@class='article__header-author'] + +date: //meta[@property='article:modified_time']/@content + +test_url: http://www.jetzt.de/hauptsache-reset/haeftlinge-als-unternehmensgruender +test_url: http://www.jetzt.de/maedchenfrage/maedchen-fragen-jungs-warum-balanciert-ihr-an-roten-ampeln-auf-dem-rad +test_url: http://www.jetzt.de/meine-theorie/das-perfekte-alter-ist-eine-frage-der-perspektive diff --git a/lalibre.be.txt b/lalibre.be.txt new file mode 100644 index 0000000..14e467c --- /dev/null +++ b/lalibre.be.txt @@ -0,0 +1,9 @@ +title: //h1[@class='mainContentTitle'] +date: //span[@class='publication']/time/@datetime + +body: //div[@class='articleText'] +strip_id_or_class: rekommenderade +strip_id_or_class: disqus +strip_id_or_class: annonser + +test_url: http://www.lalibre.be/actu/international/emmanuelle-mignon-l-ex-directrice-de-cabinet-de-sarkozy-mise-en-examen-56b10c9d3570b1fc10e24f20 diff --git a/letemps.ch.txt b/letemps.ch.txt deleted file mode 100644 index 49b019f..0000000 --- a/letemps.ch.txt +++ /dev/null @@ -1,3 +0,0 @@ -title: //h2 -strip_image_src: logo.gif -test_url: http://www.letemps.ch/Facet/print/Uuid/7c9f912c-07c9-11e0-9b50-4d96c9eca37f
\ No newline at end of file diff --git a/lwn.net.txt b/lwn.net.txt new file mode 100644 index 0000000..06a6dc6 --- /dev/null +++ b/lwn.net.txt @@ -0,0 +1,39 @@ +# HTML5 anyone? The 1980s called, they want their HTML4 back. +# LWN uses so little markup that you really have to be creative. + +tidy: yes +prune: no + +single_page_link: //div[@class='ArticleText']//a[contains(text(), 'Full Story')]/@href +single_page_link: concat(//div[@class='ArticleText']//a[contains(text(), 'Read more')]/@href, 'bigpage') + +title: //h1 + +# After tiding the document, <b> becomes <strong>. +author: //div[@class='FeatureByline']/strong +date: //div[@class='FeatureByline']/text()[preceding-sibling::br] +strip: //div[@class='FeatureByline'] +author: substring-after(//div[@class='GAByline']/p[2], 'by ') +date: //div[@class='GAByline']/p[1] +strip: //div[@class='GAByline'] + +# tidy will take care of fixing the tag mess that we make here. +replace_string(<p class="Cat1HL">): <h1> +replace_string(<h2 class="SummaryHL">): <h3> +replace_string(<p class="Cat2HL">): <h2> + +# Make extracting the content before "Log in to post comments" easier. +# And by "easier" I mean possible in all cases without going through +# a lot of XPath pain. +replace_string(<hr width="60%" align="left">): <div class="ftrss-strip"> +replace_string(to post comments)): </div> +strip: //div[@class='ftrss-strip'] +body: //div[@class='ArticleText'] + +test_url: http://lwn.net/Articles/668318/ +test_url: http://lwn.net/Articles/668695/ +test_url: http://lwn.net/Articles/669114/ +test_url: http://lwn.net/Articles/670209/ +test_url: http://lwn.net/Articles/670209/rss +test_url: http://lwn.net/Articles/668318/rss +test_url: http://lwn.net/Articles/670062/ diff --git a/m.facebook.com.txt b/m.facebook.com.txt index 1b9c1b3..81d2b00 100644 --- a/m.facebook.com.txt +++ b/m.facebook.com.txt @@ -8,5 +8,5 @@ strip: //*[contains(@data-sigil, 'm-mentions-expand') or contains(@data-sigil, ' prune: no tidy: no -test_url: https://www.facebook.com/permalink.php?story_fbid=10154584776550183&id=294468630182 +test_url: https://m.facebook.com/permalink.php?story_fbid=10154584776550183&id=294468630182 test_contains: holding an extraordinary session in Brussels this month diff --git a/mainichi.jp.txt b/mainichi.jp.txt deleted file mode 100644 index 414a2f5..0000000 --- a/mainichi.jp.txt +++ /dev/null @@ -1,3 +0,0 @@ -body: //div[@class='NewsArticle'] - -test_url: http://mainichi.jp/select/weathernews/20110311/news/20110520k0000e040062000c.html
\ No newline at end of file diff --git a/manga-news.com.txt b/manga-news.com.txt new file mode 100644 index 0000000..2fc5464 --- /dev/null +++ b/manga-news.com.txt @@ -0,0 +1,7 @@ +title: //h2[@class='actu-title'] + +body: //div[@class='actu-content'] + +prune: no + +test_url: http://www.manga-news.com/index.php/actus/2016/02/02/Deux-retours-dans-le-planning-des-editions-Pika diff --git a/medium.com.txt b/medium.com.txt index cc373f3..2c969c0 100644 --- a/medium.com.txt +++ b/medium.com.txt @@ -1,7 +1,11 @@ body: //main[contains(@class, 'postArticle-content')] body: //div[contains(@class, 'postArticle-content')] +body: //div[contains(@class, 'section-inner')] strip_id_or_class: supplementalPostContent +parser: html5php + +tidy: no prune: no test_url: https://medium.com/@savolai/kaytettavyyden-haasteet-keskustelukulttuurista-2-3-6844c0d7893b diff --git a/mobile.nytimes.com.txt b/mobile.nytimes.com.txt index ef1ce98..3dcbdf4 100644 --- a/mobile.nytimes.com.txt +++ b/mobile.nytimes.com.txt @@ -3,5 +3,7 @@ body: //article[contains(@class, 'full-art')] date: //meta[@name="pdate"]/@content author: //meta[@name="byl"]/@content +tidy: no + strip_id_or_class: image-credit -test_url: http://mobile.nytimes.com/2014/06/19/opinion/gail-collins-romney-and-the-2016-contenders-huddle.html
\ No newline at end of file +test_url: http://mobile.nytimes.com/2014/06/19/opinion/gail-collins-romney-and-the-2016-contenders-huddle.html diff --git a/mobile.twitter.com.txt b/mobile.twitter.com.txt new file mode 100644 index 0000000..110df84 --- /dev/null +++ b/mobile.twitter.com.txt @@ -0,0 +1,12 @@ +# mobile site (automatic redirect - noscript meta refresh) +author: (//*[contains(@class, 'UserNames-displayName') or contains(@class, 'fullname')])[1] +body: (//div[contains(@class, 'TweetDetail-text') or contains(@class, 'tweet-text')])[1] +date: (//div[contains(@class, 'TweetDetail-timeAndGeo') or contains(@class, 'metadata')])[1] + +parser: html5php + +prune: no +tidy: yes + +test_url: https://mobile.twitter.com/medialens/status/216883678582804480 +test_contains: is all but alone in challenging the tsunami of UK diff --git a/momentumsaga.com.txt b/momentumsaga.com.txt new file mode 100644 index 0000000..83c2988 --- /dev/null +++ b/momentumsaga.com.txt @@ -0,0 +1,5 @@ +title: //div[contains(concat(' ',normalize-space(@class),' '),' post-header ')]//h1 +body: //div[contains(concat(' ',normalize-space(@class),' '),' post-entry ')] +date: //span[contains(concat(' ',normalize-space(@class),' '),' date ') + +test_url: http://www.momentumsaga.com/2015/03/meus-dez-livros-preferidos-de-ficcao-cientifica.html diff --git a/moreintelligentlife.com.txt b/moreintelligentlife.com.txt deleted file mode 100644 index 780cca4..0000000 --- a/moreintelligentlife.com.txt +++ /dev/null @@ -1,7 +0,0 @@ -title: //h1[@class='print-title'] -body: //div[@class='print-submitted' or @class='print-created' or @class='print-content'] -prune: no - -single_page_link: //li[@class='print']/a - -test_url: http://moreintelligentlife.com/content/places/paul-markillie/they-trash-cars-dont-they
\ No newline at end of file diff --git a/news.orf.at.txt b/news.orf.at.txt deleted file mode 100644 index 3b1d3cc..0000000 --- a/news.orf.at.txt +++ /dev/null @@ -1,11 +0,0 @@ -single_page_link: //div[@id='content']//p[@class='readMore']/a - -title: //div[@class='hidden offscreen']/h2 -body: //div[@id="storyText"] -move_into(//div[@id='storyText']): //div[@class='fact'] -strip: //small[@class='credit'] -strip: //small[@class='caption'] -date: substring-after(//div[@id='storyMeta']//p[@class='date'],'Publiziert am') -strip: //p[@class='toplink'] - -test_url: http://news.orf.at/stories/2084731/
\ No newline at end of file @@ -1,11 +1,25 @@ -single_page_link: //div[@id='content']//p[@class='readMore']/a +single_page_link: //div[@id='ss-storyText']//p[@class='readMore']/a -title: //div[@class='hidden offscreen']/h2 -body: //div[@id="storyText"] -move_into(//div[@id='storyText']): //div[@class='fact'] -strip: //small[@class='credit'] -strip: //small[@class='caption'] -date: substring-after(//div[@id='storyMeta']//p[@class='date'],'Publiziert am') +title: substring-before(//title,' - ') +body: //div[@id="ss-storyText"] +author: substring-before(//div[@id="ss-storyText"]//p[contains(text(), ', ORF.at')], ', ORF.at') +strip: //div[@id="ss-storyText"]//p[contains(text(), ', ORF.at')] +date: substring-after(//div[@class='storyMeta socialshare']//p[@class='date'],'Publiziert am') +strip: //p[@class='date'] + +strip: //p[@class='credit'] strip: //p[@class='toplink'] +strip: //div[@id="ss-storyText"]/h1 +strip: //div[@class='socialButtons'] +strip: //div[@class='storyMeta socialshare'] +strip: //div[@class='socialShareWrapper'] +strip: //div[@id='socialshareprivacy'] + +prune: no +tidy: no -test_url: http://orf.at/stories/2084731/
\ No newline at end of file +test_url: http://orf.at/stories/2317355/ +test_url: http://orf.at/stories/2084731/ +test_url: http://orf.at/stories/2317313/2317311/ +test_url: http://wien.orf.at/news/stories/2746414/ +test_url: http://ooe.orf.at/news/stories/2750613/ diff --git a/parool.nl.txt b/parool.nl.txt index 2ceef94..65fc5a9 100644 --- a/parool.nl.txt +++ b/parool.nl.txt @@ -1,7 +1,9 @@ #bypass cookie check -single_page_link: //a[contains(@href, '/acceptCookieCheck.do?url=')] +single_page_link: //a[contains(@href, '/accept?url=')] test_url: http://www.parool.nl/parool/nl/4/AMSTERDAM/article/detail/4042734/2015/05/29/MRSA-bacterie-niet-verder-verspreid-in-Bijlmerbajes.dhtml test_contains: De twee gevangenen die -test_url: http://www.parool.nl/amsterdam/rss.xml
\ No newline at end of file +http_header(user-agent): Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + +test_url: http://www.parool.nl/amsterdam/rss.xml diff --git a/phonandroid.com.txt b/phonandroid.com.txt new file mode 100644 index 0000000..138cf19 --- /dev/null +++ b/phonandroid.com.txt @@ -0,0 +1,9 @@ +body: //section[@class='content-txt'] +title: //h1[@class='entry-title'] +author: //span[@class='author'] +date: //span[@class='infos']/time/@datetime + +prune: no +test_url: http://www.phonandroid.com/marche-tablettes-encore-pris-belle-claque-2015.html +test_url: http://krugman.blogs.nytimes.com/2012/09/12/a-vote-of-confidence/ +test_url: http://bits.blogs.nytimes.com/2012/01/16/wikipedia-plans-to-go-dark-on-wednesday-to-protest-sopa/ diff --git a/quora.com.txt b/quora.com.txt index f2b75a9..cfa7bd6 100644 --- a/quora.com.txt +++ b/quora.com.txt @@ -22,8 +22,7 @@ find_string: <div class="Answer" replace_string: <hr /><div class="Answer" test_url: http://www.quora.com/What-everyday-habit-do-you-wish-you-had-developed-earlier-in-life -test_contains: Please provide a specific practical/measurable action-based everyday -test_contains: Exercise every day +test_contains: Exercising every day test_url: http://www.quora.com/What-is-the-greatest-illusion-in-life test_contains: What is the greatest illusion in life? diff --git a/richardmuscat.wordpress.com.txt b/richardmuscat.wordpress.com.txt deleted file mode 100644 index b0ee92d..0000000 --- a/richardmuscat.wordpress.com.txt +++ /dev/null @@ -1,5 +0,0 @@ -body: //div[@id="post"] -strip: //div[@id="author-description"] -date: //span[@class="entry-date"] -author: //span[@class="author vcard"] -test_url: http://richardmuscat.wordpress.com/2011/06/20/the-price-of-free/?utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+TheBrooksReview+%28The+Brooks+Review%29
\ No newline at end of file diff --git a/ritholtz.com.txt b/ritholtz.com.txt deleted file mode 100644 index d598479..0000000 --- a/ritholtz.com.txt +++ /dev/null @@ -1,5 +0,0 @@ -title: //div[@class='post']/h2 -author: substring-before(substring-after(//div[@class='alignright']/small, 'By '),'-') -date: substring-after(//div[@class='alignright']/small, '-') -strip: //div[@class='alignleft'] -test_url: http://www.ritholtz.com/blog/2012/09/situational-awareness/
\ No newline at end of file diff --git a/rolfinjapan.nl.txt b/rolfinjapan.nl.txt deleted file mode 100644 index 2365c42..0000000 --- a/rolfinjapan.nl.txt +++ /dev/null @@ -1,6 +0,0 @@ -body: //div[contains(@class, 'inhoud')] -date: //span[@class ='published'] -author: //span[@class ='author'] -strip: //div[@class = 'grid_2'] -strip: //div[@class = 'block-citation-text'] -test_url: http://www.rolfinjapan.nl/2011/06/duizend-kraanvogels/
\ No newline at end of file diff --git a/scraplab.net.txt b/scraplab.net.txt deleted file mode 100644 index ca7ec19..0000000 --- a/scraplab.net.txt +++ /dev/null @@ -1,3 +0,0 @@ -title: //h2 -body: //div[@class='body'] -test_url: http://scraplab.net/2010/10/26/please-keep-your-belongings-with-you-at-all-times/
\ No newline at end of file diff --git a/staradvertiser.com.txt b/staradvertiser.com.txt deleted file mode 100644 index 254e2c2..0000000 --- a/staradvertiser.com.txt +++ /dev/null @@ -1,11 +0,0 @@ -title: //h1[@id='storyTitle'] -author: substring-after(//span[@class='hsa_postCredit'], 'By ') -date://span[@class='hsa_dateStamp'] -body: //div[@class='storytext'] -strip_id_or_class: insideStoryAd -strip_id_or_class: printDesc -strip_id_or_class: sb_2010_story_tools -strip_id_or_class: FBConnectButton_Text -strip_id_or_class: breadcrumbs -prune: no -test_url: http://www.staradvertiser.com/news/20111112_World_leaders_step_onto_isle_stage.html
\ No newline at end of file diff --git a/summify.com.txt b/summify.com.txt deleted file mode 100644 index 1128e1b..0000000 --- a/summify.com.txt +++ /dev/null @@ -1,2 +0,0 @@ -strip_id_or_class: toolbar -test_url: http://summify.com/story/Tmt1YQ0JBgKTAHwK/www.nybooks.com/articles/archives/2003/jan/16/fixed-opinions-or-the-hinge-of-history/?pagination=false
\ No newline at end of file diff --git a/tagesschau.de.txt b/tagesschau.de.txt index ba3b1d3..507aab2 100644 --- a/tagesschau.de.txt +++ b/tagesschau.de.txt @@ -1,4 +1,4 @@ -body: //div[contains(@class, 'sectionArticle')]//div[contains(concat(' ',normalize-space(@class),' '),' box ')] +body: //div[contains(@class, 'sectionArticle') and contains(@class, 'sectionZ')] strip_id_or_class: infokasten strip_id_or_class: teaserImTeaser diff --git a/tech.gilt.com.txt b/tech.gilt.com.txt deleted file mode 100644 index ab56460..0000000 --- a/tech.gilt.com.txt +++ /dev/null @@ -1,5 +0,0 @@ -title: //div[@class="title"]/h1 -title: //div[@class="caption"]/h1 -author: substring-after(//div[@class="metadata"]/div[@class="date"]/a[2], 'by ') -date: //div[@class="metadata"]/div[@class="date"]/a -test_url: http://tech.gilt.com/post/46359463184/26-3-13-todays-noon-outage-and-what-were-doing-to
\ No newline at end of file diff --git a/techcrunch.com.txt b/techcrunch.com.txt index 1509c46..64e3e18 100644 --- a/techcrunch.com.txt +++ b/techcrunch.com.txt @@ -1,11 +1,12 @@ -body: //div[contains(@class, 'media-container') or contains(@class, 'body-copy')] +body: //div[@class='rticle-entry text'] -author: //a[@class="name"] +author: //div[@class='byline']/a -date: //div[@class="post-time"] +date: //div[@class="byline"]/time/@datetime -title: //h1[@class="headline"] +title: //h1[@class="tweet-title"] strip_id_or_class: module-crunchbase +strip_id_or_class: aside aside-related-articles # The following is for the mobile site body: //div[@id="singlentry"] @@ -15,4 +16,4 @@ title: //a[@class="sh2"] prune: no -test_url: http://techcrunch.com/2011/10/18/apples-insanely-great-q1-2012/
\ No newline at end of file +test_url: http://techcrunch.com/2016/02/02/spotcap/?ncid=rss&utm_source=feedburner&utm_medium=feed&utm_campaign=Feed%3A+francaistechcrunch+%28TechCrunch+en+Francais%29 diff --git a/techhive.com.txt b/techhive.com.txt index 29720b0..48ecf30 100644 --- a/techhive.com.txt +++ b/techhive.com.txt @@ -15,4 +15,4 @@ strip: //div[@class="department"] #strip product cap links strip: //div[@class="cap-main"] strip: //div[@id="compare-lede"] -test_url: http://www.techhive.com/article/2010549/up-close-with-blackberry-10.html
\ No newline at end of file +test_url: http://www.techhive.com/article/3023383/streaming-media/vlc-for-apple-tv-review-goodbye-format-woes.html diff --git a/techno-science.net.txt b/techno-science.net.txt new file mode 100644 index 0000000..31dd7f4 --- /dev/null +++ b/techno-science.net.txt @@ -0,0 +1,3 @@ +title://div[@class="news"]/div[@class="titre"] +body://div[@class="news"]/div[@class="texte"] +test_url: http://www.techno-science.net/?onglet=news&news=14808 diff --git a/theguardian.com.txt b/theguardian.com.txt index c8b70e6..0a21de2 100644 --- a/theguardian.com.txt +++ b/theguardian.com.txt @@ -1,18 +1,30 @@ -title: //div[@id='main-article-info']//h1 -body: //figure[contains(@itemprop, "associatedMedia")] | //div[contains(@itemprop, "articleBody")] -date: //li[@class='publication']//time[@pubdate] | //li[@class='publication']//data[@pubdate] -strip: //div[contains(@class, 'email-subscription')] -strip: //div[contains(@class, 'kindleWidget')] -#strip: //a[not(text())] -strip_id_or_class: pocket-btn -author: //li[@class='byline'] -native_ad_clue: //meta[@property="article:tag" and contains(@content, "Partner zone")] -native_ad_clue: //meta[@property="video:tag" and contains(@content, "Partner zone")] +title: //h1[@itemprop='headline'] + +body: //article +strip: //article/header/div[contains(@class, 'content__header')] +strip: //article/header/div[contains(@class, 'content__logo-container')] +strip: //article//div[contains(@class, 'content__secondary-column')] +strip: //article//aside +strip: //article//div[contains(@class, 'block-share')] +strip: //article//div[@class='submeta'] +strip: //article//span[contains(@class, 'inline-expand-image')] +strip: //article//div[@class='kindleWidget'] +strip: //article//div[@class='email-subscription'] +strip: //article//script +strip: //article//figure[contains(@class, 'element-audio')] +strip: //article//a[contains(@style, 'display: none')] +strip: //article/div[@class='paidfor-band'] + +author: //article//p[@class='byline'] +date: //article//time/@datetime +strip: //article//div[contains(@class, 'content__meta-container')] + +native_ad_clue: //meta[@property='article:tag' and contains(@content, 'partner zone')] +native_ad_clue: //meta[@property='video:tag' and contains(@content, 'partner zone')] + prune: no tidy: no -strip_id_or_class: -expand- - test_url: http://www.theguardian.com/world/2013/oct/04/nsa-gchq-attack-tor-network-encryption test_contains: The National Security Agency has made repeated attempts to develop test_contains: The agency did not directly address those questions, instead providing a statement. @@ -22,5 +34,8 @@ test_contains: In August, the editor of the Guardian rang me up and asked if I w test_contains: As the second most senior judge in the country, Lord Hoffmann, said in 2004 about a previous version of our anti-terrorism laws test_url: http://www.theguardian.com/commentisfree/2014/jun/15/britishness-search-identity-my-part-in-camerons-odyssey +test_url: http://www.theguardian.com/world/2016/feb/17/ankara-explosion-turkey-injures-large-number-of-people-reports-say +test_url: http://www.theguardian.com/uk-news/2016/feb/11/trident-the-british-question + # Native ad -test_url: http://www.theguardian.com/sustainable-business/2014/jul/18/ben-jerry-turn-ice-cream-into-energy +test_url: http://www.theguardian.com/sustainable-business/fairtrade-partner-zone/chocolate-cocoa-production-risk diff --git a/themuseumofinnocence.com.txt b/themuseumofinnocence.com.txt deleted file mode 100644 index 80aba44..0000000 --- a/themuseumofinnocence.com.txt +++ /dev/null @@ -1,7 +0,0 @@ -body: single-review -strip_id_or_class: featured-review -strip_id_or_class: resources -strip_id_or_class: rate-the-book -strip_id_or_class: write-review - -test_url: http://themuseumofinnocence.com/review.php?id=1179
\ No newline at end of file diff --git a/thenextweb.com.txt b/thenextweb.com.txt index e72a1f5..83bc759 100644 --- a/thenextweb.com.txt +++ b/thenextweb.com.txt @@ -1,10 +1,13 @@ +title: //h1[@class='title'] body: //div[@class='l-postSingle-content-main-video'] | //div[contains(concat(' ',normalize-space(@class),' '),' l-postSingle-content-main-content ')] -author: //meta[@name="author"]/@content +author: //a[@class='author'] +date: //div[@class='meta']/time/@datetime strip: //div[contains(concat(' ',normalize-space(@class),' '),' l-postSingle-content-bottom-share ')] +strip_id_or_class: articleShare-buttons replace_string(data-original=): src= tidy: no test_url: http://thenextweb.com/apple/2011/10/12/tnw-review-a-complete-guide-to-apples-ios-5-with-icloud-an-os-14-years-in-the-making/ -test_url: http://thenextweb.com/insider/2015/12/06/pornhubs-christmas-ad-wants-you-to-give-premium-subscriptions-to-your-grandparents/
\ No newline at end of file +test_url: http://thenextweb.com/insider/2015/12/06/pornhubs-christmas-ad-wants-you-to-give-premium-subscriptions-to-your-grandparents/ diff --git a/theverge.com.txt b/theverge.com.txt index cee50c9..56e7b37 100644 --- a/theverge.com.txt +++ b/theverge.com.txt @@ -1,8 +1,8 @@ author: //p[contains(@class, "byline")]/a[contains(@class, "author")] - +title: //[h1=@class="stream_title"] date: //span[contains(@class, "publish-date")]/time[@pubdate]/@datetime -body: //div[contains(@class, 'entry-content')] +body: //div[contains(@class, 'm-article__entry-section')] # for vergecasts, e.g. http://www.theverge.com/2013/8/22/4648566/the-vergecast-090-august-22th-2013-video body: //article prune: no @@ -40,7 +40,7 @@ replace_string(<noscript>): <div> replace_string(</noscript>): </div> find_string: <script -replace_string: <div style="display:none" +replace_string: <div style="display:none" find_string: </script> replace_string: </div> diff --git a/twitter.com.txt b/twitter.com.txt index 0e5b748..0a010a4 100644 --- a/twitter.com.txt +++ b/twitter.com.txt @@ -3,6 +3,12 @@ body: (//p[contains(@class, 'js-tweet-text')])[1] author: (//strong[contains(@class, 'fullname')])[1] date: //span[contains(@class, 'js-short-timestamp')]/@data-time +# mobile site (automatic redirect - noscript meta refresh) +author: (//div[contains(@class, 'fullname')])[1] +body: (//div[contains(@class, 'TweetDetail-text')])[1] + +parser: html5php + prune: no tidy: no diff --git a/vanityfair.com.txt b/vanityfair.com.txt index f52339c..e340ed3 100644 --- a/vanityfair.com.txt +++ b/vanityfair.com.txt @@ -1,33 +1,10 @@ -title: //meta[@property="og:title"]/@content -author: //div[contains(@class, 'byline')]//span[contains(@class, 'name')] -date: //div[contains(@class, 'cn_date_time')] -body: //div[contains(@class, 'pageContainers')] -body: //div[@id='main'] -body: //article[@id='items-container'] -#body: //h2[@class='sub-header'] | //div[contains(@class, 'contributor-type') or @class='display-date' or @class='content-container'] +body: //div[(contains(concat(' ',normalize-space(@class),' '),' article-content-body '))] -strip_id_or_class: bc -strip_id_or_class: utilities -strip_id_or_class: list-supporting -strip_id_or_class: yrail -strip_id_or_class: urail +// we can't load this (JS load) so may as well strip it to remove byline +strip: //figure[contains(@class, 'main-image')] -prune: no -#tidy: no - -strip_id_or_class: super-rubric-section -strip_id_or_class: cn_date_time -strip_id_or_class: cn_contributors -strip_id_or_class: cn_pagination_controls -strip_id_or_class: cn_features_container -strip_id_or_class: global-footer -strip_id_or_class: cn_ecom_placement -strip: //li[@class='blogNavPrev'] - -single_page_link: //a[@title='Print this page'] - -test_url: http://www.vanityfair.com/politics/features/2011/05/egypt-revolutionaries-201105 -test_contains: nothing can take away from the miracle of Tahrir Square +test_url: http://www.vanityfair.com/news/2016/02/george-w-bush-donald-trump-iraq +test_contains: riding high off a fourth-place finish in New Hampshire test_url: http://www.vanityfair.com/politics/features/2008/08/hitchens200808 test_url: http://www.vanityfair.com/style/2012/01/prisoners-of-style-201201 diff --git a/volkskrant.nl.txt b/volkskrant.nl.txt index 8c311f1..f2f630f 100644 --- a/volkskrant.nl.txt +++ b/volkskrant.nl.txt @@ -4,12 +4,16 @@ single_page_link: //a[contains(@href, '/cookiewall/accept')] title: //h1[@itemprop="headline"] body: //figure[contains(@class, 'article__top-image')] | //div[@itemprop="articleBody"] +strip_id_or_class: ad-space + strip: //div[contains(@class, 'media-container') and contains(@class, 'pull-right')] tidy: no prune: no +parser: html5php + test_url: http://www.volkskrant.nl/sport/dossier-wereldvoetbalbond-fifa-wankelt~a4042695/ -test_contains: De belangrijkste spil in het +test_contains: Het ging om de omstreden -test_url: http://www.volkskrant.nl/nieuws/rss.xml
\ No newline at end of file +test_url: http://www.volkskrant.nl/nieuws/rss.xml diff --git a/williampfaff.com.txt b/williampfaff.com.txt deleted file mode 100644 index cefabec..0000000 --- a/williampfaff.com.txt +++ /dev/null @@ -1,3 +0,0 @@ -title: substring-after(//span[@class='itemTitle'], ':') -body: //div[@id='content'] -test_url: http://www.williampfaff.com/modules/news/article.php?storyid=491
\ No newline at end of file diff --git a/wsj.com.txt b/wsj.com.txt index 467c39c..4b63433 100644 --- a/wsj.com.txt +++ b/wsj.com.txt @@ -18,6 +18,8 @@ strip: //*[contains(@style, 'visibility: hidden;')] strip: //div[contains(@class, 'insetContent') and not(contains(@class, 'image'))] strip: //div[contains(@class, 'carousel')] +http_header(user-agent): Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html) + prune: no tidy: no diff --git a/zeit.de.txt b/zeit.de.txt index 4e00894..0b6a837 100644 --- a/zeit.de.txt +++ b/zeit.de.txt @@ -1,48 +1,62 @@ -# 2015.07.08 [Marvin Dickhaus] fixed single_page_link -# 2013.10.30 [rezor92] fixed single_page_link -# 2012-12-23 [carlo@...] fixed half-assed headlines in articles, removed inline author profiles, adjusted picture captions -# 2012-03-17 [dkless@...] Cut metadata parts in the beginning and the ends of the content block; copyright entries for pictures removed; Author fixed, not sure if old entries still valid (I left them); Weird problems with some pages addressed (see last section for removing hidden section) -# 2011-12-09 [carlo@...] Removed "related articles" block -# 2011-08-23 [carlo@...] changed single page link to use print version: page works better, less ambiguity. Related cleanups and simplifications. -# 2011-08-20 [carlo@...] added author, fixed date - -single_page_link: //a[contains(@href, 'komplettansicht')] tidy: no +prune: no + +# Figures are wrapped in a noscript tag which is itself wrapped in a +# conditional comment. Feed readers will fail to parse this correctly +# so get rid of the noscript tag altogether. +replace_string(<noscript): <foo -title: //title -date: substring-before( //li[@class="date"], " " ) -author: //li[@class="author"]/a/text() | //li[@class="author first"]/a/text() -author: substring-after(//li[@class='source first '], 'Quelle: ') +title: //meta[@property='og:title']/@content +date: //meta[@name='date']/@content -strip_id_or_class: articleheader -strip: //div[@id="comments"] | //div[@class="pagination block"] | //p[@class="ressortbacklink"] | //div[@id="relatedArticles"] | // div[@class="inline portrait"] +strip: //span[@class='figure__copyright'] -#Removes author and date from the start -strip: //ul[@class="tools"] -#Removes copyright statement - often disturb as first line of the news -strip: //p[@class="copyright"] -strip: //div[@class="copyright"] -#Removes pagination links at the end -strip: //div[@class="pagination"] -#Removes link to main page at the bottom of some articles (Zur Startseite) -strip: //a[@href='http://www.zeit.de'] +# Self advertisements +strip: //figure[@class='figure-stamp'] +strip: //a[contains(@title, 'Dieser Text ')] +strip: //a[contains(@title, 'Dieser Artikel ')] +strip: //span[@class='figure__text']/text()[contains(., 'Dieser Text ')] +strip: //span[@class='figure__text']/text()[contains(., 'Dieser Artikel ')] + +####################################### +# ZEIT: +####################################### + +single_page_link: //a[contains(@href, 'komplettansicht')] -# Fix picture captions -wrap_in(small): //p[@class="caption"]/text() +author: //a[@class='byline__author']/span +author: substring-after(//span[@class='metadata__source'], 'Quelle: ') -# Fix sub-headlines -wrap_in(h2): //p/strong -dissolve: //h2/strong +body: //main/article/div[@itemprop='articleBody'] -#Sometimes things are embedded in the print version that are not displayed on the web, but will be displayed in the mobilized versions and lead even to problems. These sections are removed here. -strip_id_or_class:"informatives" -strip_id_or_class:"bottom" -strip_id_or_class:"teasermosaic" -strip_id_or_class:"comments" -strip_id_or_class:"articlefooter af" -strip_id_or_class:"relateds" -strip_id_or_class:"pagination" +strip: //a[@href='#'] +strip: //form[@id='newsletter-teaser-form'] +strip_id_or_class: 'article-pagination article__item ' -footnotes: no test_url: http://www.zeit.de/kultur/film/2012-12/Kurzfilmtag -test_url: http://www.zeit.de/kultur/2015-07/kapitalismuskritik-selbstberuhigung-armin-nassehi +test_url: http://www.zeit.de/sport/2016-01/darts-wm-finale-anderson-lewis/komplettansicht +test_url: http://www.zeit.de/kultur/2015-12/selbstverwirklichung-optimierung-essay +test_url: http://www.zeit.de/2015/51/selbstdiagnose-gesundheit-zuhause-blut-urin-selbsttest +test_url: http://www.zeit.de/campus/2015/s2/nebenjob-master-studium-finanzierung +test_url: http://www.zeit.de/hamburg/politik-wirtschaft/2015-12/hamburg-wohnungsbau-fluechtlinge + +####################################### +# ZEIT MAGAZIN: +####################################### + +next_page_link: //meta[@rel='next']/@href + +author: //a[@class='article__head__meta__author']/span + +body: //main/article/div[@class='article__wrap'] + +strip: //div[@class='article__head-wrap']//h1 +strip_id_or_class: article__socialbox +strip_id_or_class: article__nextread nextread-base is-centered +strip_id_or_class: js-comments +strip_id_or_class: article__pagination is-constrained is-centered +strip_id_or_class: article__head__meta + +test_url: http://www.zeit.de/zeit-magazin/2015/51/daisy-ridley-traum +test_url: http://www.zeit.de/zeit-magazin/2015/52/cyberstalking-internet-stalker-familie-mierau +test_url: http://www.zeit.de/zeit-magazin/leben/2015-12/rueckblick-2015-magnum-fs |