All sites in the IDG network have the same structure

7 files changed, 101 insertions, 35 deletions

diff --git a/cio.com.txt b/cio.com.txt
new file mode 100644
index 0000000..9898309
--- /dev/null
+++ b/cio.com.txt
@@ -0,0 +1,17 @@
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
+
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
+
+next_page_link: //a[@rel="next"]
+
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/computerworld.com.txt b/computerworld.com.txt
index 7f20a4d..9898309 100644
--- a/computerworld.com.txt
+++ b/computerworld.com.txt
@@ -1,22 +1,17 @@
-title: //meta[@name='headline']/@content
-date: //meta[@name='date']/@content
-author: //meta[@name='author']/@content
-body: //div[contains(@class, 'article')]
-body://div[@id="article_body"]
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
 
-strip_id_or_class: banner
-strip: //noscript
-strip: //div[@style='width:1px;height:130px;float:right;']
-strip: //div[@class='storyby']
-strip_image_src: twitter_icon
-strip_image_src: rss_bug
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
 
-tidy: no
-prune: no
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
 
-next_page_link://div[@id="next_page"]/a
+next_page_link: //a[@rel="next"]
 
-single_page_link: concat('http://www.computerworld.com/s/article/print/', substring-after(//link[@rel='canonical']/@href, '/s/article/'))
-
-test_url: http://www.computerworld.com/s/article/9224348/Apple_s_new_OS_X_tightens_screws_on_some_malware
-test_url: http://www.computerworld.com/s/article/9227679/Windows_8_Release_Preview_Updated_but_still_uneasy
\ No newline at end of file
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/csoonline.com.txt b/csoonline.com.txt
new file mode 100644
index 0000000..9898309
--- /dev/null
+++ b/csoonline.com.txt
@@ -0,0 +1,17 @@
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
+
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
+
+next_page_link: //a[@rel="next"]
+
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/infoworld.com.txt b/infoworld.com.txt
index d335bc4..9898309 100644
--- a/infoworld.com.txt
+++ b/infoworld.com.txt
@@ -1,12 +1,17 @@
-body: //div[@id='main_text']
-title: //div[@id='main_text']/h1
-strip: //div[@id='main_text']/h1
-strip: //div[@id='main_text']/h2
-strip_id_or_class: tools
-strip_id_or_class: articleTools
-strip_id_or_class: pagination
-strip_id_or_class: byline
-strip_id_or_class: tweet
-date: //div[@class='date']
-strip: //div[@class='date']
-test_url: http://www.infoworld.com/d/the-industry-standard/it-jobs-the-rise-both-offshore-and-in-us-187689
\ No newline at end of file
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
+
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
+
+next_page_link: //a[@rel="next"]
+
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/itworld.com.txt b/itworld.com.txt
index 1ee0ee5..9898309 100644
--- a/itworld.com.txt
+++ b/itworld.com.txt
@@ -1,5 +1,17 @@
-title: //*[@id="article-title"]
-author: //*[@id="article-info"]/strong
-date: //*[@class="article-dateline"]/strong
-body: //*[@id="article-content"]
-test_url: http://www.itworld.com/open-source/140916/android-sued-microsoft-not-linux
\ No newline at end of file
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
+
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
+
+next_page_link: //a[@rel="next"]
+
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/javaworld.com.txt b/javaworld.com.txt
new file mode 100644
index 0000000..9898309
--- /dev/null
+++ b/javaworld.com.txt
@@ -0,0 +1,17 @@
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
+author: //meta[@name="author"]/@content
+date: //meta[@name="DC.date.issued"]/@content
+
+body: //div[@itemprop="articleBody"]
+body: //div[@itemprop="reviewBody"]
+body: //figcaption|//div[@class="img-wrapper"]/noscript/img
+
+next_page_link: //a[@rel="next"]
+
+strip: //aside
+strip: //h3[contains(., "See also:")]
+strip: //div[@id="article-top-page-number"]
+strip: //p[starts-with(normalize-space(.), '[')]
+strip: //p[starts-with(normalize-space(.), '+')]
diff --git a/networkworld.com.txt b/networkworld.com.txt
index d692072..f097298 100644
--- a/networkworld.com.txt
+++ b/networkworld.com.txt
@@ -1,3 +1,6 @@
+# All sites of the IDG network can be extracted using the same rules,
+# make sure to update all of them
+
 author: //meta[@name="author"]/@content
 date: //meta[@name="DC.date.issued"]/@content