[SYNC-218][BE] Refactor crawlers

.gitignore

@@ -1,4 +1,5 @@
 __pycache__/
 database_config.ini
 error_test.log
-test/
+test/
+env/

crawler_settings.ini

@@ -1,15 +1,15 @@
 [freq]
 # crawl frequencies (in minutes)
-chinatimes      = 100
-cna             = 100
-cts             = 100
-ebc             = 0
-ettoday         = 0
-factcheckcenter = 100
-ltn             = 100
-storm           = 100
-udn             = 4
-setn            = 100
+chinatimes      = 20
+cna             = 20
+cts             = 20
+ebc             = 20
+ettoday         = 20
+factcheckcenter = 20
+ltn             = 20
+storm           = 20
+udn             = 20
+setn            = 20
 
 [number]
 chinatimes      = 30

news/chinatimes_crawler.py

@@ -1,32 +1,34 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
 
 def chinatimes_crawler(size=30):
 
     media = '中時'
     article_list = []
 
-    #retrieve news list for first five pages
-    links = ['https://www.chinatimes.com/realtimenews/','https://www.chinatimes.com/realtimenews/?page=2',
+    # get news list for first five pages
+    newslist_page = ['https://www.chinatimes.com/realtimenews/','https://www.chinatimes.com/realtimenews/?page=2',
             'https://www.chinatimes.com/realtimenews/?page=3','https://www.chinatimes.com/realtimenews/?page=4',
             'https://www.chinatimes.com/realtimenews/?page=5']
     urls = []
-    for link in links:
-        soup = get_page(link)
-        sel = soup.find_all('div', class_='articlebox-compact')
+    for page in newslist_page:
+        try:
+            soup = get_page(page)
+            sel = soup.find_all('div', class_='articlebox-compact')
 
-        for s in sel:
-            u = s.find(class_='title').find('a')['href']
-            urls.append('https://chinatimes.com'+u)
+            for s in sel:
+                u = s.find(class_='title').find('a')['href']
+                urls.append('https://www.chinatimes.com'+u)
+        except Exception as error:
+            print("Get article url link error.")
+            print(page)
 
     news_count = 0
     for url in urls:
         try:
+
             soup = get_page(url)
 
             title = soup.find(class_='article-title').text

news/cna_crawler.py

@@ -1,18 +1,18 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
 
 def cna_crawler(size=30):
 
 	media = '中央社'
 	article_list = list()
-
-	soup = get_page('https://www.cna.com.tw/list/aall.aspx')
-	sel = soup.find('ul', 'mainList imgModule', id = 'jsMainList').find_all('li')
+	try:
+		soup = get_page('https://www.cna.com.tw/list/aall.aspx')
+		sel = soup.find('ul', 'mainList imgModule', id = 'jsMainList').find_all('li')
+	except Exception as error:
+		print("URL list fetch error")
+		return article_list
 
 	#add each url to url list
 	urls = []
@@ -33,11 +33,9 @@ def cna_crawler(size=30):
 			for s in sel:
 				article_content.append(s.text)
 				content_str += s.text
-
-			modified_date = soup.find('meta',itemprop='dateModified')['content']
-			modified_date = datetime.datetime.strptime(modified_date, "%Y/%m/%d %H:%M")
+			modified_date = soup.find('meta', attrs={"property": "article:modified_time"})['content']
+			modified_date = datetime.datetime.strptime(modified_date, "%Y-%m-%dT%H:%M:%S%z")
 			modified_date = utilities.convert_to_utc(modified_date)
-
 			url_hash = generate_hash(url)
 			content_hash = generate_hash(content_str)
 
@@ -66,4 +64,7 @@ def cna_crawler(size=30):
 			print(e)
 			continue
 
-	return article_list
+	return article_list
+
+if __name__ == "__main__":
+	print(cna_crawler(30))

news/cts_crawler.py

@@ -1,10 +1,6 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time
-import hashlib
 import datetime
 
 def cts_crawler(size=30):

news/ebc_crawler.py

@@ -1,10 +1,7 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
 
 def ebc_crawler(size=30):
 

news/ettoday_crawler.py

@@ -1,50 +1,29 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
-from selenium import webdriver
-from selenium.webdriver.chrome.options import Options
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
+import feedparser
 
 def ettoday_crawler(size=30):
 
-	base = "https://www.ettoday.net"
-
 	media = 'ettoday'
 	article_list = list()
 
-	#initiate chrome webdriver
-
-	options = Options()
-	options.add_argument("--disable-notifications")
-	options.headless = True
-
-	driver = webdriver.Chrome('chromedriver', options=options)
-	driver.get("https://www.ettoday.net/news/news-list.htm")
-	for _ in range(1,3):
-		driver.execute_script("window.scrollTo(0,document.body.scrollHeight)")
-		time.sleep(3)
-
-	soup = BeautifulSoup(driver.page_source, 'html.parser')
-	sel = soup.find('div', 'part_list_2').find_all('h3')
 	article_count = 0
+	# get newslist from rss feed
+	NewsFeed = feedparser.parse("https://feeds.feedburner.com/ettoday/realtime")
+	news_link_list = [ entry['link'][:-9] for entry in NewsFeed.entries ] # remove "fromrss" from link
 
-	#exit selenium
-	driver.quit()
-
-	for s in sel:
-		modified_date = s.find('span').text
-		modified_date = datetime.datetime.strptime(modified_date, "%Y/%m/%d %H:%M")
-		modified_date = utilities.convert_to_utc(modified_date)
-
-		category = s.find('em').text
-		url = base + s.find('a')['href']
+	for url in news_link_list:
 		try:
 			soup = get_page(url)
 			title = soup.find('h1', 'title').text
+			category = soup.find(class_=['menu_bread_crumb', 'part_breadcrumb']).find_all('div')[1].text.strip()
 
+			modified_date = soup.find('time', attrs={'itemprop': 'datePublished'})['datetime']
+			modified_date = datetime.datetime.fromisoformat(modified_date)
+			modified_date = utilities.convert_to_utc(modified_date)
+
 			article_content = []
 			content_str = ""
 			content_str += title
@@ -77,7 +56,6 @@ def ettoday_crawler(size=30):
 			news_dict['url_hash'] = url_hash
 			news_dict['content_hash'] = content_hash
 
-			#print(news_dict)
 			article_list.append(news_dict)
 
 			article_count+=1

news/factcheckcenter_crawler.py

@@ -1,10 +1,7 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
 
 def factcheckcenter_crawler(size=10):
 
@@ -23,8 +20,8 @@ def factcheckcenter_crawler(size=10):
 
 	article_count = 0
 	for url in urls:
-		soup = get_page(url)
 		try:
+			soup = get_page(url)
 			category = soup.find('div','field-name-field-taxo-report-attr').text.strip()
 			title = soup.find('h2', 'node-title').text
 			para = soup.find('div', 'field field-name-body field-type-text-with-summary field-label-hidden').find_all(['p','h2']) #get headings and content

news/ltn_crawler.py

@@ -1,23 +1,20 @@
 # -!- coding: utf-8 -!-
 import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import json
-import time,datetime
-import hashlib
+import datetime
 
 url_title = ["https://news.ltn.com.tw/ajax/breakingnews/all/1", "https://news.ltn.com.tw/ajax/breakingnews/all/2", "https://news.ltn.com.tw/ajax/breakingnews/all/3", "https://news.ltn.com.tw/ajax/breakingnews/all/4", "https://news.ltn.com.tw/ajax/breakingnews/all/5"]
 headers =  {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.130 Safari/537.36'}
 
 def get_one_page(url):
-    try:
-        response = requests.get(url,headers=headers)
-        if response.status_code == 200:
-        	return response.json()
-        return None
-    except :
-        print("page fetch fail")
+	try:
+		response = requests.get(url,headers=headers)
+		if response.status_code == 200:
+			return response.json()
+		return None
+	except:
+		print("page fetch fail")
 
 def parse_data(urls):
 	first = True
@@ -118,7 +115,6 @@ def ltn_crawler(size=30):
 			news_dict['url_hash'] = url_hash
 			news_dict['content_hash'] = content_hash
 
-			#print(news_dict)
 			article_list.append(news_dict)
 
 			article_count+=1

news/setn_crawler.py

@@ -1,10 +1,7 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
 import utilities
-import time,datetime
-import hashlib
+import datetime
 
 def setn_crawler(size=30):
 
@@ -30,8 +27,8 @@ def setn_crawler(size=30):
 	article_count = 0
 
 	for url in urls:
-		soup = get_page(url)
 		try:
+			soup = get_page(url)
 			category = soup.find('meta',attrs={'property':'article:section'})['content']
 			title = soup.find('h1').text
 			content_sel = soup.find('article').find_all('p')

news/storm_crawler.py

@@ -1,10 +1,7 @@
 # -!- coding: utf-8 -!-
-import requests
-from bs4 import BeautifulSoup
 from utilities import get_page,generate_hash
-import time,datetime
+import datetime
 import utilities
-import hashlib
 
 def storm_crawler(size=30):
 
@@ -18,11 +15,18 @@ def storm_crawler(size=30):
 			"https://www.storm.mg/articles/5","https://www.storm.mg/articles/6"]
 
 	for link in links:
-		soup = get_page(link)
-		sel = soup.find_all('div', 'category_card card_thumbs_left')
-
-		for s in sel:
-			urls.append(s.find('a')['href'])
+		try:
+			soup = get_page(link)
+			sel = soup.find_all('div', 'category_card card_thumbs_left')
+			for s in sel:
+				urls.append(s.find('a')['href'])
+		except Exception as error:
+			utilities.log_info("URL list request error.")
+			print("風傳媒storm")
+			print(link)
+			print(error)
+			continue
+
 
 	article_count = 0
 	for url in urls: