package kr.co.swh.lecture.opensource.jsoup.news;

import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.google.common.collect.Lists;

import scala.Tuple2;


public class NaverNewsCrawlSource 
{
	private String domain, startDate, endDate;
	private String[] category;
	private Boolean fastMode;
	
	private ICrawlService service;
	private final int maxContentsLength = 300000;
	public static LinkedList<String> execTailQueue = new LinkedList<String>();
	
	//	HTML TAG
	public final static String TAG_SCRIPT = "script";
	public final static String META_KEY_CONTENT = "content";
	public final static String TAG_A = "a";
	public final static String TAG_IMG = "img";
	public final static String TAG_HREF = "href";
	public final static String TAG_HTTP = "http";
	
	//	REGX
	public final static String REGX_META_CONTENTS_TITLE = "meta[property=og:title]";
	public final static String REGX_META_CONTENTS_REGDATE = "meta[property=og:regDate]";
	public final static String REGX_META_CONTENTS_PUBDATE = "meta[property=article:published_time]";
	
	//////////////////////////////////////////////////////
	//	NAVER
	//	CATEGORY
	//	정치, 경제, 사회, 문화, 세계, 과학, IT  (except. 연예, 스포츠, 기후)
	public final static String[] NAVER_CATEGORY = {"politics", "economy", "society", "culture", "world", "science", "info-tech", "sports", "entertainment", "weather"};
	public final static Map<String, Tuple2<String, String>> NAVER_CATEGORY_MAP = new HashMap<String, Tuple2<String, String>>();
	static{
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[0], new Tuple2<String, String>("sid1=100", "sid2=269"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[1], new Tuple2<String, String>("sid1=101", "sid2=263"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[2], new Tuple2<String, String>("sid1=102", "sid2=257"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[3], new Tuple2<String, String>("sid1=103", "sid2=245"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[4], new Tuple2<String, String>("sid1=104", "sid2=322"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[5], new Tuple2<String, String>("sid1=105", "sid2=228"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[6], new Tuple2<String, String>("sid1=105", "sid2=230"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[7], new Tuple2<String, String>("sid1=100", "sid2=269"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[8], new Tuple2<String, String>("sid1=100", "sid2=269"));
		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[9], new Tuple2<String, String>("sid1=100", "sid2=269"));
	}
	//	URL
	public final static String NAVER_URL_HEAD = "http://news.naver.com";
	public final static String NAVER_FETCH_URL_HEAD = NAVER_URL_HEAD + "/main/list.nhn?%s&%s&mid=shm&mode=LS2D&date=%s&page=%s";	//	sid1, sid2, date, paging
	
	//	NAVER REGX
	public final static String NAVER_REGX_FETCH_SUBURL = "a[href]";
	public final static String NAVER_REGX_FETCH_NEWS_ID = "(?<=(&oid=|\\?oid=)|(&aid=|\\?aid=))[\\d]+";
	
	public final static String NAVER_REGX_CONTENTS_REGDATE_GET = ".sponsor > span";
	public final static String NAVER_REGX_CONTENTS_GET = "#articleBodyContents";
	
	public void init() {
		domain = "naver";
		category = new String[]{"politics"};
		startDate = "20191117";
		endDate = "20191117";
		fastMode = false;
		service = new NaverCrawlManager(domain, startDate, endDate, category, fastMode);
		
		try {
			service.crawlling();
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
	}

	public static void main(String[] args) {
		
		NaverNewsCrawlSource source = new NaverNewsCrawlSource();
		source.init();
		System.out.println(NaverCrawlManager.totalDayFromCalendar(2019,10,20));
		System.out.println(NaverCrawlManager.endDayFromTotalDay(2019,10));
//		System.out.println(StringUtils.leftPad(String.valueOf(1), 2, "0"));
//		System.out.println(StringUtils.leftPad(String.valueOf(9), 5, "4"));
	}
	
	class NaverCrawlManager extends A_CrawlClient{
		
		public NaverCrawlManager(String domain, String startDate, String endDate, String[] category, boolean fastMode)
		{
			super(domain, startDate, endDate, category, fastMode);
		}
		
		@Override
		public void fetch(String sourceDate) throws Exception{
			List<String> fetchDateList = getFetchDateList(sourceDate);
			rootBreak:
				for(String crawlDate : fetchDateList){
					currentStartDate = crawlDate;
					for(String category : categories){
						for(int pageIndex = 1; pageIndex < Integer.MAX_VALUE; pageIndex++){
							//	shutdown
							if(articleBreakLatch.getCount() < 1) break rootBreak;
							try {
								
								List<String> findNewsDocList = getArticleURLHeader(pageIndex, category, crawlDate);
								int cnt = findNewsDocList.size();
								if(cnt <= 0) {
									break;
								}
								contentsCrawlling(findNewsDocList, category);
								
								Thread.sleep(sleepTime);
							} catch (IOException | InterruptedException e) {
								// TODO Auto-generated catch block
								throw e;
							}
						}
					}
					cacheNewsId.clear();
				}
		}
		
		@Override
		public void monitor(){
			new Thread(new Runnable(){
				@Override
				public void run() {
					// TODO Auto-generated method stub
					while(true){
						String currentSystemDate = currentDate();
						
						List<String> monitorArticleList = getStringFormatDateList(monitoringStartDate, currentSystemDate);
						for(String crawlDate : monitorArticleList){
							currentStartDate = crawlDate;
							for(String category : categories){
								int checkPageBlockNum = monitoringNewPageBlockNum;
								for(int pageIndex = 1; pageIndex < Integer.MAX_VALUE; pageIndex++){
									try {
										List<String> findNewsDocList = getArticleURLHeader(pageIndex, category, crawlDate);
										if(findNewsDocList.size() <= 0) {
											checkPageBlockNum--;
										}
										else checkPageBlockNum = monitoringNewPageBlockNum;
										
										if(checkPageBlockNum <= 0) {
											checkPageBlockNum = monitoringNewPageBlockNum;
											break;
										}
										contentsCrawlling(findNewsDocList, category);
										
										Thread.sleep(sleepTime);
									} catch (IOException | InterruptedException e) {
										// TODO Auto-generated catch block
										articleBreakLatch.countDown();
									}
								}
							}
						}
						if(updateSystemDateMonitoringDate(currentSystemDate)) cacheNewsId.clear();
						if(!isContinueCrawl()) {
							articleBreakLatch.countDown();
						}
					}
				}
			}).start();
			
			try {
				articleBreakLatch.await();
			} catch (InterruptedException e) {
				// TODO Auto-generated catch block
			} finally {
			}
		}
		
		private List<String> getArticleURLHeader(int pageIndex, String category, String crawlDate) throws IOException{
			List<String> findNewsDocList = Lists.newArrayList();
			String page = String.valueOf(pageIndex);
			Tuple2<String, String> seedKey = NAVER_CATEGORY_MAP.get(category);
			String fetchURL = String.format(NAVER_FETCH_URL_HEAD, seedKey._1(), seedKey._2(), crawlDate, page);
			Document fetchData = Jsoup.connect(fetchURL).timeout(0).get();
			Elements fetchTmpDatas = fetchData.select(NAVER_REGX_FETCH_SUBURL);
			for(Element doc : fetchTmpDatas){
				if(!doc.select(TAG_A).is(TAG_IMG) && 
						doc.attr(TAG_HREF).startsWith(TAG_HTTP) &&
						doc.attr(TAG_HREF).contains(seedKey._1()) &&
						doc.attr(TAG_HREF).contains(seedKey._2())){
					
					//	네이버에서는 페이지 카운트에 대한 종료를 위해서는 페이지의 주소 페이지 주소로 종료
					if(cacheNewsId.contains(doc.attr(TAG_HREF))) continue;
					cacheNewsId.add(doc.attr(TAG_HREF));
					findNewsDocList.add(doc.attr(TAG_HREF));
				}
			}
			return findNewsDocList;
		}
		
		//	CONTENTS GET
		private void contentsCrawlling(List<String> findNewsDocList, String category) throws IOException{
			int retry = 3;
			for(int index = 0; index < findNewsDocList.size(); index++){
				String newsDocUrl = findNewsDocList.get(index);
				Document crawlDoc;
				try {
					crawlDoc = Jsoup.connect(newsDocUrl).timeout(0).get();
				} catch (IOException e) {
					// TODO Auto-generated catch block
					if(retry < 0) throw e;
					index--;
					continue;
				}

				//	newsId
				String newsId = "";
				Pattern pattern = Pattern.compile(NAVER_REGX_FETCH_NEWS_ID, Pattern.CASE_INSENSITIVE);
				Matcher m = pattern.matcher(crawlDoc.baseUri());
				int findIndex = 0;
				while(m.find(findIndex)){
					newsId += crawlDoc.baseUri().substring(m.start(), m.end());
					newsId += "-";
					findIndex = m.end();
				}
				newsId = newsId.substring(0, newsId.length()-1);
				
				//	title
				String title = "";
				Elements metaOgTitle = crawlDoc.select(REGX_META_CONTENTS_TITLE);
				if (metaOgTitle!=null) {
					title = metaOgTitle.attr(META_KEY_CONTENT);
				}
				
				//	regDate
				String regDate = "";
				Elements metaOgRegDate = crawlDoc.select(NAVER_REGX_CONTENTS_REGDATE_GET);
				if (metaOgRegDate != null) {
					regDate = metaOgRegDate.text();
				}

				StringBuilder contentsBuilder = new StringBuilder();
				Elements articleBodyContentsRows = crawlDoc.select(NAVER_REGX_CONTENTS_GET);
				String contents;
				int docContentsLength = 0;
				boolean contentsStatus = false;
				loop:
				for (Element articleBodyContentsRow : articleBodyContentsRows) {
					Iterator<Element> iterElem = articleBodyContentsRow.getAllElements().iterator();
					while(iterElem.hasNext()){
						String text = iterElem.next().text();
						docContentsLength += text.length();
						contentsBuilder.append(text);
						if(docContentsLength > maxContentsLength){
							contentsStatus = true;
							break loop;
						}
					}
				}
				if(contentsStatus) contents = contentsBuilder.toString().substring(0, maxContentsLength);
				else contents = contentsBuilder.toString();
				

				ArticleCrawlerModel article = new ArticleCrawlerModel();
				article.setId(newsId);
				article.setCategory(category);
				article.setUrl(crawlDoc.baseUri());
				article.setTitle(title);
				article.setRegDate(regDate);
				article.setContents(contents);
				System.out.println(article);
			}
		}
		
		@Override
		public void shutDown(){
			super.shutDown();
		}
		
	}
}

