퀴즈 크롤링 코드 분석

pom.xml

@@ -196,6 +196,24 @@
 		 
 		<!-- CLIENT SIDE -->
 		
+		<!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+		<dependency>
+		    <groupId>org.apache.commons</groupId>
+		    <artifactId>commons-lang3</artifactId>
+		    <version>3.9</version>
+		</dependency>
+		<dependency>
+			<groupId>org.scala-lang</groupId>
+			<artifactId>scala-library</artifactId>
+			<version>2.12.7</version>
+		</dependency>
+		
+		<dependency>
+		    <groupId>com.ibm.icu</groupId>
+			<artifactId>icu4j</artifactId>
+			<version>63.1</version>
+		</dependency>
+		
 	</dependencies>
 	
 	<build>

src/main/java/kr/co/swh/lecture/opensource/icu/HolidayUtils.java

@@ -0,0 +1,331 @@
+package kr.co.swh.lecture.opensource.icu;
+
+import java.text.ParseException;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedSet;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+import com.google.common.collect.Lists;
+import com.ibm.icu.util.ChineseCalendar;
+
+import scala.Tuple2;
+ 
+public class HolidayUtils {
+ 
+    public Map<String, String> solarHolidayMap = new HashMap<String, String>();
+    public Map<String, String> lunarHolidayMap = new HashMap<String, String>();
+ 
+    public String getDateByString(Date date) {
+        return getDateByString(date, "-");
+    }
+ 
+    public String getDateByString(Date date, String separator) {
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyy"+separator+"MM"+separator+"dd");
+        return sdf.format(date);
+    }
+ 
+    /**
+     * 양력날짜를 음력날짜로 변환
+     * @param 양력날짜 (yyyyMMdd)
+     * @return 음력날짜 (yyyyMMdd)
+     */
+//    private String converSolarToLunar(String date) {
+//        return converSolarToLunar(date, "-");
+//    }
+ 
+    private String converSolarToLunar(String date, String separator) {
+        ChineseCalendar cc = new ChineseCalendar();
+        Calendar cal = Calendar.getInstance();
+ 
+        cal.set(Calendar.YEAR, Integer.parseInt(date.substring(0, 4)));
+        cal.set(Calendar.MONTH, Integer.parseInt(date.substring(4, 6)) - 1);
+        cal.set(Calendar.DAY_OF_MONTH, Integer.parseInt(date.substring(6)));
+ 
+        cc.setTimeInMillis(cal.getTimeInMillis());
+ 
+        int y = cc.get(ChineseCalendar.EXTENDED_YEAR) - 2637;
+        int m = cc.get(ChineseCalendar.MONTH) + 1;
+        int d = cc.get(ChineseCalendar.DAY_OF_MONTH);
+ 
+        StringBuffer ret = new StringBuffer();
+        ret.append(String.format("%04d", y)).append(separator);
+        ret.append(String.format("%02d", m)).append(separator);
+        ret.append(String.format("%02d", d));
+ 
+        return ret.toString();
+    } // end converSolarToLunar
+ 
+    public String getDay(String date, int amount) {
+        Calendar cal = Calendar.getInstance();
+        cal.set(Integer.parseInt(date.substring(0, 4)), Integer.parseInt(date.substring(4, 6)) - 1, Integer.parseInt(date.substring(6)));
+        cal.add(Calendar.DAY_OF_MONTH, amount);
+ 
+        return getDateByString(cal.getTime(), "");
+    }
+    
+    /**
+     * 해당일자가 대체공휴일에 해당하는 지 확인
+     * @param 양력날짜 (yyyyMMdd)
+     * @return 대체 공휴일이면 true
+     */
+    private boolean isHolidayAlternate(String date) {
+        
+        String[] altHoliday = new String[] {
+                "20150929", "20160210", "20170130", "20180926", 
+                "20180507", "20190506", "20200127", "20220912", 
+                "20230124", "20240212", "20240506", "20251008", 
+                "20270209", "20290924", "20290507"}; 
+        
+        return Arrays.asList(altHoliday).contains(date); 
+        
+        /*
+        int year = Integer.parseInt(date.substring(0, 4));
+        SimpleDateFormat sdf = new SimpleDateFormat("yyyyMMdd");
+        
+        // 설날
+        String dayFirst2 = convertLunarToSolar(year + "0101");
+        String dayFirst3 = convertLunarToSolar(year + "0102");
+        String dayFirst1 = String.valueOf(Integer.parseInt(dayFirst2) - 1);     
+        
+        // 추석
+        String dayThanks1 = convertLunarToSolar(year + "0814");
+        String dayThanks2 = convertLunarToSolar(year + "0815");
+        String dayThanks3 = convertLunarToSolar(year + "0816");
+        
+        // 어린이날
+        String dayChild = year + "0505";
+        
+        // 해당 년도의 대체휴일 목록
+        List<String> altHolyday = new ArrayList<String>();
+        
+        if(getDayOfWeek(dayFirst1) == Calendar.SUNDAY || getDayOfWeek(dayFirst2) == Calendar.SUNDAY || getDayOfWeek(dayFirst3) == Calendar.SUNDAY || isHolidaySolar(dayFirst1) || isHolidaySolar(dayFirst2) || isHolidaySolar(dayFirst3)) {
+            int y = Integer.parseInt(dayFirst3.substring(0, 4));
+            int m = Integer.parseInt(dayFirst3.substring(4, 6)) - 1;
+            int d = Integer.parseInt(dayFirst3.substring(6)) + 1;
+            Calendar c = Calendar.getInstance();
+            c.set(y, m, d);
+            altHolyday.add(sdf.format(c.getTime()));
+        }
+            
+        if(getDayOfWeek(dayThanks1) == Calendar.SUNDAY || getDayOfWeek(dayThanks2) == Calendar.SUNDAY || getDayOfWeek(dayThanks3) == Calendar.SUNDAY || isHolidaySolar(dayThanks1) || isHolidaySolar(dayThanks2) || isHolidaySolar(dayThanks3)) {
+            int y = Integer.parseInt(dayThanks3.substring(0, 4));
+            int m = Integer.parseInt(dayThanks3.substring(4, 6)) - 1;
+            int d = Integer.parseInt(dayThanks3.substring(6)) + 1;
+            Calendar c = Calendar.getInstance();
+            c.set(y, m, d);
+            altHolyday.add(sdf.format(c.getTime()));
+        }
+        
+        int childWeek = getDayOfWeek(dayChild); 
+        
+        if(childWeek == Calendar.SATURDAY) {
+            int y = Integer.parseInt(dayChild.substring(0, 4));
+            int m = Integer.parseInt(dayChild.substring(4, 6)) - 1;
+            int d = Integer.parseInt(dayChild.substring(6)) + 2;
+            Calendar c = Calendar.getInstance();
+            c.set(y, m, d);
+            altHolyday.add(sdf.format(c.getTime()));
+        }
+            
+        if(childWeek == Calendar.SUNDAY) {
+            int y = Integer.parseInt(dayChild.substring(0, 4));
+            int m = Integer.parseInt(dayChild.substring(4, 6)) - 1;
+            int d = Integer.parseInt(dayChild.substring(6)) + 1;
+            Calendar c = Calendar.getInstance();
+            c.set(y, m, d);
+            altHolyday.add(sdf.format(c.getTime()));
+        }
+        
+        return altHolyday.contains(date); 
+        */
+    }
+    
+    public HashMap<String, Tuple2<String, String>> init() {
+    	HashMap<String, Tuple2<String, String>> holidays = new HashMap<String, Tuple2<String, String>>();	//	선생님이 추가한 코드
+    	SortedSet<String> sortHoliday = new TreeSet<String>();
+    	solarHolidayMap.put("0101", "신정"); solarHolidayMap.put("0301", "삼일절");
+        solarHolidayMap.put("0505", "어린이날"); solarHolidayMap.put("0606", "현충일");
+        solarHolidayMap.put("0815", "광복절"); solarHolidayMap.put("1003", "개천절");
+        solarHolidayMap.put("1009", "한글날"); solarHolidayMap.put("1225", "성탄절");
+ 
+        lunarHolidayMap.put("0101", "구정"); lunarHolidayMap.put("0102", "구정");
+        lunarHolidayMap.put("0408", "석가탄신일"); lunarHolidayMap.put("0814", "추석");
+        lunarHolidayMap.put("0815", "추석"); lunarHolidayMap.put("0816", "추석");
+ 
+        int year = 2015;
+        int endYear = 2030;
+ 
+        Calendar c = Calendar.getInstance();
+        c.set(year, 0, 1); // 1월 1일부터 시작
+        
+        String solarDate = "";
+        String lunarDate = "";
+        for(int i=year; i<=endYear;) {
+            solarDate = getDateByString(c.getTime(), "");
+            lunarDate = converSolarToLunar(solarDate, "");
+ 
+            c.add(Calendar.DAY_OF_MONTH, 1);
+ 
+            //	대체휴일 체크
+            if(isHolidayAlternate(solarDate)) {
+            	String day = solarDate.substring(0,4) + "-" + solarDate.substring(4,6) + "-" + solarDate.substring(6,8);
+                System.out.println(day + " ==> " + "대체공휴일");
+                holidays.put(day, new Tuple2<String, String>(day, "대체공휴일"));	//	선생님이 추가한 코드
+                sortHoliday.add(day);
+            }
+            
+            // 양력휴일 체크
+            String solarMmdd = solarDate.substring(4,8);
+            if(solarHolidayMap.containsKey(solarMmdd)) {
+            	String day = solarDate.substring(0,4) + "-" + solarDate.substring(4,6) + "-" + solarDate.substring(6,8);
+                System.out.println(day + " ==> " + solarHolidayMap.get(solarMmdd));
+                holidays.put(day, new Tuple2<String, String>(day, solarHolidayMap.get(solarMmdd)));	//	선생님이 추가한 코드
+                sortHoliday.add(day);
+            }
+ 
+            // 음력휴일 체크
+            String lunarMmdd = lunarDate.substring(4,8);
+            if(lunarHolidayMap.containsKey(lunarMmdd)) {
+                // 음력 12월은 마지막날이 29일, 30일 계속 번갈아가면서 있으므로
+                // 양력에서 하루를 빼준날이 구정시작하는 날짜이다.
+                if(lunarMmdd.equals("0101")) {
+                	String tmp = getDay(solarDate, -1);
+                	String day = tmp.substring(0,4) + "-" + tmp.substring(4,6) + "-" + tmp.substring(6,8);
+                	System.out.println(day + " ==> 구정");
+                	holidays.put(day, new Tuple2<String, String>(day, "구정"));	//	선생님이 추가한 코드
+                	sortHoliday.add(day);
+                }
+                String day = solarDate.substring(0,4) + "-" + solarDate.substring(4,6) + "-" + solarDate.substring(6,8);
+                System.out.println(day + " ==> " + lunarHolidayMap.get(lunarMmdd));
+                holidays.put(day, new Tuple2<String, String>(day, lunarHolidayMap.get(lunarMmdd)));	//	선생님이 추가한 코드
+                sortHoliday.add(day);
+            }
+            
+            year = c.get(Calendar.YEAR);
+            if(year != i) {
+                i++;
+                System.out.println("");
+            }
+            if(i > endYear) break;
+        } // end for_i
+        
+        
+        // 	Quiz. 연속된 휴일을 시작과 종료로 연결
+        
+        //	holidays에 있는 값 출력
+        Iterator<String> holidayKeys = holidays.keySet().iterator();
+        while(holidayKeys.hasNext()) {
+        	String key = holidayKeys.next();
+        	Tuple2<String, String> tuple = holidays.get(key);
+        	System.out.println(key + " ~ " + tuple._1() + " ==> "  + tuple._2());
+        }
+        return holidays;
+    }
+    
+    public static ArrayList<String> getCurrentWeek(String day) throws ParseException {
+    	ArrayList<String> list = new ArrayList<String>();
+		SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd");
+		Date date = format.parse(day);
+		
+		Calendar calendar = Calendar.getInstance();
+		calendar.setTime(date);
+		int dayOfWeekDay = calendar.get(Calendar.DAY_OF_WEEK) - calendar.getFirstDayOfWeek();
+		calendar.add(Calendar.DAY_OF_MONTH, -dayOfWeekDay);
+		
+		String[] days = {"일","월","화","수","목","금","토"};
+		for (int i = 0; i < days.length; i++) {
+			list.add(format.format(calendar.getTime()));
+			calendar.add(Calendar.DAY_OF_MONTH, 1);
+		}
+		return list;
+    }
+    
+    //	종료날짜는 포함하지 않는다.
+    public static List<String> getDateListEndDateNotInclude(String startDate, String endDate) throws ParseException{
+    	SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd");
+    	Calendar calendar = Calendar.getInstance();
+    	calendar.clear();
+    	List<String> holidayList = Lists.newArrayList();
+    	
+    	Date sDate = simpleDateFormat.parse(startDate);
+    	Date eDate = simpleDateFormat.parse(endDate);
+
+    	//	두 날짜의 차이를 계산
+    	int count = (int)( (eDate.getTime() - sDate.getTime()) / (1000 * 60 * 60 * 24));
+    	calendar.setTime(sDate);
+    	
+    	//	차이가 0 보다 큰 경우에는 시작날짜 추가
+    	if(count >= 0) holidayList.add(startDate);
+    	//	시작일과 종료일이 1보다 큰 경우에는 날짜를 모두 등록
+    	for(int dayCount = 1; dayCount <= count; dayCount++) {
+    		calendar.add(Calendar.DAY_OF_MONTH, 1);
+    		Date time = calendar.getTime();
+    		String day = simpleDateFormat.format(time);
+    		holidayList.add(day);
+    	}
+    	return holidayList;
+    }
+    
+    public static Tuple2<ArrayList<String>, Map<String, Integer>> getStartDayEndDay(List<String> holidays, String startDay, String endDay, String dayOfWeek) throws ParseException {
+    	
+		SimpleDateFormat originalFormat = new SimpleDateFormat("yyyy-MM-dd");
+		Date startDate = originalFormat.parse(startDay);
+		Date endDate = originalFormat.parse(endDay);		//	종료일짜를 출석일 제일 마지막 날짜로 초기화
+		Calendar calendar = Calendar.getInstance();
+		calendar.clear();
+		
+		//	종료 날짜는 출석일 제일 마지막 날짜의 다음 달 말일까지 계산
+		calendar.setTime(endDate);
+		calendar.add(Calendar.MONTH, 1);
+		int monthEndDay = calendar.getActualMaximum(Calendar.DAY_OF_MONTH);
+		calendar.set(Calendar.DATE, monthEndDay);
+		endDate = calendar.getTime();
+	
+		//	시작 날짜는 수강 시작일
+		calendar.setTime(startDate);
+//		시작 날짜가 포함된 주의 첫번째 날짜 계산
+//		calendar.setTime(startDate);
+//		int dayOfWeekDay = calendar.get(Calendar.DAY_OF_WEEK) - calendar.getFirstDayOfWeek();
+//		calendar.add(Calendar.DAY_OF_MONTH, -dayOfWeekDay);
+		
+		
+		//	두 날짜의 차이를 계산
+		int count = (int)( (endDate.getTime() - calendar.getTime().getTime()) / (1000 * 60 * 60 * 24));
+		
+		SimpleDateFormat mapFormat = new SimpleDateFormat("yyyy년 MM월");
+		Map<String, Integer> monthTotalCountMap = new TreeMap<String, Integer>();
+		ArrayList<String> subjectDays = new ArrayList<String>();
+		for (int i = 0; i <= count; i++) {
+			Date time = calendar.getTime();
+			String day = originalFormat.format(time);
+			
+			//	과정의 수강요일만 포함하며 휴강일은 카운트에서 제외
+			if(dayOfWeek != null && dayOfWeek.charAt(calendar.get(Calendar.DAY_OF_WEEK)-1) == '1' &&
+					!holidays.contains(day)){
+				
+				//	해당 월에 대한 총 카운트를 저장
+				String monthMap = mapFormat.format(time);
+				int monthMapKey = (monthTotalCountMap.containsKey(monthMap)) ? monthTotalCountMap.get(monthMap)+1 : 1;
+				monthTotalCountMap.put(monthMap, monthMapKey);
+				subjectDays.add(day);
+			}
+			calendar.add(Calendar.DAY_OF_MONTH, 1);
+		}
+		
+		return new Tuple2<ArrayList<String>, Map<String,Integer>>(subjectDays, monthTotalCountMap);
+    }
+    
+    public static void main(String[] args) {
+    	HolidayUtils holiday = new HolidayUtils();
+    	holiday.init();
+    }
+}

src/main/java/kr/co/swh/lecture/opensource/jsoup/news/A_CrawlClient.java

@@ -0,0 +1,237 @@
+package kr.co.swh.lecture.opensource.jsoup.news;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.lang3.StringUtils;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.google.common.collect.Lists;
+
+
+public abstract class A_CrawlClient implements ICrawlService{
+
+	protected transient ObjectMapper mapper;
+	protected transient CountDownLatch articleBreakLatch;
+	
+	protected String domainName, startDate, endDate;
+	protected String[] categories;
+	protected Integer sleepTime;
+	
+	protected List<String> cacheNewsId;
+	protected final String autoModeTag = "~";
+	protected boolean autoMonitor;
+	protected boolean continueCrawl;
+	protected String monitoringStartDate;					//	모니터링 시작 날짜
+	protected String currentStartDate;
+	
+	//	final value
+	protected final int monitoringNewPageBlockNum = 10;		//	모니터링 페이지 임계치
+	protected final int fastModeSleepTime = 0;				//	milisecond
+	protected final int safeModeSleepTime = 500;			//	milisecond
+	
+	//	pause
+	protected String lastProcessId;
+	
+	
+	public A_CrawlClient(String domain, String startDate, String endDate, String[] category, boolean fastMode)
+	{
+		this.mapper = new ObjectMapper();
+		this.articleBreakLatch = new CountDownLatch(1);
+		
+		this.domainName = domain.toLowerCase();
+		this.startDate = startDate;
+		this.endDate = endDate;
+		this.categories = category;
+		this.currentStartDate = startDate;
+		
+		this.monitoringStartDate = currentDate();
+		this.cacheNewsId = Lists.newArrayList();
+		
+		if(fastMode) sleepTime = fastModeSleepTime;
+		else sleepTime = safeModeSleepTime;
+		
+		//	현 시스템 날짜와 종료날짜가 동일하다면, 하루전 종료일짜까지 수집한 후, 현 시스템 날짜는 모니터링을 통해 실시간으로 수집, 날짜가 지나면 종료
+		//	'~' 표시인 경우 계속 수집
+		if(endDate.equals(this.autoModeTag)) {
+			this.continueCrawl = true;
+		}
+
+		//	종료날짜가 현 시스템 날짜보다 같거나 크다면, 현 시스템 날짜로 종료날짜를 등록하고 모니터링
+		if(this.continueCrawl || compareToDates(monitoringStartDate, endDate) >= 0) {
+			this.autoMonitor = true;
+		}
+	}
+	
+	
+	@Override
+	public void crawlling() throws Exception{
+		//	최초 스타트 일짜로 수행
+		fetch(startDate);
+		
+		//	장시간 기사를 수집할 시, 패치날짜의 간격이 커짐으로써, 모니터링 대상 날짜가 늘어나는 것을 방지하기 위해 한번 더 패치를 수행
+		String oldMonitoringStartDate = monitoringStartDate;
+		String currentSystemDate = currentDate();
+		if(updateSystemDateMonitoringDate(currentSystemDate)){
+			fetch(oldMonitoringStartDate);
+		}
+		
+		//	모니터링 모드일 경우 수행
+		if(autoMonitor) monitor();
+	}
+	
+	protected String currentDate(){
+		return new SimpleDateFormat("yyyyMMdd").format(new Date(System.currentTimeMillis()));
+	}
+	
+	protected boolean updateSystemDateMonitoringDate(String systemDate){
+		//	수집 시작한 날짜와 현 시스템의 날짜가 다른 경우,
+		if(!monitoringStartDate.equals(systemDate)) {
+			//	시스템 수집날짜를 현 시간으로 설정하여, 모니터링 대상 날짜로 지정
+			monitoringStartDate = systemDate;
+			return true;
+		}
+		return false;
+	}
+	
+	protected boolean isContinueCrawl(){
+		if(continueCrawl) return true;
+		else{
+			//	모니터링 날짜가 종료날짜보다 커지는 경우는 돌면 안됨. 
+			if(compareToDates(endDate, monitoringStartDate) > 0) return false;
+			else return true;
+		}
+	}
+	
+	protected List<String> getFetchDateList(String sourceDate){
+		List<String> fetchDateList = null;
+		
+		//	패치대상 날짜들은 현 시스템 날짜보다 하루 전 까지만 수행(모니터링 대상 날짜로부터)
+		if(autoMonitor) {
+			fetchDateList = getStringFormatDateList(sourceDate, monitoringStartDate);
+			fetchDateList.remove(fetchDateList.size()-1);
+		}
+		//	모니터링이 필요없는 경우
+		else fetchDateList = getStringFormatDateList(sourceDate, endDate);
+		return fetchDateList;
+	}
+	
+	protected List<String> getStringFormatDateList(String sourceDate, String targetDate){
+		List<String> fetchDateList = new ArrayList<String>();
+		int startYear = Integer.parseInt(sourceDate.substring(0, 4));
+		int startMonth = Integer.parseInt(sourceDate.substring(4, 6));
+		int startDay = Integer.parseInt(sourceDate.substring(6, 8));
+		int endYear = Integer.parseInt(targetDate.substring(0, 4));
+		int endMonth = Integer.parseInt(targetDate.substring(4, 6));
+		int endDay = Integer.parseInt(targetDate.substring(6, 8));
+		
+		int startTotalDay = totalDayFromCalendar(startYear, startMonth, startDay);
+		int endTotalDay = totalDayFromCalendar(endYear, endMonth, endDay);
+		
+		for(; startTotalDay <= endTotalDay; startTotalDay++){
+			int startEndDay = endDayFromTotalDay(startYear, startMonth);
+			if(startDay > startEndDay) {
+				startMonth++;
+				startDay = 1;
+			}
+			if(startMonth > 12){
+				startYear++;
+				startMonth = 1;
+				startDay = 1;
+			}
+			String fetchDate = startYear + StringUtils.leftPad(String.valueOf(startMonth), 2, "0") + StringUtils.leftPad(String.valueOf(startDay), 2, "0");
+			fetchDateList.add(fetchDate);
+			startDay++;
+		}
+		return fetchDateList;
+	}
+	
+	private int compareToDates(String sourceDate, String targetDate){
+		int startYear = Integer.parseInt(sourceDate.substring(0, 4));
+		int startMonth = Integer.parseInt(sourceDate.substring(4, 6));
+		int startDay = Integer.parseInt(sourceDate.substring(6, 8));
+		int endYear = Integer.parseInt(targetDate.substring(0, 4));
+		int endMonth = Integer.parseInt(targetDate.substring(4, 6));
+		int endDay = Integer.parseInt(targetDate.substring(6, 8));
+		
+		int startTotalDay = totalDayFromCalendar(startYear, startMonth, startDay);
+		int endTotalDay = totalDayFromCalendar(endYear, endMonth, endDay);
+		
+		return endTotalDay - startTotalDay;
+	}
+	
+	public static int totalDayFromCalendar(int year, int month, int day){
+		int totaldays;
+		totaldays = 365 * (year - 1);
+		for (int i = 1; i < year; i++) {
+			if (i % 4 == 0 && i % 100 != 0 || i % 400 == 0)
+				totaldays++;
+		}
+		// totaldays = 365 * (year-1) + (year-1)/4 - (year-1)/100 + (year-1)/400
+		int premonth = month - 1;
+		if (premonth >= 1)
+			totaldays += 31;
+		if (premonth >= 2)
+			totaldays += 28;
+		if (premonth >= 3)
+			totaldays += 31;
+		if (premonth >= 4)
+			totaldays += 30;
+		if (premonth >= 5)
+			totaldays += 31;
+		if (premonth >= 6)
+			totaldays += 30;
+		if (premonth >= 7)
+			totaldays += 31;
+		if (premonth >= 8)
+			totaldays += 31;
+		if (premonth >= 9)
+			totaldays += 30;
+		if (premonth >= 10)
+			totaldays += 31;
+		if (premonth >= 11)
+			totaldays += 30;
+		if (month > 2 && (year % 4 == 0 && year % 100 != 0 || year % 400 == 0))
+			totaldays++;
+		totaldays++;
+
+//		int day = totaldays % 7;
+		
+		totaldays = totaldays + day;
+		return totaldays;
+	}
+	
+	public static int endDayFromTotalDay(int year, int month){
+		int lastday;
+		switch (month) {
+		case 2:
+			lastday = 28;
+			if (year % 4 == 0 && year % 100 != 0 || year % 400 == 0)
+				lastday = 29;
+			else
+				lastday = 28;
+			break;
+		case 4:
+		case 6:
+		case 9:
+		case 11:
+			lastday = 30;
+			break;
+		default:
+			lastday = 31;
+		}
+		return lastday;
+	}
+	
+	
+	public synchronized void shutDown(){
+		this.articleBreakLatch.countDown();
+	}
+	
+	
+	public abstract void fetch(String sourceDate) throws Exception;
+	public abstract void monitor();
+}

src/main/java/kr/co/swh/lecture/opensource/jsoup/news/ArticleCrawlerModel.java

@@ -0,0 +1,66 @@
+package kr.co.swh.lecture.opensource.jsoup.news;
+
+import java.io.Serializable;
+
+public class ArticleCrawlerModel implements Serializable{
+	/**
+	 * 
+	 */
+	private static final long serialVersionUID = 1L;
+	
+	private String id;
+	private String category;
+	private String url;
+	private String title;
+	private String regDate;
+	private String contents;
+	
+	
+	public String getId() {
+		return id;
+	}
+	public void setId(String id) {
+		this.id = id;
+	}
+	public String getCategory() {
+		return category;
+	}
+	public void setCategory(String category) {
+		this.category = category;
+	}
+	public String getUrl() {
+		return url;
+	}
+	public void setUrl(String url) {
+		this.url = url;
+	}
+	public String getTitle() {
+		return title;
+	}
+	public void setTitle(String title) {
+		this.title = title;
+	}
+	public String getRegDate() {
+		return regDate;
+	}
+	public void setRegDate(String regDate) {
+		this.regDate = regDate;
+	}
+	public String getContents() {
+		return contents;
+	}
+	public void setContents(String contents) {
+		this.contents = contents;
+	}
+	@Override
+	public String toString() {
+		return "News{" +
+				"regDate=" + regDate +
+				", category=" + category +
+				", id=" + id +
+				", url=" + url +
+				", title=" + title +
+				", contents=" + contents +
+				'}';
+	}
+}

src/main/java/kr/co/swh/lecture/opensource/jsoup/news/ICrawlService.java

@@ -0,0 +1,8 @@
+package kr.co.swh.lecture.opensource.jsoup.news;
+
+public interface ICrawlService {
+	public void crawlling() throws Exception;
+	public void fetch(String sourceDate) throws Exception;
+	public void monitor();
+	public void shutDown();
+}

src/main/java/kr/co/swh/lecture/opensource/jsoup/news/NaverNewsCrawlSource.java

@@ -0,0 +1,288 @@
+package kr.co.swh.lecture.opensource.jsoup.news;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+import org.jsoup.nodes.Element;
+import org.jsoup.select.Elements;
+
+import com.google.common.collect.Lists;
+
+import scala.Tuple2;
+
+
+public class NaverNewsCrawlSource 
+{
+	private String domain, startDate, endDate;
+	private String[] category;
+	private Boolean fastMode;
+	
+	private ICrawlService service;
+	private final int maxContentsLength = 300000;
+	public static LinkedList<String> execTailQueue = new LinkedList<String>();
+	
+	//	HTML TAG
+	public final static String TAG_SCRIPT = "script";
+	public final static String META_KEY_CONTENT = "content";
+	public final static String TAG_A = "a";
+	public final static String TAG_IMG = "img";
+	public final static String TAG_HREF = "href";
+	public final static String TAG_HTTP = "http";
+	
+	//	REGX
+	public final static String REGX_META_CONTENTS_TITLE = "meta[property=og:title]";
+	public final static String REGX_META_CONTENTS_REGDATE = "meta[property=og:regDate]";
+	public final static String REGX_META_CONTENTS_PUBDATE = "meta[property=article:published_time]";
+	
+	//////////////////////////////////////////////////////
+	//	NAVER
+	//	CATEGORY
+	//	정치, 경제, 사회, 문화, 세계, 과학, IT  (except. 연예, 스포츠, 기후)
+	public final static String[] NAVER_CATEGORY = {"politics", "economy", "society", "culture", "world", "science", "info-tech", "sports", "entertainment", "weather"};
+	public final static Map<String, Tuple2<String, String>> NAVER_CATEGORY_MAP = new HashMap<String, Tuple2<String, String>>();
+	static{
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[0], new Tuple2<String, String>("sid1=100", "sid2=269"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[1], new Tuple2<String, String>("sid1=101", "sid2=263"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[2], new Tuple2<String, String>("sid1=102", "sid2=257"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[3], new Tuple2<String, String>("sid1=103", "sid2=245"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[4], new Tuple2<String, String>("sid1=104", "sid2=322"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[5], new Tuple2<String, String>("sid1=105", "sid2=228"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[6], new Tuple2<String, String>("sid1=105", "sid2=230"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[7], new Tuple2<String, String>("sid1=100", "sid2=269"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[8], new Tuple2<String, String>("sid1=100", "sid2=269"));
+		NAVER_CATEGORY_MAP.put(NAVER_CATEGORY[9], new Tuple2<String, String>("sid1=100", "sid2=269"));
+	}
+	//	URL
+	public final static String NAVER_URL_HEAD = "http://news.naver.com";
+	public final static String NAVER_FETCH_URL_HEAD = NAVER_URL_HEAD + "/main/list.nhn?%s&%s&mid=shm&mode=LS2D&date=%s&page=%s";	//	sid1, sid2, date, paging
+	
+	//	NAVER REGX
+	public final static String NAVER_REGX_FETCH_SUBURL = "a[href]";
+	public final static String NAVER_REGX_FETCH_NEWS_ID = "(?<=(&oid=|\\?oid=)|(&aid=|\\?aid=))[\\d]+";
+	
+	public final static String NAVER_REGX_CONTENTS_REGDATE_GET = ".sponsor > span";
+	public final static String NAVER_REGX_CONTENTS_GET = "#articleBodyContents";
+	
+	public void init() {
+		domain = "naver";
+		category = new String[]{"politics"};
+		startDate = "20191019";
+		endDate = "20191020";
+		fastMode = false;
+		service = new NaverCrawlManager(domain, startDate, endDate, category, fastMode);
+		
+		try {
+			service.crawlling();
+		} catch (Exception e) {
+			// TODO Auto-generated catch block
+			e.printStackTrace();
+		}
+	}
+
+	public static void main(String[] args) {
+		
+		NaverNewsCrawlSource source = new NaverNewsCrawlSource();
+		source.init();
+	}
+	
+	class NaverCrawlManager extends A_CrawlClient{
+		
+		public NaverCrawlManager(String domain, String startDate, String endDate, String[] category, boolean fastMode)
+		{
+			super(domain, startDate, endDate, category, fastMode);
+		}
+		
+		@Override
+		public void fetch(String sourceDate) throws Exception{
+			List<String> fetchDateList = getFetchDateList(sourceDate);
+			rootBreak:
+				for(String crawlDate : fetchDateList){
+					currentStartDate = crawlDate;
+					for(String category : categories){
+						for(int pageIndex = 1; pageIndex < Integer.MAX_VALUE; pageIndex++){
+							//	shutdown
+							if(articleBreakLatch.getCount() < 1) break rootBreak;
+							try {
+								
+								List<String> findNewsDocList = getArticleURLHeader(pageIndex, category, crawlDate);
+								int cnt = findNewsDocList.size();
+								if(cnt <= 0) {
+									break;
+								}
+								contentsCrawlling(findNewsDocList, category);
+								
+								Thread.sleep(sleepTime);
+							} catch (IOException | InterruptedException e) {
+								// TODO Auto-generated catch block
+								throw e;
+							}
+						}
+					}
+					cacheNewsId.clear();
+				}
+		}
+		
+		@Override
+		public void monitor(){
+			new Thread(new Runnable(){
+				@Override
+				public void run() {
+					// TODO Auto-generated method stub
+					while(true){
+						String currentSystemDate = currentDate();
+						
+						List<String> monitorArticleList = getStringFormatDateList(monitoringStartDate, currentSystemDate);
+						for(String crawlDate : monitorArticleList){
+							currentStartDate = crawlDate;
+							for(String category : categories){
+								int checkPageBlockNum = monitoringNewPageBlockNum;
+								for(int pageIndex = 1; pageIndex < Integer.MAX_VALUE; pageIndex++){
+									try {
+										List<String> findNewsDocList = getArticleURLHeader(pageIndex, category, crawlDate);
+										if(findNewsDocList.size() <= 0) {
+											checkPageBlockNum--;
+										}
+										else checkPageBlockNum = monitoringNewPageBlockNum;
+										
+										if(checkPageBlockNum <= 0) {
+											checkPageBlockNum = monitoringNewPageBlockNum;
+											break;
+										}
+										contentsCrawlling(findNewsDocList, category);
+										
+										Thread.sleep(sleepTime);
+									} catch (IOException | InterruptedException e) {
+										// TODO Auto-generated catch block
+										articleBreakLatch.countDown();
+									}
+								}
+							}
+						}
+						if(updateSystemDateMonitoringDate(currentSystemDate)) cacheNewsId.clear();
+						if(!isContinueCrawl()) {
+							articleBreakLatch.countDown();
+						}
+					}
+				}
+			}).start();
+			
+			try {
+				articleBreakLatch.await();
+			} catch (InterruptedException e) {
+				// TODO Auto-generated catch block
+			} finally {
+			}
+		}
+		
+		private List<String> getArticleURLHeader(int pageIndex, String category, String crawlDate) throws IOException{
+			List<String> findNewsDocList = Lists.newArrayList();
+			String page = String.valueOf(pageIndex);
+			Tuple2<String, String> seedKey = NAVER_CATEGORY_MAP.get(category);
+			String fetchURL = String.format(NAVER_FETCH_URL_HEAD, seedKey._1(), seedKey._2(), crawlDate, page);
+			Document fetchData = Jsoup.connect(fetchURL).timeout(0).get();
+			Elements fetchTmpDatas = fetchData.select(NAVER_REGX_FETCH_SUBURL);
+			for(Element doc : fetchTmpDatas){
+				if(!doc.select(TAG_A).is(TAG_IMG) && 
+						doc.attr(TAG_HREF).startsWith(TAG_HTTP) &&
+						doc.attr(TAG_HREF).contains(seedKey._1()) &&
+						doc.attr(TAG_HREF).contains(seedKey._2())){
+					
+					//	네이버에서는 페이지 카운트에 대한 종료를 위해서는 페이지의 주소 페이지 주소로 종료
+					if(cacheNewsId.contains(doc.attr(TAG_HREF))) continue;
+					cacheNewsId.add(doc.attr(TAG_HREF));
+					findNewsDocList.add(doc.attr(TAG_HREF));
+				}
+			}
+			return findNewsDocList;
+		}
+		
+		//	CONTENTS GET
+		private void contentsCrawlling(List<String> findNewsDocList, String category) throws IOException{
+			int retry = 3;
+			for(int index = 0; index < findNewsDocList.size(); index++){
+				String newsDocUrl = findNewsDocList.get(index);
+				Document crawlDoc;
+				try {
+					crawlDoc = Jsoup.connect(newsDocUrl).timeout(0).get();
+				} catch (IOException e) {
+					// TODO Auto-generated catch block
+					if(retry < 0) throw e;
+					index--;
+					continue;
+				}
+
+				//	newsId
+				String newsId = "";
+				Pattern pattern = Pattern.compile(NAVER_REGX_FETCH_NEWS_ID, Pattern.CASE_INSENSITIVE);
+				Matcher m = pattern.matcher(crawlDoc.baseUri());
+				int findIndex = 0;
+				while(m.find(findIndex)){
+					newsId += crawlDoc.baseUri().substring(m.start(), m.end());
+					newsId += "-";
+					findIndex = m.end();
+				}
+				newsId = newsId.substring(0, newsId.length()-1);
+				
+				//	title
+				String title = "";
+				Elements metaOgTitle = crawlDoc.select(REGX_META_CONTENTS_TITLE);
+				if (metaOgTitle!=null) {
+					title = metaOgTitle.attr(META_KEY_CONTENT);
+				}
+				
+				//	regDate
+				String regDate = "";
+				Elements metaOgRegDate = crawlDoc.select(NAVER_REGX_CONTENTS_REGDATE_GET);
+				if (metaOgRegDate != null) {
+					regDate = metaOgRegDate.text();
+				}
+
+				StringBuilder contentsBuilder = new StringBuilder();
+				Elements articleBodyContentsRows = crawlDoc.select(NAVER_REGX_CONTENTS_GET);
+				String contents;
+				int docContentsLength = 0;
+				boolean contentsStatus = false;
+				loop:
+				for (Element articleBodyContentsRow : articleBodyContentsRows) {
+					Iterator<Element> iterElem = articleBodyContentsRow.getAllElements().iterator();
+					while(iterElem.hasNext()){
+						String text = iterElem.next().text();
+						docContentsLength += text.length();
+						contentsBuilder.append(text);
+						if(docContentsLength > maxContentsLength){
+							contentsStatus = true;
+							break loop;
+						}
+					}
+				}
+				if(contentsStatus) contents = contentsBuilder.toString().substring(0, maxContentsLength);
+				else contents = contentsBuilder.toString();
+				
+
+				ArticleCrawlerModel article = new ArticleCrawlerModel();
+				article.setId(newsId);
+				article.setCategory(category);
+				article.setUrl(crawlDoc.baseUri());
+				article.setTitle(title);
+				article.setRegDate(regDate);
+				article.setContents(contents);
+				System.out.println(article);
+			}
+		}
+		
+		@Override
+		public void shutDown(){
+			super.shutDown();
+		}
+		
+	}
+}
+

src/main/resources/application.properties

@@ -0,0 +1,2 @@
+demo.value=test
+demo.type=dev