java, python twitter 수집

pom.xml

@@ -220,6 +220,21 @@
 			<version>2.9.0</version>
 		</dependency>
 		
+		<!-- https://mvnrepository.com/artifact/org.twitter4j/twitter4j-core -->
+		<dependency>
+		    <groupId>org.twitter4j</groupId>
+		    <artifactId>twitter4j-core</artifactId>
+		    <version>4.0.7</version>
+		</dependency>
+		
+		<!-- https://mvnrepository.com/artifact/org.twitter4j/twitter4j-stream -->
+		<dependency>
+		    <groupId>org.twitter4j</groupId>
+		    <artifactId>twitter4j-stream</artifactId>
+		    <version>4.0.7</version>
+		</dependency>
+		
+		
 		<!-- https://mvnrepository.com/artifact/com.rabbitmq/amqp-client -->
 		<dependency>
 		    <groupId>com.rabbitmq</groupId>

src/main/java/kr/co/swh/lecture/opensource/twitter/TwitterBasic.java

@@ -0,0 +1,96 @@
+package kr.co.swh.lecture.opensource.twitter; 
+
+import java.util.ArrayList;
+import java.util.List;
+
+import twitter4j.Query;
+import twitter4j.QueryResult;
+import twitter4j.Status;
+import twitter4j.Twitter;
+import twitter4j.TwitterException;
+import twitter4j.TwitterFactory;
+import twitter4j.conf.ConfigurationBuilder;
+
+/**
+ * <pre>
+ * kr.co.swh.lecture.opensource.twitter 
+ * TwitterBasic.java
+ *
+ * 설명 :	https://github.com/Twitter4J/Twitter4J
+ * </pre>
+ * 
+ * @since : 2019. 12. 13.
+ * @author : tobby48
+ * @version : v1.0
+ */
+public class TwitterBasic {
+
+	private Twitter getInstance() {
+		//	twitter 계정이 있다면 앱을 만들어서 등록
+		//	그렇지 않을 시에는 데이터베이스에 있는 정보 등록
+		String consumerKey = "xx";
+        String consumerSecret = "xx";
+        String acessToken = "xx";
+        String acessTokenSecret = "xx";
+
+        ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
+        configurationBuilder.setDebugEnabled(true)
+                .setOAuthConsumerKey(consumerKey)
+                .setOAuthConsumerSecret(consumerSecret)
+                .setOAuthAccessToken(acessToken)
+                .setOAuthAccessTokenSecret(acessTokenSecret);
+        TwitterFactory tf = new TwitterFactory(configurationBuilder.build());
+        Twitter twitter = tf.getInstance();
+        return twitter;
+	}
+	
+	public List<Status> getTwitterContents(Query query, int totalCount){
+		List<Status> tweetList = null;
+	    if(totalCount < 100){
+	        query.setCount(totalCount);
+	        tweetList = this.getTweetBlock(query);
+	    }else{
+	    	//	Quiz.
+	    	// 	따로 만들어야 함. 트위터는 한번에 100개씩 제한
+	    }
+		return tweetList;
+	}
+	
+	private List<Status> getTweetBlock(Query query){
+        if(query.getMaxId() != 0){
+            query.setMaxId(query.getMaxId());
+        }
+        List<Status> result = null;
+        QueryResult queryResult = null;
+        try{
+        	Twitter twitter = getInstance();
+            queryResult = twitter.search(query);
+        }catch (TwitterException e){
+            e.printStackTrace();
+        }
+
+        if(queryResult != null){
+            result = new ArrayList<>();
+            for (Status status : queryResult.getTweets()) {
+                result.add(status);
+            }
+        }
+        return result;
+    }
+
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+		TwitterBasic twitter = new TwitterBasic();
+        Query query = new Query();
+        query.setLang("ko");
+        query.setQuery("손흥민");
+        query.setSince("2019-09-28");
+        
+        List<Status> result = twitter.getTwitterContents(query, 90);
+        for(Status s : result) {
+        	System.out.println(s);
+        	//	VO 클래스로 객체형태로 변환하여 다양하게 활용
+        }
+	}
+
+}

src/main/java/kr/co/swh/lecture/opensource/twitter/TwitterStreaming.java

@@ -0,0 +1,84 @@
+package kr.co.swh.lecture.opensource.twitter; 
+
+import twitter4j.StallWarning;
+import twitter4j.Status;
+import twitter4j.StatusDeletionNotice;
+import twitter4j.StatusListener;
+import twitter4j.TwitterStream;
+import twitter4j.TwitterStreamFactory;
+import twitter4j.conf.ConfigurationBuilder;
+
+/**
+ * <pre>
+ * kr.co.swh.lecture.opensource.twitter 
+ * TwitterStreaming.java
+ *
+ * 설명 : https://github.com/Twitter4J/Twitter4J
+ * </pre>
+ * 
+ * @since : 2019. 12. 13.
+ * @author : tobby48
+ * @version : v1.0
+ */
+public class TwitterStreaming {
+
+	private TwitterStream getInstance() {
+		//	twitter 계정이 있다면 앱을 만들어서 등록
+		//	그렇지 않을 시에는 데이터베이스에 있는 정보 등록
+		String consumerKey = "xx";
+        String consumerSecret = "xx";
+        String acessToken = "xx";
+        String acessTokenSecret = "xx";
+
+        ConfigurationBuilder configurationBuilder = new ConfigurationBuilder();
+        configurationBuilder.setDebugEnabled(true)
+                .setOAuthConsumerKey(consumerKey)
+                .setOAuthConsumerSecret(consumerSecret)
+                .setOAuthAccessToken(acessToken)
+                .setOAuthAccessTokenSecret(acessTokenSecret);
+        TwitterStreamFactory tf = new TwitterStreamFactory(configurationBuilder.build());
+        TwitterStream twitterStream = tf.getInstance();
+        return twitterStream;
+	}
+	
+	public static void main(String[] args) {
+		// TODO Auto-generated method stub
+		TwitterStreaming twitter = new TwitterStreaming();
+		StatusListener listener = new StatusListener(){
+			@Override
+            public void onStatus(Status status) {
+                System.out.println("@" + status.getUser().getScreenName() + " - " + status.getText());
+            }
+
+            @Override
+            public void onDeletionNotice(StatusDeletionNotice statusDeletionNotice) {
+                System.out.println("Got a status deletion notice id:" + statusDeletionNotice.getStatusId());
+            }
+
+            @Override
+            public void onTrackLimitationNotice(int numberOfLimitedStatuses) {
+                System.out.println("Got track limitation notice:" + numberOfLimitedStatuses);
+            }
+
+            @Override
+            public void onScrubGeo(long userId, long upToStatusId) {
+                System.out.println("Got scrub_geo event userId:" + userId + " upToStatusId:" + upToStatusId);
+            }
+
+            @Override
+            public void onStallWarning(StallWarning warning) {
+                System.out.println("Got stall warning:" + warning);
+            }
+
+            @Override
+            public void onException(Exception ex) {
+                ex.printStackTrace();
+            }
+	    };
+	    TwitterStream twitterStream = twitter.getInstance();
+	    twitterStream.addListener(listener);
+	    // sample() method internally creates a thread which manipulates TwitterStream and calls these adequate listener methods continuously.
+	    twitterStream.sample("ko");
+	}
+
+}

src/main/python/kr/co/swh/lecture/opensource/twitter/twitter_crawler_basic.py

@@ -0,0 +1,25 @@
+# https://marcobonzanini.com/2015/03/02/mining-twitter-data-with-python-part-1/comment-page-1/
+import tweepy
+from tweepy import OAuthHandler
+import json
+
+# twitter 계정이 있다면 앱을 만들어서 등록
+# 그렇지 않을 시에는 데이터베이스에 있는 정보 등록
+consumer_key = 'xx'
+consumer_secret = 'xx'
+access_token = 'xx'
+access_secret = 'xx'
+
+auth = OAuthHandler(consumer_key, consumer_secret)
+auth.set_access_token(access_token, access_secret)
+api = tweepy.API(auth)
+
+def toJSON(tweet):
+    print(json.dumps(tweet, ensure_ascii=False))
+
+# 팔로잉 친구 목록
+for friend in tweepy.Cursor(api.friends).items():
+    toJSON(friend._json)
+# 트윗 및 답글 목록
+for tweet in tweepy.Cursor(api.user_timeline).items():
+    toJSON(tweet._json)

src/main/python/kr/co/swh/lecture/opensource/twitter/twitter_crawler_streaming.py

@@ -0,0 +1,45 @@
+# https://imasoftwareengineer.tistory.com/97?category=791547
+# conda install -c conda-forge tweepy
+import tweepy
+from tweepy.streaming import StreamListener
+from tweepy import OAuthHandler
+from tweepy import Stream
+import json
+
+# twitter 계정이 있다면 앱을 만들어서 등록
+# 그렇지 않을 시에는 데이터베이스에 있는 정보 등록
+consumer_key = 'xx'
+consumer_secret = 'xx'
+access_token = 'xx'
+access_secret = 'xx'
+
+# 리스너 - 스트리밍 API로 부터 값이 들어오면 아래의 리스너가 실행된다.
+class Listener(StreamListener):
+    def on_data(self, data):
+        try:
+            with open('sample.json', 'a', encoding='utf-8') as f:
+                json_data = json.loads(data)    # str -> json으로 변환
+                json.dump(json_data, f, ensure_ascii=False) # json을 원형 그대로 파일에 저장(인코딩 깨짐을 방지)
+                print(json_data)
+                return True
+        except BaseException as e:
+            print("Error on_data: %s" % str(e))
+        return True
+
+    def on_error(self, status):
+        print("ERROR: " + str(status))
+
+def main():
+    # 리스너 생성
+    l = Listener()
+    auth = OAuthHandler(consumer_key, consumer_secret)
+    auth.set_access_token(access_token, access_secret)
+    stream = Stream(auth, l) # 리스너와 인증키을 통해 스트리밍 객체 생성
+
+    # 언어=한국어(ko), 해시태그=필터할 단어
+    stream.filter(languages=["ko"], track=['손흥민'])
+
+if __name__ == '__main__':
+    main()
+
+main()

src/main/python/kr/co/swh/lecture/opensource/twitter/twitter_json_load.py

@@ -0,0 +1,6 @@
+import json
+ 
+with open('sample.json', 'r', encoding='utf-8') as f:
+    line = f.readline() # read only the first tweet/line
+    tweet = json.loads(line) # load it as Python dict
+    print(tweet)