diff --git a/src/kafka-connect/README.md b/src/kafka-connect/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1e2d921ad51f56062add89c54b6196129cfb7993 --- /dev/null +++ b/src/kafka-connect/README.md @@ -0,0 +1,30 @@ +<div align="center"> +<p> + <img width="480" src="https://www.andplus.com/hs-fs/hubfs/kafkalogo.jpg?&name=kafkalogo.jpg"> +</p> +<h1>Apache Kafka 공부 (Java, Python, Go)</h1> + <h5>v2.13-2.6.0</h5> + +[Apache Kafka](https://kafka.apache.org/) + +</div> + +## Twitter Kafka Connect + +[Github Link](https://github.com/jcustenborder/kafka-connect-twitter) + +1. 토픽 만들기 (파티션=3, 복제 계수=1) +```console +WIN10@DESKTOP:~$ kafka-topics --zookeeper localhost:2181 --topic twitter-status-connect --create --partitions 3 --replication-factor 1 +WIN10@DESKTOP:~$ kafka-topics --zookeeper localhost:2181 --topic twitter-delete-connect --create --partitions 3 --replication-factor 1 +``` + +2. 트위터 컨슈머 (프로듀서 작동할 때) +```console +WIN10@DESKTOP:~$ kafka-console-consumer --bootstrap-server localhost:9092 --topic twitter-status-connect --from-beginning +``` + +2. 트위터 프로듀서 +```console +WIN10@DESKTOP:~$ connect-standalone connect-standalone.properties twitter.properties +``` \ No newline at end of file diff --git a/src/kafka-connect/connect-standalone.properties b/src/kafka-connect/connect-standalone.properties new file mode 100644 index 0000000000000000000000000000000000000000..5690185576856f032bc9e3a6b57d8b53a47c47c4 --- /dev/null +++ b/src/kafka-connect/connect-standalone.properties @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# These are defaults. This file just demonstrates how to override some settings. +bootstrap.servers=localhost:9092 + +# The converters specify the format of data in Kafka and how to translate it into Connect data. Every Connect user will +# need to configure these based on the format they want their data in when loaded from or stored into Kafka +key.converter=org.apache.kafka.connect.json.JsonConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +# Converter-specific settings can be passed in by prefixing the Converter's setting with the converter we want to apply +# it to +key.converter.schemas.enable=true +value.converter.schemas.enable=true + +offset.storage.file.filename=/tmp/connect.offsets +# Flush much faster than normal, which is useful for testing/debugging +offset.flush.interval.ms=10000 + +# Set to a list of filesystem paths separated by commas (,) to enable class loading isolation for plugins +# (connectors, converters, transformations). The list should consist of top level directories that include +# any combination of: +# a) directories immediately containing jars with plugins and their dependencies +# b) uber-jars with plugins and their dependencies +# c) directories immediately containing the package directory structure of classes of plugins and their dependencies +# Note: symlinks will be followed to discover dependencies or plugins. +# Examples: +# plugin.path=/usr/local/share/java,/usr/local/share/kafka/plugins,/opt/connectors, +plugin.path=kafka-connect-twitter diff --git a/src/kafka-connect/kafka-connect-twitter/annotations-2.0.1.jar b/src/kafka-connect/kafka-connect-twitter/annotations-2.0.1.jar new file mode 100644 index 0000000000000000000000000000000000000000..8a0204fc28d78a15a848f8d2d4af467e52bb43c8 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/annotations-2.0.1.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/connect-utils-0.3.105.jar b/src/kafka-connect/kafka-connect-twitter/connect-utils-0.3.105.jar new file mode 100644 index 0000000000000000000000000000000000000000..7f3edaaf228ba763350bce9fa8be6ddd7b4966c4 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/connect-utils-0.3.105.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/freemarker-2.3.25-incubating.jar b/src/kafka-connect/kafka-connect-twitter/freemarker-2.3.25-incubating.jar new file mode 100644 index 0000000000000000000000000000000000000000..76be1c0e4391fa2018384bfaf597c6aed6de7ba1 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/freemarker-2.3.25-incubating.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/guava-18.0.jar b/src/kafka-connect/kafka-connect-twitter/guava-18.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..8f89e490180121500dfb1e65b3ebd8f1a7a4c3b1 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/guava-18.0.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/jackson-annotations-2.8.0.jar b/src/kafka-connect/kafka-connect-twitter/jackson-annotations-2.8.0.jar new file mode 100644 index 0000000000000000000000000000000000000000..d19b67b0f11086544c2dc2ab36a6b09abc9962a0 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/jackson-annotations-2.8.0.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/jackson-core-2.8.5.jar b/src/kafka-connect/kafka-connect-twitter/jackson-core-2.8.5.jar new file mode 100644 index 0000000000000000000000000000000000000000..05e1be2922e3c81aefdc5fea61a60f95f640cde4 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/jackson-core-2.8.5.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/jackson-databind-2.8.5.jar b/src/kafka-connect/kafka-connect-twitter/jackson-databind-2.8.5.jar new file mode 100644 index 0000000000000000000000000000000000000000..2794824673f4ee5061eb5576e7366e8a5b99263b Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/jackson-databind-2.8.5.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/javassist-3.19.0-GA.jar b/src/kafka-connect/kafka-connect-twitter/javassist-3.19.0-GA.jar new file mode 100644 index 0000000000000000000000000000000000000000..7ac42641f9f2806669e76fedbc52969e883ffea3 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/javassist-3.19.0-GA.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/kafka-connect-twitter-0.2.26.jar b/src/kafka-connect/kafka-connect-twitter/kafka-connect-twitter-0.2.26.jar new file mode 100644 index 0000000000000000000000000000000000000000..f3ce1104b0b4323b9aeb4d30d051270fda3898b5 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/kafka-connect-twitter-0.2.26.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/reflections-0.9.10.jar b/src/kafka-connect/kafka-connect-twitter/reflections-0.9.10.jar new file mode 100644 index 0000000000000000000000000000000000000000..c255641827d2303aa0727cac66531df1efcc54b5 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/reflections-0.9.10.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/twitter4j-core-4.0.6.jar b/src/kafka-connect/kafka-connect-twitter/twitter4j-core-4.0.6.jar new file mode 100644 index 0000000000000000000000000000000000000000..a1f86cce16d46ce0af4189ddd4b315b4d3d58641 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/twitter4j-core-4.0.6.jar differ diff --git a/src/kafka-connect/kafka-connect-twitter/twitter4j-stream-4.0.6.jar b/src/kafka-connect/kafka-connect-twitter/twitter4j-stream-4.0.6.jar new file mode 100644 index 0000000000000000000000000000000000000000..f6e4ccc00d31be6e5d7ebe89fe2768814503d728 Binary files /dev/null and b/src/kafka-connect/kafka-connect-twitter/twitter4j-stream-4.0.6.jar differ diff --git a/src/kafka-connect/twitter.properties b/src/kafka-connect/twitter.properties new file mode 100644 index 0000000000000000000000000000000000000000..743aed2e3adaa3988751e831442a54ef04b3cbc5 --- /dev/null +++ b/src/kafka-connect/twitter.properties @@ -0,0 +1,15 @@ +# FROM: https://github.com/jcustenborder/kafka-connect-twitter + +name=TwitterSourceDemo +tasks.max=1 +connector.class=com.github.jcustenborder.kafka.connect.twitter.TwitterSourceConnector + +# Set these required values +process.deletes=false +filter.keywords=kafka +kafka.status.topic=twitter-status-connect +kafka.delete.topic=twitter-delete-connect +twitter.oauth.accessToken= +twitter.oauth.accessTokenSecret= +twitter.oauth.consumerKey= +twitter.oauth.consumerSecret= \ No newline at end of file