Flink 消费 kafka 消息实战

version: '2'services:  zookeeper:    image: wurstmeister/zookeeper    ports:      - "2181:2181"  kafka1:    image: wurstmeister/kafka:2.11-0.11.0.3    ports:      - "9092:9092"    environment:      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka1:9092      KAFKA_LISTENERS: PLAINTEXT://:9092      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181      KAFKA_CREATE_TOPICS: "topic001:2:1"    volumes:      - /var/run/docker.sock:/var/run/docker.sock  producer:    image: bolingcavalry/kafka01103producer:0.0.1-SNAPSHOT    ports:      - "8080:8080"

mvn archetype:generate -DarchetypeGroupId=org.apache.flink -DarchetypeArtifactId=flink-quickstart-java -DarchetypeVersion=1.7.0

<dependency>  <groupId>org.apache.flink</groupId>    <artifactId>flink-connector-kafka-0.11_2.12</artifactId>    <version>${flink.version}</version></dependency>
<dependency>  <groupId>com.alibaba</groupId>  <artifactId>fastjson</artifactId>  <version>1.2.28</version></dependency>

/** * @Description: 解析原始消息的辅助类 * @author: willzhao E-mail: zq2599@gmail.com * @date: 2019/1/1 20:13 */public class JSONHelper {
    /**     * 解析消息，得到时间字段     * @param raw     * @return     */    public static long getTimeLongFromRawMessage(String raw){        SingleMessage singleMessage = parse(raw);        return null==singleMessage ? 0L : singleMessage.getTimeLong();    }
    /**     * 将消息解析成对象     * @param raw     * @return     */    public static SingleMessage parse(String raw){        SingleMessage singleMessage = null;
        if (raw != null) {            singleMessage = JSONObject.parseObject(raw, SingleMessage.class);        }
        return singleMessage;    }}

public class SingleMessage {
    private long timeLong;    private String name;    private String bizID;    private String time;    private String message;
    public long getTimeLong() {        return timeLong;    }
    public void setTimeLong(long timeLong) {        this.timeLong = timeLong;    }
    public String getName() {        return name;    }
    public void setName(String name) {        this.name = name;    }
    public String getBizID() {        return bizID;    }
    public void setBizID(String bizID) {        this.bizID = bizID;    }
    public String getTime() {        return time;    }
    public void setTime(String time) {        this.time = time;    }
    public String getMessage() {        return message;    }
    public void setMessage(String message) {        this.message = message;    }}

package com.bolingcavalry;
import org.apache.flink.api.common.functions.FlatMapFunction;import org.apache.flink.api.common.serialization.SimpleStringSchema;import org.apache.flink.api.java.tuple.Tuple;import org.apache.flink.api.java.tuple.Tuple2;import org.apache.flink.streaming.api.TimeCharacteristic;import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment;import org.apache.flink.streaming.api.functions.AssignerWithPunctuatedWatermarks;import org.apache.flink.streaming.api.functions.windowing.WindowFunction;import org.apache.flink.streaming.api.watermark.Watermark;import org.apache.flink.streaming.api.windowing.time.Time;import org.apache.flink.streaming.api.windowing.windows.TimeWindow;import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer011;
import javax.annotation.Nullable;import java.util.Properties;
/** * Skeleton for a Flink Streaming Job. * * <p>For a tutorial how to write a Flink streaming application, check the * tutorials and examples on the <a href="http://flink.apache.org/docs/stable/">Flink Website</a>. * * <p>To package your application into a JAR file for execution, run * 'mvn clean package' on the command line. * * <p>If you change the name of the main class (with the public static void main(String[] args)) * method, change the respective entry in the POM.xml file (simply search for 'mainClass'). */public class StreamingJob {
  public static void main(String[] args) throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();    env.enableCheckpointing(5000); // 要设置启动检查点    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime);
    Properties props = new Properties();    props.setProperty("bootstrap.servers", "kafka1:9092");    props.setProperty("group.id", "flink-group");
    //数据源配置，是一个kafka消息的消费者    FlinkKafkaConsumer011<String> consumer =        new FlinkKafkaConsumer011<>("topic001", new SimpleStringSchema(), props);
    //增加时间水位设置类    consumer.assignTimestampsAndWatermarks(new AssignerWithPunctuatedWatermarks<String> (){      @Override      public long extractTimestamp(String element, long previousElementTimestamp) {        return JSONHelper.getTimeLongFromRawMessage(element);      }
      @Nullable      @Override      public Watermark checkAndGetNextWatermark(String lastElement, long extractedTimestamp) {        if (lastElement != null) {          return new Watermark(JSONHelper.getTimeLongFromRawMessage(lastElement));        }        return null;      }    });
    env.addSource(consumer)        //将原始消息转成Tuple2对象，保留用户名称和访问次数(每个消息访问次数为1)        .flatMap((FlatMapFunction<String, Tuple2<String, Long>>) (s, collector) -> {          SingleMessage singleMessage = JSONHelper.parse(s);
          if (null != singleMessage) {            collector.collect(new Tuple2<>(singleMessage.getName(), 1L));          }        })        //以用户名为key        .keyBy(0)        //时间窗口为2秒        .timeWindow(Time.seconds(2))        //将每个用户访问次数累加起来        .apply((WindowFunction<Tuple2<String, Long>, Tuple2<String, Long>, Tuple, TimeWindow>) (tuple, window, input, out) -> {          long sum = 0L;          for (Tuple2<String, Long> record: input) {            sum += record.f1;          }
          Tuple2<String, Long> result = input.iterator().next();          result.f1 = sum;          out.collect(result);        })        //输出方式是STDOUT        .print();
    env.execute("Flink-Kafka demo");  }}

mvn clean package -Dmaven.test.skip=true -U

ab -n 10000 -c 2 http://192.168.1.101:8080/send/Jack/hello