ITPub博客

首页 > 大数据 > 数据分析 > Flink 的几个核心概念相关的源码实现我们已经了解了

Flink 的几个核心概念相关的源码实现我们已经了解了

投稿 数据分析 作者:ecclw459 时间:2026-01-30 18:09:47 0 删除 编辑

StreamMap.processElement  方法中,调用完 map 方法之后,就会调用  output.collect  方法将数据输出,这里的 output 就是 RecordWriterOutput。在 RecordWriterOutput 中,会调用 RecordWriter 的 emit 方法。


private <X> void pushToRecordWriter (StreamRecord<X> record) {    serializationDelegate.setInstance(record);     try {        recordWriter.emit(serializationDelegate);    } catch (IOException e) {         throw new UncheckedIOException (e.getMessage(), e);


这里的 serializationDelegate 是用来对 record 进行序列化的。RecordWriter 有两个实现类,一个是 ChannelSelectorRecordWriter,另一个是 BroadcastRecordWriter。ChannelSelectorRecordWriter 需要先调用 ChannelSelector 选择对应的 subparition,然后进行写入。BroadcastRecordWriter 则是写到所有的 subparition。


public void emitRecord (ByteBuffer record, int targetSubpartition) throws IOException {    totalWrittenBytes += record.remaining();     BufferBuilder buffer = appendUnicastDataForNewRecord(record, targetSubpartition);     while (record.hasRemaining()) {         // full buffer, partial record        finishUnicastBufferBuilder(targetSubpartition);        buffer = appendUnicastDataForRecordContinuation(record, targetSubpartition);    }     if (buffer.isFull()) {         // full buffer, full record        finishUnicastBufferBuilder(targetSubpartition);    }


private MemorySegment requestMemorySegment ( int targetChannel) {     MemorySegment segment = null ;     synchronized (availableMemorySegments) {        checkDestroyed();         if (!availableMemorySegments.isEmpty()) {            segment = availableMemorySegments.poll();        } else if (isRequestedSizeReached()) {             // Only when the buffer request reaches the upper limit(i.e. current pool size),             // requests an overdraft buffer.            segment = requestOverdraftMemorySegmentFromGlobal();        }         if (segment == null ) {             return null ;        }         if (targetChannel != UNKNOWN_CHANNEL) {             if (++subpartitionBuffersCount[targetChannel] == maxBuffersPerChannel) {                unavailableSubpartitionsCount++;


private void addToSubpartition (        BufferBuilder buffer,         int targetSubpartition,         int partialRecordLength,         int minDesirableBufferSize)  q8m.ttlyf.cn         throws IOException {     int desirableBufferSize =            subpartitions[targetSubpartition].add(                    buffer.createBufferConsumerFromBeginning(), partialRecordLength);


public DataInputStatus emitNext (DataOutput<T> output) throws Exception {     while ( true ) {         // get the stream element from the deserializer    ablzyu.ttlyf.cn           if (currentRecordDeserializer != null ) {            RecordDeserializer.DeserializationResult result;             try {                result = currentRecordDeserializer.getNextRecord(deserializationDelegate);            } catch (IOException e) {                 throw new IOException (                        String.format( "Can't get next record for channel %s" , lastChannel), e);            }             if (result.isBufferConsumed()) {                currentRecordDeserializer = null ;            }             if (result.isFullRecord()) {                 final boolean breakBatchEmitting =                        processElement(deserializationDelegate.getInstance(), output);                 if (canEmitBatchOfRecords.check() && !breakBatchEmitting) {                     continue ;                }  hnbhvd.ttlyf.cn                 return DataInputStatus.MORE_AVAILABLE;            }        }        Optional<BufferOrEvent> bufferOrEvent = checkpointedInputGate.pollNext();         if (bufferOrEvent.isPresent()) {             // return to the mailbox after receiving a checkpoint barrier to avoid processing of             // data after the barrier before checkpoint is performed for unaligned checkpoint             // mode             if (bufferOrEvent.get().isBuffer()) {                processBuffer(bufferOrEvent.get());            } else {                 DataInputStatus status = processEvent(bufferOrEvent.get(), output);                 if (status == DataInputStatus.MORE_AVAILABLE && canEmitBatchOfRecords.check()) {                     continue ;                }                 return status;dcydqe.ttlyf.cn            }        } else {             if (checkpointedInputGate.isFinished()) {                checkState(                        checkpointedInputGate.getAvailableFuture().isDone(),                         "Finished BarrierHandler should be available" );      m.junpin001.com             return DataInputStatus.END_OF_INPUT;


public Optional<BufferAndAvailability> getNextBuffer () throws IOException {    checkError();     if (!toBeConsumedBuffers.isEmpty()) {         return getBufferAndAvailability(toBeConsumedBuffers.removeFirst());    }     ResultSubpartitionView subpartitionView = this .subpartitionView;     if (subpartitionView == null ) {         // There is a possible race condition between writing a EndOfPartitionEvent (1) and         // flushing (3) the Local         // channel on the sender side, and reading EndOfPartitionEvent (2) and processing flush         // notification (4). When         // they happen in that order (1 - 2 - 3 - 4), flush notification can re-enqueue         // LocalInputChannel after (or         // during) it was released during reading the EndOfPartitionEvent (2).         if (isReleased) {             return Optional.empty();wap.gz-lzd.com        }         // this can happen if the request for the partition was triggered asynchronously         // by the time trigger   eyisoa.ttlyf.cn         // would be good to avoid that, by guaranteeing that the requestPartition() and         // getNextBuffer() always come from the same thread         // we could do that by letting the timer insert a special "requesting channel" into the         // input gate's queue   ar83xxm.ttlyf.cn        subpartitionView = checkAndWaitForSubpartitionView();    }     BufferAndBacklog next = subpartitionView.getNextBuffer();     // ignore the empty buffer directly     while (next != null && next.buffer().readableBytes() == 0 ) {        next.buffer().recycleBuffer();        next = subpartitionView.getNextBuffer();        numBuffersIn.inc();dsmtcy.ttlyf.cn    }     if (next == null ) {         if (subpartitionView.isReleased()) {             throw new CancelTaskException (                     "Consumed partition " + subpartitionView + " has been released." );        } else {             return Optional.empty();cs.ttlyf.cn        }xfnz.ttlyf.cn      }     Buffer buffer = next.buffer();     if (buffer instanceof FullyFilledBuffer) {        List<Buffer> partialBuffers = ((FullyFilledBuffer) buffer).getPartialBuffers();         int seq = next.getSequenceNumber();         for (Buffer partialBuffer : partialBuffers) {            toBeConsumedBuffers.add(     mwdvqy.ttlyf.cn                new BufferAndBacklog (                            partialBuffer,  junpin001.com                            next.buffersInBacklog(),                            buffer.getDataType(),                            seq++));  jrszb.ttLyf.cn        }





来自 “ ITPUB博客 ” ,链接:https://blog.itpub.net/70048957/viewspace-3109573/ ,如需转载,请注明出处,否则将追究法律责任。

请登录后发表评论 登录
全部评论

注册时间:2026-01-07

  • 博文量
    108
  • 访问量
    102108